use crate::lazy::dsl::JsExpr; use crate::prelude::*; use napi::bindgen_prelude::*; use polars_arrow::temporal_conversions::time64ns_to_time; use polars_core::config::verbose_print_sensitive; use polars_core::series::ops::NullBehavior; use polars_dtype::categorical::{CatSize, FrozenCategories}; use polars_io::cloud::{CloudOptions, CloudRetryConfig}; use polars_io::utils::sync_on_close::SyncOnCloseType; use polars_io::ExternalCompression; use polars_io::RowIndex; use polars_utils::compression::{BrotliLevel, GzipLevel, ZstdLevel}; use polars_utils::total_ord::TotalOrdWrap; use std::collections::HashMap; use std::num::NonZero; #[derive(Debug)] pub struct Wrap(pub T); impl Clone for Wrap where T: Clone, { fn clone(&self) -> Self { Wrap(self.0.clone()) } } impl From for Wrap { fn from(t: T) -> Self { Wrap(t) } } impl TypeName for Wrap { fn type_name() -> &'static str { "StringChunked" } fn value_type() -> ValueType { ValueType::Object } } pub(crate) fn bigint_to_i64(big: &BigInt) -> JsResult { let (val, lossless) = big.get_i64(); if !lossless { return Err(napi::Error::from_reason( "BigInt value is out of range for Int64".to_owned(), )); } Ok(val) } pub(crate) fn bigint_to_u64(big: &BigInt) -> JsResult { let (sign_bit, val, lossless) = big.get_u64(); if sign_bit { return Err(napi::Error::from_reason( "Cannot convert negative BigInt to UInt64".to_owned(), )); } if !lossless { return Err(napi::Error::from_reason( "BigInt value is out of range for UInt64".to_owned(), )); } Ok(val) } pub(crate) fn bigint_to_anyvalue(big: &BigInt) -> JsResult> { if big.sign_bit { bigint_to_i64(big).map(AnyValue::Int64) } else { bigint_to_u64(big).map(AnyValue::UInt64) } } pub(crate) fn bigint_to_f64(big: &BigInt) -> f64 { let mut mag = 0.0_f64; for &word in big.words.iter().rev() { mag = mag * 18446744073709551616.0 + (word as f64); } if big.sign_bit { -mag } else { mag } } pub(crate) fn any_values_to_series(name: &str, mut values: Vec) -> Series { let mut has_i64 = false; let mut has_u64 = false; let mut all_i64_compatible = true; for av in &values { match av { AnyValue::Null => {} AnyValue::Int64(_) => has_i64 = true, AnyValue::UInt64(u) => { has_u64 = true; if *u > i64::MAX as u64 { all_i64_compatible = false; } } _ => { all_i64_compatible = false; break; } } } if has_i64 && has_u64 && all_i64_compatible { for av in &mut values { if let AnyValue::UInt64(u) = *av { *av = AnyValue::Int64(u as i64); } } } Series::new(name.into(), values) } /// Safety. /// it is up to the consumer to make sure the item is valid pub(crate) trait ToSeries { fn to_series(&self, name: &str) -> Series; } impl ToSeries for Array<'_> { fn to_series(&self, name: &str) -> Series { let len = self.len(); let mut v: Vec = Vec::with_capacity(len as usize); for i in 0..len { let av: Wrap = self.get(i).ok().flatten().unwrap_or(Wrap(AnyValue::Null)); v.push(av.0); } any_values_to_series(name, v) } } impl ToSeries for Unknown<'_> { fn to_series(&self, name: &str) -> Series { let obj = unsafe { self.cast::() }; let obj = match obj.as_ref() { Ok(obj) => obj, Err(_) => return Series::new(name.into(), Vec::::new()), }; let len = obj.get_array_length_unchecked().unwrap_or(0); let mut v: Vec = Vec::with_capacity(len as usize); for i in 0..len { let av = obj .get_element::(i) .ok() .and_then(|unknown| AnyValue::from_js(unknown).ok()) .unwrap_or(AnyValue::Null); v.push(av); } any_values_to_series(name, v) } } impl ToNapiValue for Wrap<&Series> { unsafe fn to_napi_value(napi_env: sys::napi_env, val: Self) -> napi::Result { let s = val.0.rechunk(); let len = s.len(); let dtype = s.dtype(); let env = Env::from_raw(napi_env); match dtype { DataType::Struct(_) => { let ca = s.struct_().map_err(JsPolarsErr::from)?; let df: DataFrame = ca.clone().unnest(); let (height, _) = df.shape(); let mut rows = env.create_array(height as u32)?; for idx in 0..height { let mut row = Object::new(&env)?; for col in df.columns() { let key = col.name(); let cell_value = col.get(idx).or_else(|_| { Err(napi::Error::from_reason(format!( "Failed to get value at index {} in column {}", idx, key ))) })?; row.set(key, Wrap(cell_value))?; } rows.set(idx as u32, row)?; } Array::to_napi_value(napi_env, rows) } _ => { let mut arr = env.create_array(len as u32)?; for (idx, cell_value) in s.iter().enumerate() { arr.set(idx as u32, Wrap(cell_value))?; } Array::to_napi_value(napi_env, arr) } } } } impl<'a> ToNapiValue for Wrap> { unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { let to_js_date = |epoch_time: f64| -> Result { let mut ptr = std::ptr::null_mut(); check_status!( napi::sys::napi_create_date(env, epoch_time, &mut ptr), "Failed to convert rust datetime/date into napi value", )?; Ok(ptr) }; let unsupported = |dtype: &str| { napi::Error::from_reason(format!( "{dtype} is not supported, please convert to string or number before collecting to js" )) }; match val.0 { AnyValue::Null => { napi::bindgen_prelude::Null::to_napi_value(env, napi::bindgen_prelude::Null) } AnyValue::Boolean(b) => bool::to_napi_value(env, b), AnyValue::Int8(n) => i32::to_napi_value(env, n as i32), AnyValue::Int16(n) => i32::to_napi_value(env, n as i32), AnyValue::Int32(n) => i32::to_napi_value(env, n), AnyValue::Int64(n) => i64::to_napi_value(env, n), AnyValue::Int128(n) => i128::to_napi_value(env, n), AnyValue::UInt8(n) => u32::to_napi_value(env, n as u32), AnyValue::UInt16(n) => u32::to_napi_value(env, n as u32), AnyValue::UInt32(n) => u32::to_napi_value(env, n), AnyValue::UInt64(n) => u64::to_napi_value(env, n), AnyValue::UInt128(n) => u128::to_napi_value(env, n), AnyValue::Float16(n) => f64::to_napi_value(env, f64::from(n)), AnyValue::Float32(n) => f64::to_napi_value(env, f64::from(n)), AnyValue::Float64(n) => f64::to_napi_value(env, n), AnyValue::String(s) => String::to_napi_value(env, s.to_owned()), AnyValue::StringOwned(s) => String::to_napi_value(env, s.to_string()), // Convert Date (days) to epoch milliseconds expected by JS Date. AnyValue::Date(v) => to_js_date((v as f64) * 86400000.0), AnyValue::Datetime(v, time_unit, _) | AnyValue::DatetimeOwned(v, time_unit, _) => { let divisor = match time_unit { TimeUnit::Milliseconds => 1.0, TimeUnit::Microseconds => 1_000.0, TimeUnit::Nanoseconds => 1_000_000.0, }; // JS Date stores milliseconds since Unix epoch. to_js_date((v as f64) / divisor) } AnyValue::Categorical(cat, &ref lmap) | AnyValue::CategoricalOwned(cat, ref lmap) => { let s = unsafe { lmap.cat_to_str_unchecked(cat) }; String::to_napi_value(env, s.to_string()) } AnyValue::Enum(cat, &ref lmap) | AnyValue::EnumOwned(cat, ref lmap) => { let s = unsafe { lmap.cat_to_str_unchecked(cat) }; String::to_napi_value(env, s.to_string()) } AnyValue::Duration(v, _) => i64::to_napi_value(env, v), AnyValue::Time(v) => { String::to_napi_value(env, time64ns_to_time(v).format("%T%.f").to_string()) } AnyValue::List(ser) => Wrap::<&Series>::to_napi_value(env, Wrap(&ser)), ref av @ AnyValue::Struct(_, _, flds) => struct_dict(env, av._iter_struct_av(), flds), AnyValue::Array(ser, _) => Wrap::<&Series>::to_napi_value(env, Wrap(&ser)), AnyValue::Object(_) => Err(unsupported("Object")), AnyValue::ObjectOwned(_) => Err(unsupported("ObjectOwned")), AnyValue::StructOwned(_) => Err(unsupported("StructOwned")), AnyValue::Binary(_) => Err(unsupported("Binary")), AnyValue::BinaryOwned(_) => Err(unsupported("BinaryOwned")), AnyValue::Decimal(_, _, _) => Err(unsupported("Decimal")), } } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::(i as u32) { Ok(val) => match val { Some(str_val) => builder.append_value(str_val), None => builder.append_null(), }, Err(_) => builder.append_null(), } } Ok(Wrap(builder.finish())) } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = BooleanChunkedBuilder::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::(i as u32) { Ok(val) => match val { Some(str_val) => builder.append_value(str_val), None => builder.append_null(), }, Err(_) => builder.append_null(), } } Ok(Wrap(builder.finish())) } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = PrimitiveChunkedBuilder::::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::(i as u32) { Ok(val) => match val { Some(v) => builder.append_value(v as f32), None => builder.append_null(), }, Err(_) => builder.append_null(), } } Ok(Wrap(builder.finish())) } } macro_rules! impl_chunked { ($type:ty, $native:ty) => { impl FromNapiValue for Wrap> { unsafe fn from_napi_value( env: sys::napi_env, napi_val: sys::napi_value, ) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = PrimitiveChunkedBuilder::<$type>::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::<$native>(i as u32) { Ok(val) => match val { Some(v) => builder.append_value(v), None => builder.append_null(), }, Err(_) => builder.append_null(), } } Ok(Wrap(builder.finish())) } } }; } impl_chunked!(Float64Type, f64); impl_chunked!(Int32Type, i32); impl_chunked!(UInt32Type, u32); impl FromNapiValue for Wrap> { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = PrimitiveChunkedBuilder::::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::(i as u32)? { Some(val) => match val.get_type()? { ValueType::Null | ValueType::Undefined => builder.append_null(), ValueType::Number => { let v: i64 = unsafe { val.cast()? }; builder.append_value(v); } ValueType::BigInt => { let big: BigInt = unsafe { val.cast()? }; builder.append_value(bigint_to_i64(&big)?); } dt => { return Err(napi::Error::from_reason(format!( "Cannot convert {dt:?} to Int64 at index {i}" ))); } }, None => builder.append_null(), } } Ok(Wrap(builder.finish())) } } impl FromNapiValue for Wrap> { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let arr = Array::from_napi_value(env, napi_val)?; let len = arr.len() as usize; let mut builder = PrimitiveChunkedBuilder::::new(PlSmallStr::EMPTY, len); for i in 0..len { match arr.get::(i as u32)? { Some(val) => match val.get_type()? { ValueType::Null | ValueType::Undefined => builder.append_null(), ValueType::BigInt => { let big: BigInt = unsafe { val.cast()? }; builder.append_value(bigint_to_u64(&big)?); } ValueType::Number => { let n: f64 = unsafe { val.cast()? }; if n < 0.0 || !n.is_finite() { return Err(napi::Error::from_reason( "Cannot convert negative or non-finite Number to UInt64".to_owned(), )); } builder.append_value(n as u64); } dt => { return Err(napi::Error::from_reason(format!( "Cannot convert {dt:?} to UInt64 at index {i}" ))); } }, None => builder.append_null(), } } Ok(Wrap(builder.finish())) } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let obj = Object::from_napi_value(env, napi_val)?; let expr: &JsExpr = obj .get("_expr")? .ok_or_else(|| napi::Error::from_reason("field '_expr' should exist".to_owned()))?; Ok(Wrap(expr.inner.clone())) } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let obj = Object::from_napi_value(env, napi_val)?; let expr: &JsExpr = obj .get("_expr")? .ok_or_else(|| napi::Error::from_reason("field '_expr' should exist".to_owned()))?; Ok(Wrap(expr.clone())) } } impl TypeName for Wrap { fn type_name() -> &'static str { "QuantileMethod" } fn value_type() -> ValueType { ValueType::Object } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let interpolation = String::from_napi_value(env, napi_val)?; let interpol = match interpolation.as_ref() { "nearest" => QuantileMethod::Nearest, "lower" => QuantileMethod::Lower, "higher" => QuantileMethod::Higher, "midpoint" => QuantileMethod::Midpoint, "linear" => QuantileMethod::Linear, v => { return Err(napi::Error::from_reason(format!( "`interpolation` must be one of {{'nearest', 'lower', 'higher', 'midpoint', 'linear'}}, got {v}", ))) } }; Ok(Wrap(interpol)) } } impl FromNapiValue for Wrap { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> JsResult { let start = String::from_napi_value(env, napi_val)?; let parsed = match start.as_ref() { "window" => StartBy::WindowBound, "datapoint" => StartBy::DataPoint, "monday" => StartBy::Monday, v => { return Err(napi::Error::from_reason(format!( "`startBy` must be one of {{'window', 'datapoint', 'monday'}}, got {v}", ))) } }; Ok(Wrap(parsed)) } } impl FromNapiValue for Wrap