diff --git a/AGENTS.md b/AGENTS.md index 569fac7cc42..6ba6006251f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,11 +64,14 @@ Examples: cargo nextest run -p vortex-array make -C docs doctest uv run --all-packages pytest vortex-python/test +cargo test --doc ``` Run docs doctests from the docs directory with `make -C docs doctest` so the correct Sphinx Makefile target is used. +If you touch documentation run doc tests via `cargo test --doc`. + ## Linting, Formatting, and Generated Files Run verification that matches the files changed. Do not run expensive Rust checks for changes that diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index 45adf5a46cd..71b51ecf9af 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -6,7 +6,6 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; @@ -17,41 +16,29 @@ use crate::alp::ALP; impl CastReduce for ALP { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Check if this is just a nullability change - if array.dtype().eq_ignore_nullability(dtype) { - // For nullability-only changes, we can avoid decoding - // Cast the encoded array (integers) to handle nullability - let new_encoded = array.encoded().cast( - array - .encoded() - .dtype() - .with_nullability(dtype.nullability()), - )?; - - let new_patches = array - .patches() - .map(|p| { - if p.values().dtype() == dtype { - Ok(p) - } else { - Patches::new( - p.array_len(), - p.offset(), - p.indices().clone(), - p.values().cast(dtype.clone())?, - p.chunk_offsets().clone(), - ) - } - }) - .transpose()?; - - // SAFETY: casting nullability doesn't alter the invariants - unsafe { - Ok(Some( - ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(), - )) - } - } else { - Ok(None) + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); + } + + // For nullability-only changes, we can avoid decoding + // Cast the encoded array (integers) to handle nullability + let new_encoded = array.encoded().cast( + array + .encoded() + .dtype() + .with_nullability(dtype.nullability()), + )?; + + let new_patches = array + .patches() + .map(|p| p.map_values(|v| v.cast(dtype.clone()))) + .transpose()?; + + // SAFETY: casting nullability doesn't alter the invariants + unsafe { + Ok(Some( + ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(), + )) } } } diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 9086877c2ce..c110475b880 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; @@ -16,38 +14,35 @@ use crate::alp_rd::ALPRD; impl CastReduce for ALPRD { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - // ALPRDArray stores floating-point values, so only cast between float types - // or if just changing nullability - // Check if this is just a nullability change - if array.dtype().eq_ignore_nullability(dtype) { - // For nullability-only changes, we need to cast the left_parts array - // since it carries the validity information - let new_left_parts = array.left_parts().cast( - array - .left_parts() - .dtype() - .with_nullability(dtype.nullability()), - )?; + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); + } + + // For nullability-only changes, we need to cast the left_parts array + // since it carries the validity information + let new_left_parts = array.left_parts().cast( + array + .left_parts() + .dtype() + .with_nullability(dtype.nullability()), + )?; - // NOTE: `CastReduce::cast` has a fixed trait signature without `ExecutionCtx`, so we - // construct a legacy ctx locally at this trait boundary. - return Ok(Some( - ALPRD::try_new( + // NOTE: `CastReduce::cast` has a fixed trait signature without `ExecutionCtx`, so we + // construct a legacy ctx locally at this trait boundary. + Ok(Some( + unsafe { + ALPRD::new_unchecked( dtype.clone(), new_left_parts, array.left_parts_dictionary().clone(), array.right_parts().clone(), array.right_bit_width(), array.left_parts_patches(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? - .into_array(), - )); - } - - // For other casts (e.g., f32 to f64), decode to canonical and let PrimitiveArray handle it - Ok(None) + ) + } + .into_array(), + )) } } @@ -99,12 +94,17 @@ mod tests { let encoder = RDEncoder::new(&values); let alprd = encoder.encode(arr.as_view(), &mut ctx); - // Cast to NonNullable should fail since we have nulls + // Cast to NonNullable should fail since we have nulls. The failure surfaces during + // execution since the reduce path defers when the validity stat is not cached. let result = alprd .clone() .into_array() - .cast(DType::Primitive(PType::F64, Nullability::NonNullable)); - assert!(result.is_err()); + .cast(DType::Primitive(PType::F64, Nullability::NonNullable)) + .and_then(|a| { + a.execute::(&mut ctx) + .map(|p| p.into_array()) + }); + assert!(result.is_err(), "Expected error, got: {result:?}"); // Cast to same type with Nullable should succeed let casted = alprd diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 53c71e3bfb6..e9f4944310f 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -64,6 +64,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_bytebool::ByteBool + +pub fn vortex_bytebool::ByteBool::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 609a7bc553f..629a7ebe3e2 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::dict::TakeExecute; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::match_each_integer_ptype; +use vortex_array::scalar_fn::fns::cast::CastKernel; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_array::validity::Validity; @@ -24,20 +25,43 @@ impl CastReduce for ByteBool { // ByteBool is essentially a bool array stored as bytes // The main difference from BoolArray is the storage format // For casting, we can decode to canonical (BoolArray) and let it handle the cast - // If just changing nullability, we can optimize - if array.dtype().eq_ignore_nullability(dtype) { - let new_validity = array - .validity()? - .cast_nullability(dtype.nullability(), array.len())?; + if !dtype.is_boolean() { + return Ok(None); + } + + let Some(new_validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; + + Ok(Some( + ByteBool::new(array.buffer().clone(), new_validity).into_array(), + )) + } +} - return Ok(Some( - ByteBool::new(array.buffer().clone(), new_validity).into_array(), - )); +impl CastKernel for ByteBool { + fn cast( + array: ArrayView<'_, Self>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // Only handle nullability changes here; non-bool targets fall through to canonicalization. + if !dtype.is_boolean() { + return Ok(None); } - // For other casts, decode to canonical and let BoolArray handle it - Ok(None) + let new_validity = + array + .validity()? + .cast_nullability(dtype.nullability(), array.len(), ctx)?; + + Ok(Some( + ByteBool::new(array.buffer().clone(), new_validity).into_array(), + )) } } diff --git a/encodings/bytebool/src/kernel.rs b/encodings/bytebool/src/kernel.rs index 4373520039a..ddf9679b3d5 100644 --- a/encodings/bytebool/src/kernel.rs +++ b/encodings/bytebool/src/kernel.rs @@ -3,8 +3,11 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::kernel::ParentKernelSet; +use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; use crate::ByteBool; -pub(crate) const PARENT_KERNELS: ParentKernelSet = - ParentKernelSet::new(&[ParentKernelSet::lift(&TakeExecuteAdaptor(ByteBool))]); +pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(ByteBool)), + ParentKernelSet::lift(&TakeExecuteAdaptor(ByteBool)), +]); diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index b313a165b79..882d07bbaef 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -7,39 +7,31 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; use crate::decimal_byte_parts::DecimalBytePartsArrayExt; + impl CastReduce for DecimalByteParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { + // Check if this is just a nullability change + if !dtype.eq_ignore_nullability(array.dtype()) { + return Ok(None); + } // DecimalBytePartsArray can only have Decimal dtype, so we only handle decimal-to-decimal casts let DType::Decimal(target_decimal, target_nullability) = dtype else { // Cannot cast decimal to non-decimal types - delegate to canonical form return Ok(None); }; - // Check if this is just a nullability change - if array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype") - == target_decimal - && array.dtype().nullability() != *target_nullability - { - // Cast the msp array to handle nullability change - let new_msp = array - .msp() - .cast(array.msp().dtype().with_nullability(*target_nullability))?; - - return Ok(Some( - DecimalByteParts::try_new(new_msp, *target_decimal)?.into_array(), - )); - } + // Cast the msp array to handle nullability change + let new_msp = array + .msp() + .cast(array.msp().dtype().with_nullability(*target_nullability))?; - // For precision/scale changes, decode to canonical and let DecimalArray handle it - Ok(None) + Ok(Some( + DecimalByteParts::try_new(new_msp, *target_decimal)?.into_array(), + )) } } diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 59dfbb4ea4f..1f800cf8b22 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -186,6 +186,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fastlanes::BitPacked + +pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 9f23750398a..75b75a51e99 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -334,19 +334,25 @@ impl> BitPackedArrayExt for T {} #[cfg(test)] mod test { + use std::sync::LazyLock; + use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::session::ArraySession; use vortex_buffer::Buffer; + use vortex_session::VortexSession; use crate::BitPackedData; use crate::bitpacking::array::BitPackedArrayExt; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_encode() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = [ Some(1u64), None, @@ -369,7 +375,7 @@ mod test { #[test] fn test_encode_too_wide() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = [Some(1u8), None, Some(1), None, Some(1), None]; let uncompressed = PrimitiveArray::from_option_iter(values); let _packed = BitPackedData::encode(&uncompressed.clone().into_array(), 8, &mut ctx) @@ -380,7 +386,7 @@ mod test { #[test] fn signed_with_patches() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values: Buffer = (0i32..=512).collect(); let parray = values.clone().into_array(); diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 8843c8e3013..86bd701e2ee 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -3,48 +3,67 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::patches::Patches; +use vortex_array::scalar_fn::fns::cast::CastKernel; use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use crate::bitpacking::BitPacked; use crate::bitpacking::array::BitPackedArrayExt; + +fn build_with_validity( + array: ArrayView<'_, BitPacked>, + dtype: &DType, + new_validity: Validity, +) -> VortexResult { + Ok(BitPacked::try_new( + array.packed().clone(), + dtype.as_ptype(), + new_validity, + array + .patches() + .map(|patches| patches.map_values(|values| values.cast(dtype.clone()))) + .transpose()?, + array.bit_width(), + array.len(), + array.offset(), + )? + .into_array()) +} + impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - if array.dtype().eq_ignore_nullability(dtype) { - let new_validity = array - .validity()? - .cast_nullability(dtype.nullability(), array.len())?; - return Ok(Some( - BitPacked::try_new( - array.packed().clone(), - dtype.as_ptype(), - new_validity, - array - .patches() - .map(|patches| { - let new_values = patches.values().cast(dtype.clone())?; - Patches::new( - patches.array_len(), - patches.offset(), - patches.indices().clone(), - new_values, - patches.chunk_offsets().clone(), - ) - }) - .transpose()?, - array.bit_width(), - array.len(), - array.offset(), - )? - .into_array(), - )); + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); } + let Some(new_validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; + build_with_validity(array, dtype, new_validity).map(Some) + } +} - Ok(None) +impl CastKernel for BitPacked { + fn cast( + array: ArrayView<'_, Self>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); + } + let new_validity = + array + .validity()? + .cast_nullability(dtype.nullability(), array.len(), ctx)?; + build_with_validity(array, dtype, new_validity).map(Some) } } diff --git a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs index 128ff77d99c..cf975602179 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs @@ -4,10 +4,12 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::kernel::ParentKernelSet; +use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; use crate::BitPacked; pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(BitPacked)), ParentKernelSet::lift(&FilterExecuteAdaptor(BitPacked)), ParentKernelSet::lift(&TakeExecuteAdaptor(BitPacked)), ]); diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 9289e0a83fc..c11a6c4ec51 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -153,18 +153,6 @@ impl VTable for BitPacked { } } - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } - - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - BitPackedSlots::NAMES[idx].to_string() - } - fn serialize( array: ArrayView<'_, Self>, _session: &VortexSession, @@ -283,6 +271,10 @@ impl VTable for BitPacked { }) } + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + BitPackedSlots::NAMES[idx].to_string() + } + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { require_patches!( array, @@ -305,6 +297,14 @@ impl VTable for BitPacked { ) -> VortexResult> { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } } #[derive(Clone, Debug)] diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 4beb558dca0..324e2fc9c22 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -9,24 +9,17 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use vortex_error::vortex_panic; use crate::delta::Delta; use crate::delta::array::DeltaArrayExt; + impl CastReduce for Delta { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - // Delta encoding stores differences between consecutive values, which requires - // unsigned integers to avoid overflow issues. Signed integers could produce - // negative deltas that wouldn't fit in the unsigned delta representation. - // This encoding is optimized for monotonically increasing sequences. let DType::Primitive(target_ptype, _) = dtype else { return Ok(None); }; - let DType::Primitive(source_ptype, _) = array.dtype() else { - vortex_panic!("delta should be primitive typed"); - }; - + let source_ptype = array.dtype().as_ptype(); // TODO(DK): narrows can be safe but we must decompress to compute the maximum value. if target_ptype.is_signed_int() || source_ptype.bit_width() > target_ptype.bit_width() { return Ok(None); diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index bb5add65aa0..17e571ba4f6 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -39,6 +39,7 @@ use crate::delta::array::DeltaArrayExt; use crate::delta::array::SLOT_NAMES; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta::array::lane_count; +use crate::delta_compress; mod operations; mod rules; @@ -200,7 +201,7 @@ impl Delta { ctx: &mut ExecutionCtx, ) -> VortexResult { let logical_len = array.len(); - let (bases, deltas) = crate::delta::array::delta_compress::delta_compress(array, ctx)?; + let (bases, deltas) = delta_compress(array, ctx)?; Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) } } diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 23231b10b12..e01d359e59b 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use crate::rle::RLE; use crate::rle::RLEArrayExt; + impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast RLE values. @@ -20,14 +21,12 @@ impl CastReduce for RLE { .cast(DType::Primitive(dtype.as_ptype(), Nullability::NonNullable))?; // Cast RLE indices such that validity matches the target dtype. - let casted_indices = if array.indices().dtype().nullability() != dtype.nullability() { - array.indices().cast(DType::Primitive( - array.indices().dtype().as_ptype(), - dtype.nullability(), - ))? - } else { - array.indices().clone() - }; + let casted_indices = array.indices().cast( + array + .indices() + .dtype() + .with_nullability(dtype.nullability()), + )?; Ok(Some( RLE::try_new( @@ -44,6 +43,8 @@ impl CastReduce for RLE { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::Canonical; use vortex_array::ExecutionCtx; @@ -57,19 +58,24 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; + use vortex_session::VortexSession; use crate::RLEData; use crate::rle::RLEArray; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + fn rle(primitive: &PrimitiveArray, ctx: &mut ExecutionCtx) -> RLEArray { RLEData::encode(primitive.as_view(), ctx).unwrap() } #[test] fn try_cast_rle_success() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::new( Buffer::from_iter([10u8, 20, 30, 40, 50]), Validity::from_iter([true, true, true, true, true]), @@ -86,7 +92,7 @@ mod tests { #[test] #[should_panic] fn try_cast_rle_fail() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::new( Buffer::from_iter([10u8, 20, 30, 40, 50]), Validity::from_iter([true, false, true, true, false]), diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 6e598e26c18..7f6bff24bcd 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -28,7 +28,7 @@ pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -70,6 +70,10 @@ impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_fss pub fn vortex_fsst::FSST::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fsst::FSST + +pub fn vortex_fsst::FSST::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fsst::FSST pub fn vortex_fsst::FSST::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index dd5214aadc0..36e57c51239 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -189,12 +189,13 @@ impl VTable for FSST { metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, + session: &VortexSession, ) -> VortexResult> { let metadata = FSSTMetadata::decode(metadata)?; let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?); let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?); + let mut ctx = session.create_execution_ctx(); if buffers.len() == 2 { return Self::deserialize_legacy( self, @@ -204,6 +205,7 @@ impl VTable for FSST { &symbols, &symbol_lengths, children, + &mut ctx, ); } @@ -237,8 +239,6 @@ impl VTable for FSST { vortex_bail!("Expected 2 or 3 children, got {}", children.len()); }; - // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); FSSTData::validate_parts( &symbols, &symbol_lengths, @@ -413,6 +413,7 @@ impl FSST { /// Legacy deserialization path (2 buffers): the codes were stored as a full /// `VarBinArray` child. We decompose the VarBinArray into its bytes (stored in /// FSSTData) and offsets/validity (stored in slots). + #[allow(clippy::too_many_arguments)] fn deserialize_legacy( &self, dtype: &DType, @@ -421,6 +422,7 @@ impl FSST { symbols: &Buffer, symbol_lengths: &Buffer, children: &dyn ArrayChildren, + ctx: &mut ExecutionCtx, ) -> VortexResult> { if children.len() != 2 { vortex_bail!(InvalidArgument: "Expected 2 children, got {}", children.len()); @@ -444,8 +446,6 @@ impl FSST { len, )?; - // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); FSSTData::validate_parts_from_codes( symbols, symbol_lengths, @@ -453,7 +453,7 @@ impl FSST { &uncompressed_lengths, dtype, len, - &mut ctx, + ctx, )?; let slots = FSSTData::make_slots(&codes, &uncompressed_lengths); let codes_bytes = codes.bytes_handle().clone(); diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 550181af309..cd8c5e3a9c3 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -3,65 +3,114 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; -use vortex_array::arrays::VarBin; -use vortex_array::builtins::ArrayBuiltins; +use vortex_array::arrays::VarBinArray; +use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::dtype::DType; +use vortex_array::scalar_fn::fns::cast::CastKernel; use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use crate::FSST; use crate::FSSTArrayExt; + +fn build_with_codes_validity( + array: ArrayView<'_, FSST>, + dtype: &DType, + new_codes_validity: Validity, +) -> VortexResult { + let codes = array.codes(); + let new_codes = VarBinArray::try_new( + codes.offsets().clone(), + codes.bytes().clone(), + codes.dtype().with_nullability(dtype.nullability()), + new_codes_validity, + )?; + + Ok(unsafe { + FSST::new_unchecked( + dtype.clone(), + array.symbols().clone(), + array.symbol_lengths().clone(), + new_codes, + array.uncompressed_lengths().clone(), + ) + } + .into_array()) +} + impl CastReduce for FSST { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - // FSST is a string compression encoding. - // For nullability changes, we can cast the codes and symbols arrays - if array.dtype().eq_ignore_nullability(dtype) { - // Cast codes array to handle nullability - let new_codes = array - .codes() - .into_array() - .cast(array.codes_dtype().with_nullability(dtype.nullability()))?; - - // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - Ok(Some( - FSST::try_new( - dtype.clone(), - array.symbols().clone(), - array.symbol_lengths().clone(), - new_codes.as_::().into_owned(), - array.uncompressed_lengths().clone(), - &mut ctx, - )? - .into_array(), - )) - } else { - Ok(None) + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); } + + let codes = array.codes(); + let Some(new_codes_validity) = codes + .validity()? + .trivial_cast_nullability(dtype.nullability(), codes.len())? + else { + return Ok(None); + }; + + Ok(Some(build_with_codes_validity( + array, + dtype, + new_codes_validity, + )?)) + } +} + +impl CastKernel for FSST { + fn cast( + array: ArrayView<'_, Self>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); + } + + let codes = array.codes(); + let new_codes_validity = + codes + .validity()? + .cast_nullability(dtype.nullability(), codes.len(), ctx)?; + + Ok(Some(build_with_codes_validity( + array, + dtype, + new_codes_validity, + )?)) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::compute::conformance::cast::test_cast_conformance; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; + use vortex_array::session::ArraySession; + use vortex_session::VortexSession; use crate::fsst_compress; use crate::fsst_train_compressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_fsst_nullability() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let strings = VarBinArray::from_iter( vec![Some("hello"), Some("world"), Some("hello world")], DType::Utf8(Nullability::NonNullable), @@ -94,7 +143,7 @@ mod tests { DType::Utf8(Nullability::NonNullable) ))] fn test_cast_fsst_conformance(#[case] array: VarBinArray) { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let compressor = fsst_train_compressor(&array); let fsst = fsst_compress(&array, array.len(), array.dtype(), &compressor, &mut ctx); test_cast_conformance(&fsst.into_array()); diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs index 079efcfecb0..7f455a06e16 100644 --- a/encodings/fsst/src/kernel.rs +++ b/encodings/fsst/src/kernel.rs @@ -5,11 +5,13 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::kernel::ParentKernelSet; use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; use vortex_array::scalar_fn::fns::like::LikeExecuteAdaptor; use crate::FSST; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(FSST)), ParentKernelSet::lift(&CompareExecuteAdaptor(FSST)), ParentKernelSet::lift(&FilterExecuteAdaptor(FSST)), ParentKernelSet::lift(&TakeExecuteAdaptor(FSST)), diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index ad95badd7f3..7cb1a28bb9c 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::vtable::child_to_validity; @@ -13,53 +11,52 @@ use vortex_error::VortexResult; use crate::Pco; use crate::PcoData; + impl CastReduce for Pco { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - if !dtype.is_nullable() - || !array - .array() - .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? - { - // TODO(joe): fixme - // We cannot cast to non-nullable since the validity containing nulls is used to decode - // the PCO array, this would require rewriting tables. + // PCO (Pcodec) stores compressed data and uses validity bits to decode (the validity + // tells PCO which logical positions correspond to compressed values). Casting away + // nullability would change the validity-to-compressed-value mapping, so we cannot + // construct a non-nullable Pco without re-encoding — we only handle nullability changes + // toward `Nullable`. Non-nullable targets fall through to canonicalization. + // + // No `CastKernel` is provided for the same reason: even with execution context, we + // cannot cast away nullability on a PCO array in place. + // + // PCO supports: F16, F32, F64, I16, I32, I64, U16, U32, U64. + if !array.dtype().eq_ignore_nullability(dtype) { return Ok(None); } - // PCO (Pcodec) is a compression encoding that stores data in a compressed format. - // It can efficiently handle nullability changes without decompression, but type changes - // require decompression since the compression algorithm is type-specific. - // PCO supports: F16, F32, F64, I16, I32, I64, U16, U32, U64 - if array.dtype().eq_ignore_nullability(dtype) { - // Create a new validity with the target nullability - let unsliced_validity = - child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability()); - let new_validity = - unsliced_validity.cast_nullability(dtype.nullability(), array.len())?; - - let data = PcoData::new( - array.chunk_metas.clone(), - array.pages.clone(), - dtype.as_ptype(), - array.metadata.clone(), - array.unsliced_n_rows(), - ) - ._slice(array.slice_start(), array.slice_stop()); - - return Ok(Some( - Pco::try_new(dtype.clone(), data, new_validity)?.into_array(), - )); - } - // For other casts (e.g., numeric type changes), decode to canonical and let PrimitiveArray handle it - Ok(None) + let unsliced_validity = + child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability()); + let Some(new_validity) = + unsliced_validity.trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; + + let data = PcoData::new( + array.chunk_metas.clone(), + array.pages.clone(), + dtype.as_ptype(), + array.metadata.clone(), + array.unsliced_n_rows(), + ) + ._slice(array.slice_start(), array.slice_stop()); + + Ok(Some( + Pco::try_new(dtype.clone(), data, new_validity)?.into_array(), + )) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -68,14 +65,19 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::Pco; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_pco_f32_to_f64() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0]); let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); @@ -96,7 +98,7 @@ mod tests { #[test] fn test_cast_pco_nullability_change() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); // Test casting from NonNullable to Nullable let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); @@ -113,7 +115,7 @@ mod tests { #[test] fn test_cast_sliced_pco_nullable_to_nonnullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::new( buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([true, true, true, true, true, true]), @@ -133,7 +135,7 @@ mod tests { #[test] fn test_cast_sliced_pco_part_valid_to_nonnullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_option_iter([ None, Some(20u32), @@ -176,7 +178,7 @@ mod tests { Validity::NonNullable, ))] fn test_cast_pco_conformance(#[case] values: PrimitiveArray) { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); test_cast_conformance(&pco.into_array()); } diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index c768bb61f62..2f11d91d5ad 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -24,11 +24,11 @@ pub fn vortex_runend::RunEnd::encode(array: vortex_array::array::erased::ArrayRe pub fn vortex_runend::RunEnd::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray -pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray +pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_runend::RunEndArray pub fn vortex_runend::RunEnd::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_runend::RunEnd::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_runend::RunEnd diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index a8f99a42e94..7fe1c70461d 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -256,12 +256,9 @@ impl RunEnd { values: ArrayRef, offset: usize, length: usize, - ctx: &mut ExecutionCtx, ) -> RunEndArray { let dtype = values.dtype().clone(); - let slots = vec![Some(ends.clone()), Some(values.clone())]; - RunEndData::validate_parts(&ends, &values, offset, length, ctx) - .vortex_expect("RunEndArray validation failed"); + let slots = vec![Some(ends), Some(values)]; let data = unsafe { RunEndData::new_unchecked(offset) }; unsafe { Array::from_parts_unchecked( @@ -277,6 +274,7 @@ impl RunEnd { ctx: &mut ExecutionCtx, ) -> VortexResult { let len = RunEndData::logical_len_from_ends(&ends, ctx)?; + RunEndData::validate_parts(&ends, &values, 0, len, ctx)?; let dtype = values.dtype().clone(); let slots = vec![Some(ends), Some(values)]; let data = RunEndData::new(0); @@ -289,7 +287,9 @@ impl RunEnd { values: ArrayRef, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { + RunEndData::validate_parts(&ends, &values, offset, length, ctx)?; let dtype = values.dtype().clone(); let slots = vec![Some(ends), Some(values)]; let data = RunEndData::new(offset); @@ -469,20 +469,15 @@ impl ValidityVTable for RunEnd { Ok(match array.values().validity()? { Validity::NonNullable | Validity::AllValid => Validity::AllValid, Validity::AllInvalid => Validity::AllInvalid, - Validity::Array(values_validity) => { - // TODO(ctx): trait fixes - ValidityVTable::validity has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - Validity::Array(unsafe { - RunEnd::new_unchecked( - array.ends().clone(), - values_validity, - array.offset(), - array.len(), - &mut ctx, - ) - .into_array() - }) - } + Validity::Array(values_validity) => Validity::Array(unsafe { + RunEnd::new_unchecked( + array.ends().clone(), + values_validity, + array.offset(), + array.len(), + ) + .into_array() + }), }) } } @@ -515,8 +510,9 @@ pub(super) fn run_end_canonicalize( #[cfg(test)] mod tests { + use std::sync::LazyLock; + use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::VarBinViewArray; @@ -524,13 +520,18 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::RunEnd; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_runend_constructor() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let arr = RunEnd::new( buffer![2u32, 5, 10].into_array(), buffer![1i32, 2, 3].into_array(), @@ -551,7 +552,7 @@ mod tests { #[test] fn test_runend_utf8() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = VarBinViewArray::from_iter_str(["a", "b", "c"]).into_array(); let arr = RunEnd::new(buffer![2u32, 5, 10].into_array(), values, &mut ctx); assert_eq!(arr.len(), 10); @@ -565,7 +566,7 @@ mod tests { #[test] fn test_runend_dict() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let dict_values = VarBinViewArray::from_iter_str(["x", "y", "z"]).into_array(); let dict_codes = buffer![0u32, 1, 2].into_array(); let dict = DictArray::try_new(dict_codes, dict_values).unwrap(); diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index f423dcaa60e..0b1d7c2ca3e 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -20,6 +20,7 @@ use vortex_error::VortexResult; use crate::RunEndData; use crate::ops::find_slice_end_index; + impl FromArrowArray<&RunArray> for RunEndData where R::Native: NativePType, @@ -147,7 +148,13 @@ mod tests { ) }; - RunEnd::try_new_offset_length(ends_slice, values_slice, offset, array.len()) + RunEnd::try_new_offset_length( + ends_slice, + values_slice, + offset, + array.len(), + &mut SESSION.create_execution_ctx(), + ) } #[test] diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index 8b5ada03ff3..f70bb0becb5 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; @@ -18,8 +16,6 @@ impl CastReduce for RunEnd { // Cast the values array to the target type let casted_values = array.values().cast(dtype.clone())?; - // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); // SAFETY: casting does not affect the ends being valid unsafe { Ok(Some( @@ -28,7 +24,6 @@ impl CastReduce for RunEnd { casted_values, array.offset(), array.len(), - &mut ctx, ) .into_array(), )) @@ -38,9 +33,10 @@ impl CastReduce for RunEnd { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -50,14 +46,19 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::RunEnd; use crate::RunEndArray; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_runend_i32_to_i64() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let runend = RunEnd::try_new( buffer![3u64, 5, 8, 10].into_array(), buffer![100i32, 200, 100, 300].into_array(), @@ -98,7 +99,7 @@ mod tests { #[test] fn test_cast_runend_nullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let runend = RunEnd::try_new( buffer![2u64, 4, 7].into_array(), PrimitiveArray::from_option_iter([Some(10i32), None, Some(20)]).into_array(), @@ -118,7 +119,7 @@ mod tests { #[test] fn test_cast_runend_with_offset() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); // Create a RunEndArray: [100, 100, 100, 200, 200, 300, 300, 300, 300, 300] let runend = RunEnd::try_new( buffer![3u64, 5, 10].into_array(), @@ -174,7 +175,7 @@ mod tests { ctx, ).unwrap())] fn test_cast_runend_conformance(#[case] build: RunEndBuilder) { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let array = build(&mut ctx); test_cast_conformance(&array.into_array()); } diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index a4597950b95..ddce31184eb 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; @@ -20,8 +18,6 @@ impl FillNullReduce for RunEnd { fill_value: &Scalar, ) -> VortexResult> { let new_values = array.values().fill_null(fill_value.clone())?; - // TODO(ctx): trait fixes - FillNullReduce::fill_null has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); // SAFETY: modifying values only, does not affect ends Ok(Some( unsafe { @@ -30,7 +26,6 @@ impl FillNullReduce for RunEnd { new_values, array.offset(), array.len(), - &mut ctx, ) } .into_array(), diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index d537845ee2b..60644e2bced 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -65,7 +65,6 @@ impl FilterKernel for RunEnd { values, 0, mask_values.true_count(), - ctx, ) .into_array(), )) diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs index 57bc207a385..32b22eb96c5 100644 --- a/encodings/runend/src/compute/take_from.rs +++ b/encodings/runend/src/compute/take_from.rs @@ -25,7 +25,7 @@ impl ExecuteParentKernel for RunEndTakeFrom { array: ArrayView<'_, RunEnd>, dict: ArrayView<'_, Dict>, child_idx: usize, - ctx: &mut ExecutionCtx, + _ctx: &mut ExecutionCtx, ) -> VortexResult> { if child_idx != 0 { return Ok(None); @@ -44,10 +44,8 @@ impl ExecuteParentKernel for RunEndTakeFrom { dict.values().take(array.values().clone())?, array.offset(), array.len(), - ctx, ) }; - // Ok(Some(ree_array.into_array())) } } @@ -109,7 +107,6 @@ mod tests { codes.values().clone(), 2, // offset 3, // len - &mut ctx, ) }; @@ -134,7 +131,6 @@ mod tests { codes.values().clone(), 3, // offset at exact run boundary 4, // len - &mut ctx, ) }; @@ -159,7 +155,6 @@ mod tests { codes.values().slice(1..3)?, 4, // offset 1, // len - &mut ctx, ) }; diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index f619dae3d67..73f6ebc524e 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -72,7 +72,6 @@ fn slice( array.values().slice(slice_begin..slice_end)?, range.start + array.offset(), new_length, - ctx, ) .into_array() }) diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index 64c3287ae79..cd80e42db46 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ScalarFnArray; @@ -82,8 +80,6 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { ScalarFnArray::try_new(parent.scalar_fn().clone(), new_children, values_len)? .into_array(); - // TODO(ctx): trait fixes - ArrayParentReduceRule::reduce_parent has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); Ok(Some( unsafe { RunEnd::new_unchecked( @@ -91,7 +87,6 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { new_values, run_end.offset(), run_end.len(), - &mut ctx, ) } .into_array(), diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index c532689a275..e6d64fdf7c5 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -87,9 +87,10 @@ impl CastReduce for Sequence { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -98,10 +99,15 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; + use vortex_session::VortexSession; use crate::Sequence; use crate::SequenceArray; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_sequence_nullability() { let sequence = Sequence::try_new_typed(0u32, 1u32, Nullability::NonNullable, 4).unwrap(); @@ -119,7 +125,7 @@ mod tests { #[test] fn test_cast_sequence_u32_to_i64() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let sequence = Sequence::try_new_typed(100u32, 10u32, Nullability::NonNullable, 4).unwrap(); let casted = sequence @@ -138,7 +144,7 @@ mod tests { #[test] fn test_cast_sequence_i16_to_i32_nullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); // Test ptype change AND nullability change in one cast let sequence = Sequence::try_new_typed(5i16, 3i16, Nullability::NonNullable, 3).unwrap(); @@ -161,7 +167,7 @@ mod tests { #[test] fn test_cast_sequence_to_float_delegates_to_canonical() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let sequence = Sequence::try_new_typed(0i32, 1i32, Nullability::NonNullable, 5).unwrap(); // Cast to float should delegate to canonical (SequenceArray doesn't support float) diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 8a27e72aed4..17455013720 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -52,7 +52,6 @@ use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_panic; use crate::ConstantArray; use crate::Sparse; @@ -83,7 +82,7 @@ pub(super) fn execute_sparse( DType::Struct(struct_fields, ..) => execute_sparse_struct( struct_fields, array.fill_scalar().as_struct(), - array.dtype(), + array.dtype().nullability(), array.patches(), array.len(), ctx, @@ -403,7 +402,7 @@ fn execute_sparse_primitives TryFrom<&'a Scalar, Error fn execute_sparse_struct( struct_fields: &StructFields, fill_struct: StructScalar, - dtype: &DType, + nullability: Nullability, // Resolution is unnecessary b/c we're just pushing the patches into the fields. unresolved_patches: &Patches, len: usize, @@ -425,28 +424,20 @@ fn execute_sparse_struct( .execute::(ctx)?; let columns_patch_values = patch_values_as_struct.unmasked_fields(); let names = patch_values_as_struct.names(); - let validity = if dtype.is_nullable() { - top_level_fill_validity.patch( - len, - unresolved_patches.offset(), - unresolved_patches.indices(), - &Validity::from_mask( - { - let v = unresolved_patches.values(); - v.validity() - .vortex_expect("validity_mask") - .execute_mask(v.len(), ctx) - .vortex_expect("Failed to compute validity mask") - }, - Nullability::Nullable, - ), - ctx, - )? - } else { - top_level_fill_validity - .into_non_nullable(len) - .unwrap_or_else(|| vortex_panic!("fill validity should match sparse array nullability")) - }; + let validity = top_level_fill_validity.patch( + len, + unresolved_patches.offset(), + unresolved_patches.indices(), + &Validity::from_mask( + patch_values_as_struct + .validity() + .vortex_expect("validity_mask") + .execute_mask(patch_values_as_struct.len(), ctx) + .vortex_expect("Failed to compute validity mask"), + nullability, + ), + ctx, + )?; Ok(StructArray::try_from_iter_with_validity( names.iter().zip_eq( @@ -1215,35 +1206,35 @@ mod test { let elements_slice = elements_array.as_slice::(); // List 0: [1] - let list0_offset = result_listview.offset_at(0) as usize; + let list0_offset = result_listview.offset_at(0); assert_eq!(elements_slice[list0_offset], 1); // List 1: [5,6,7,8] - let list1_offset = result_listview.offset_at(1) as usize; - let list1_size = result_listview.size_at(1) as usize; + let list1_offset = result_listview.offset_at(1); + let list1_size = result_listview.size_at(1); assert_eq!( &elements_slice[list1_offset..list1_offset + list1_size], &[5, 6, 7, 8] ); // List 2: [5,6,7,8] - let list2_offset = result_listview.offset_at(2) as usize; - let list2_size = result_listview.size_at(2) as usize; + let list2_offset = result_listview.offset_at(2); + let list2_size = result_listview.size_at(2); assert_eq!( &elements_slice[list2_offset..list2_offset + list2_size], &[5, 6, 7, 8] ); // List 3: [2] - let list3_offset = result_listview.offset_at(3) as usize; + let list3_offset = result_listview.offset_at(3); assert_eq!(elements_slice[list3_offset], 2); // List 4: [1] - let list4_offset = result_listview.offset_at(4) as usize; + let list4_offset = result_listview.offset_at(4); assert_eq!(elements_slice[list4_offset], 1); // List 5: [2] - let list5_offset = result_listview.offset_at(5) as usize; + let list5_offset = result_listview.offset_at(5); assert_eq!(elements_slice[list5_offset], 2); Ok(()) } diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 8ef09360e41..663539a6030 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::Sparse; + impl CastReduce for Sparse { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { let casted_patches = array @@ -34,9 +35,10 @@ impl CastReduce for Sparse { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -46,14 +48,19 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; + use vortex_array::session::ArraySession; use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::Sparse; use crate::SparseArray; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_sparse_i32_to_i64() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let sparse = Sparse::try_new( buffer![2u64, 5, 8].into_array(), buffer![100i32, 200, 300].into_array(), @@ -127,7 +134,7 @@ mod tests { #[test] fn test_cast_sparse_null_fill_all_patched_to_non_nullable() -> vortex_error::VortexResult<()> { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); // Regression test for https://github.com/vortex-data/vortex/issues/6932 // // When all positions are patched the null fill is unused, so a cast to diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index ae3032cdc03..e295e556566 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -7,12 +7,12 @@ use vortex_array::IntoArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::cast::CastReduce; -use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; use vortex_error::VortexResult; use crate::Zstd; use crate::ZstdData; + impl CastReduce for Zstd { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !dtype.eq_ignore_nullability(array.dtype()) { @@ -24,71 +24,57 @@ impl CastReduce for Zstd { let src_nullability = array.dtype().nullability(); let target_nullability = dtype.nullability(); - match (src_nullability, target_nullability) { + let new_validity = match (src_nullability, target_nullability) { // Same type case. This should be handled in the layer above but for // completeness of the match arms we also handle it here. (Nullability::Nullable, Nullability::Nullable) | (Nullability::NonNullable, Nullability::NonNullable) => { - Ok(Some(array.array().clone())) + return Ok(Some(array.array().clone())); } (Nullability::NonNullable, Nullability::Nullable) => { // nonnull => null, trivial cast by altering the validity - let unsliced_validity = - child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability()); - Ok(Some( - Zstd::try_new( - dtype.clone(), - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - ), - unsliced_validity, - )? - .into_array() - .slice(array.slice_start()..array.slice_stop())?, - )) + child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability()) } (Nullability::Nullable, Nullability::NonNullable) => { // null => non-null works if there are no nulls in the sliced range let unsliced_validity = child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability()); - let has_nulls = !matches!( - unsliced_validity.slice(array.slice_start()..array.slice_stop())?, - Validity::AllValid | Validity::NonNullable - ); + let has_nulls = !unsliced_validity + .slice(array.slice_start()..array.slice_stop())? + .no_nulls(); // We don't attempt to handle casting when there are nulls. if has_nulls { return Ok(None); } - - // If there are no nulls, the cast is trivial - Ok(Some( - Zstd::try_new( - dtype.clone(), - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - ), - unsliced_validity, - )? - .into_array() - .slice(array.slice_start()..array.slice_stop())?, - )) + unsliced_validity } - } + }; + + // If there are no nulls, the cast is trivial + Ok(Some( + Zstd::try_new( + dtype.clone(), + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + ), + new_validity, + )? + .into_array() + .slice(array.slice_start()..array.slice_stop())?, + )) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -97,14 +83,19 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::Zstd; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_zstd_i32_to_i64() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap(); @@ -123,7 +114,7 @@ mod tests { #[test] fn test_cast_zstd_nullability_change() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap(); @@ -139,7 +130,7 @@ mod tests { #[test] fn test_cast_sliced_zstd_nullable_to_nonnullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::new( buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([true, true, true, true, true, true]), @@ -160,7 +151,7 @@ mod tests { #[test] fn test_cast_sliced_zstd_part_valid_to_nonnullable() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_option_iter([ None, Some(20u32), @@ -201,8 +192,8 @@ mod tests { Validity::NonNullable, ))] fn test_cast_zstd_conformance(#[case] values: PrimitiveArray) { - let zstd = Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(); + let zstd = + Zstd::from_primitive(&values, 0, 0, &mut SESSION.create_execution_ctx()).unwrap(); test_cast_conformance(&zstd.into_array()); } } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 903b9d633b9..0d52ce0a6ec 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -1432,6 +1432,10 @@ pub type vortex_array::arrays::bool::BoolMaskedValidityRule::Parent = vortex_arr pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, child_idx: usize) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -1948,7 +1952,11 @@ pub fn vortex_array::arrays::Decimal::between(arr: vortex_array::ArrayView<'_, v impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal -pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Decimal @@ -2882,6 +2890,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSiz pub fn vortex_array::arrays::FixedSizeList::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -3026,6 +3038,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::List pub fn vortex_array::arrays::List::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List + +pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::List pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -3202,6 +3218,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -3914,6 +3934,10 @@ impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::fill_null(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, fill_value: &vortex_array::scalar::Scalar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -4640,7 +4664,11 @@ pub fn vortex_array::arrays::Struct::slice(array: vortex_array::ArrayView<'_, Se impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Struct @@ -4806,6 +4834,10 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array pub fn vortex_array::arrays::VarBin::compare(lhs: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -5160,6 +5192,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinVi pub fn vortex_array::arrays::VarBinView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -5420,6 +5456,10 @@ pub type vortex_array::arrays::bool::BoolMaskedValidityRule::Parent = vortex_arr pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, child_idx: usize) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -5686,7 +5726,11 @@ pub fn vortex_array::arrays::Decimal::between(arr: vortex_array::ArrayView<'_, v impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal -pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Decimal @@ -5992,6 +6036,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSiz pub fn vortex_array::arrays::FixedSizeList::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -6070,6 +6118,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::List pub fn vortex_array::arrays::List::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List + +pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::List pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -6148,6 +6200,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -6470,6 +6526,10 @@ impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::fill_null(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, fill_value: &vortex_array::scalar::Scalar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -6724,7 +6784,11 @@ pub fn vortex_array::arrays::Struct::slice(array: vortex_array::ArrayView<'_, Se impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Struct @@ -6812,6 +6876,10 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array pub fn vortex_array::arrays::VarBin::compare(lhs: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -6886,6 +6954,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinVi pub fn vortex_array::arrays::VarBinView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -16150,9 +16222,25 @@ pub trait vortex_array::scalar_fn::fns::cast::CastKernel: vortex_array::VTable pub fn vortex_array::scalar_fn::fns::cast::CastKernel::cast(array: vortex_array::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal -pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList + +pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List + +pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Primitive @@ -16160,7 +16248,15 @@ pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub trait vortex_array::scalar_fn::fns::cast::CastReduce: vortex_array::VTable @@ -16178,6 +16274,10 @@ impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Co pub fn vortex_array::arrays::Constant::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Constant>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Extension pub fn vortex_array::arrays::Extension::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -16194,6 +16294,14 @@ impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> @@ -19438,7 +19546,7 @@ pub fn vortex_array::validity::Validity::and(self, rhs: vortex_array::validity:: pub fn vortex_array::validity::Validity::as_array(&self) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::validity::Validity::cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize) -> vortex_error::VortexResult +pub fn vortex_array::validity::Validity::cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::validity::Validity::execute_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19446,7 +19554,7 @@ pub fn vortex_array::validity::Validity::filter(&self, mask: &vortex_mask::Mask) pub fn vortex_array::validity::Validity::into_array(self) -> core::option::Option -pub fn vortex_array::validity::Validity::into_non_nullable(self, len: usize) -> core::option::Option +pub fn vortex_array::validity::Validity::into_non_nullable(self, len: usize, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option pub fn vortex_array::validity::Validity::into_nullable(self) -> vortex_array::validity::Validity @@ -19474,7 +19582,9 @@ pub fn vortex_array::validity::Validity::to_array(&self, len: usize) -> vortex_a pub fn vortex_array::validity::Validity::to_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::validity::Validity::uncompressed_size(&self) -> usize +pub fn vortex_array::validity::Validity::trivial_cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::validity::Validity::trivial_into_non_nullable(self, len: usize) -> vortex_error::VortexResult> pub fn vortex_array::validity::Validity::union_nullability(self, nullability: vortex_array::dtype::Nullability) -> Self diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index 3b5fbea7607..6bfaa03b9db 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -4,24 +4,48 @@ use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; impl CastReduce for Bool { fn cast(array: ArrayView<'_, Bool>, dtype: &DType) -> VortexResult> { - if !matches!(dtype, DType::Bool(_)) { + if !dtype.is_boolean() { return Ok(None); } - let new_nullability = dtype.nullability(); - let new_validity = array + let Some(new_validity) = array .validity()? - .cast_nullability(new_nullability, array.len())?; + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; + Ok(Some( + BoolArray::new(array.to_bit_buffer(), new_validity).into_array(), + )) + } +} + +impl CastKernel for Bool { + fn cast( + array: ArrayView<'_, Bool>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if !dtype.is_boolean() { + return Ok(None); + } + + let new_validity = + array + .validity()? + .cast_nullability(dtype.nullability(), array.len(), ctx)?; Ok(Some( BoolArray::new(array.to_bit_buffer(), new_validity).into_array(), )) @@ -30,14 +54,23 @@ impl CastReduce for Bool { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; + use vortex_session::VortexSession; + use crate::Canonical; use crate::IntoArray; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; + use crate::session::ArraySession; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); #[test] fn try_cast_bool_success() { @@ -51,12 +84,16 @@ mod tests { } #[test] - #[should_panic] fn try_cast_bool_fail() { + // When the validity array's min stat is not cached, the reduce rule defers and the + // failure surfaces during execution via the kernel (cast_nullability -> compute_min). let bool = BoolArray::from_iter(vec![Some(true), Some(false), None]); - bool.into_array() + let mut ctx = SESSION.create_execution_ctx(); + let result = bool + .into_array() .cast(DType::Bool(Nullability::NonNullable)) - .unwrap(); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); + assert!(result.is_err(), "Expected error, got: {result:?}"); } #[rstest] diff --git a/vortex-array/src/arrays/bool/vtable/kernel.rs b/vortex-array/src/arrays/bool/vtable/kernel.rs index e8fa5f30e9c..4f1047a5132 100644 --- a/vortex-array/src/arrays/bool/vtable/kernel.rs +++ b/vortex-array/src/arrays/bool/vtable/kernel.rs @@ -4,9 +4,11 @@ use crate::arrays::Bool; use crate::arrays::dict::TakeExecuteAdaptor; use crate::kernel::ParentKernelSet; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(Bool)), ParentKernelSet::lift(&FillNullExecuteAdaptor(Bool)), ParentKernelSet::lift(&TakeExecuteAdaptor(Bool)), ]); diff --git a/vortex-array/src/arrays/constant/compute/cast.rs b/vortex-array/src/arrays/constant/compute/cast.rs index 68c6dbeb758..2fd9b3ded32 100644 --- a/vortex-array/src/arrays/constant/compute/cast.rs +++ b/vortex-array/src/arrays/constant/compute/cast.rs @@ -15,7 +15,7 @@ impl CastReduce for Constant { fn cast(array: ArrayView<'_, Constant>, dtype: &DType) -> VortexResult> { match array.scalar().cast(dtype) { Ok(scalar) => Ok(Some(ConstantArray::new(scalar, array.len()).into_array())), - Err(_e) => Ok(None), + Err(_) => Ok(None), } } } diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 75446b17e4e..34ccc27d094 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -18,12 +18,53 @@ use crate::dtype::DecimalType; use crate::dtype::NativeDecimalType; use crate::match_each_decimal_value_type; use crate::scalar_fn::fns::cast::CastKernel; +use crate::scalar_fn::fns::cast::CastReduce; + +impl CastReduce for Decimal { + fn cast(array: ArrayView<'_, Decimal>, dtype: &DType) -> VortexResult> { + // Only nullability changes within the same decimal dtype are reducible without execution. + // Precision/scale changes need the kernel. + let DType::Decimal(to_decimal_dtype, to_nullability) = dtype else { + return Ok(None); + }; + let DType::Decimal(from_decimal_dtype, _) = array.dtype() else { + vortex_panic!( + "DecimalArray must have decimal dtype, got {:?}", + array.dtype() + ); + }; + + if from_decimal_dtype != to_decimal_dtype { + return Ok(None); + } + + let Some(new_validity) = array + .validity()? + .trivial_cast_nullability(*to_nullability, array.len())? + else { + return Ok(None); + }; + + // SAFETY: validity has the same length, only its nullability tag changes. + unsafe { + Ok(Some( + DecimalArray::new_unchecked_handle( + array.buffer_handle().clone(), + array.values_type(), + *to_decimal_dtype, + new_validity, + ) + .into_array(), + )) + } + } +} impl CastKernel for Decimal { fn cast( array: ArrayView<'_, Decimal>, dtype: &DType, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { // Early return if not casting to decimal let DType::Decimal(to_decimal_dtype, to_nullability) = dtype else { @@ -62,7 +103,7 @@ impl CastKernel for Decimal { // Cast the validity to the new nullability let new_validity = array .validity()? - .cast_nullability(*to_nullability, array.len())?; + .cast_nullability(*to_nullability, array.len(), ctx)?; // If the target needs a wider physical type, upcast the values let target_values_type = DecimalType::smallest_decimal_value_type(to_decimal_dtype); diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs index fae0c3dd866..31df664e5c7 100644 --- a/vortex-array/src/arrays/decimal/compute/rules.rs +++ b/vortex-array/src/arrays/decimal/compute/rules.rs @@ -16,10 +16,12 @@ use crate::arrays::slice::SliceReduceAdaptor; use crate::match_each_decimal_value_type; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; +use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DecimalMaskedValidityRule), + ParentRuleSet::lift(&CastReduceAdaptor(Decimal)), ParentRuleSet::lift(&MaskReduceAdaptor(Decimal)), ParentRuleSet::lift(&SliceReduceAdaptor(Decimal)), ]); diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index c9575c32ae2..ee80a1140b4 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -13,19 +13,12 @@ use crate::arrays::dict::DictArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; -use crate::validity::Validity; impl CastReduce for Dict { fn cast(array: ArrayView<'_, Dict>, dtype: &DType) -> VortexResult> { // Can have un-reference null values making the cast of values fail without a possible mask. // TODO(joe): optimize this, could look at accessible values and fill_null not those? - if !dtype.is_nullable() - && array.values().dtype().is_nullable() - && !matches!( - array.values().validity()?, - Validity::NonNullable | Validity::AllValid - ) - { + if !dtype.is_nullable() && !array.values().validity()?.no_nulls() { return Ok(None); } // Cast the dictionary values to the target type diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index af42fd5c9ea..c8e7fa17c36 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; @@ -12,10 +14,7 @@ use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; impl CastReduce for Extension { - fn cast( - array: ArrayView<'_, Extension>, - dtype: &DType, - ) -> vortex_error::VortexResult> { + fn cast(array: ArrayView<'_, Extension>, dtype: &DType) -> VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { // Target is not the same extension type. // Delegate to the storage array's cast. @@ -45,10 +44,12 @@ impl CastReduce for Extension { #[cfg(test)] mod tests { + use std::sync::LazyLock; use rstest::rstest; use vortex_buffer::Buffer; use vortex_buffer::buffer; + use vortex_session::VortexSession; use super::*; use crate::IntoArray; @@ -59,8 +60,13 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; + use crate::executor::VortexSessionExecute; use crate::extension::datetime::TimeUnit; use crate::extension::datetime::Timestamp; + use crate::session::ArraySession; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); #[test] fn cast_same_ext_dtype() { @@ -105,16 +111,18 @@ mod tests { let storage = buffer![1i64].into_array(); let arr = ExtensionArray::new(original_dtype, storage); - #[expect(deprecated)] let result = arr .into_array() .cast(DType::Extension(target_dtype)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| { + a.execute::(&mut SESSION.create_execution_ctx()) + .map(|c| c.into_array()) + }); assert!(result.is_err()); } #[test] - fn cast_timestamp_to_i64() -> vortex_error::VortexResult<()> { + fn cast_timestamp_to_i64() -> VortexResult<()> { let ext_dtype = Timestamp::new_with_tz( TimeUnit::Nanoseconds, Some("UTC".into()), diff --git a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs index 791462d51c9..a54fe5c7c4d 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs @@ -4,6 +4,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::FixedSizeList; @@ -11,7 +12,20 @@ use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; +use crate::validity::Validity; + +fn build_with_validity( + array: ArrayView<'_, FixedSizeList>, + elements: ArrayRef, + validity: Validity, +) -> ArrayRef { + // SAFETY: The only requirements for safety here are related to lengths, and no lengths have + // changed here. So as long as the original array is valid, this is also valid. + unsafe { FixedSizeListArray::new_unchecked(elements, array.list_size(), validity, array.len()) } + .into_array() +} /// Cast implementation for [`FixedSizeListArray`]. /// @@ -23,23 +37,33 @@ impl CastReduce for FixedSizeList { return Ok(None); }; + let Some(validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; let elements = array.elements().cast((**target_element_type).clone())?; + + Ok(Some(build_with_validity(array, elements, validity))) + } +} + +impl CastKernel for FixedSizeList { + fn cast( + array: ArrayView<'_, FixedSizeList>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let Some(target_element_type) = dtype.as_fixed_size_list_element_opt() else { + return Ok(None); + }; + let validity = array .validity()? - .cast_nullability(dtype.nullability(), array.len())?; - - Ok(Some( - // SAFETY: The only requirements for safety here are related to lengths, and no lengths - // have changed here. So as long as the original array is valid, this is also valid. - unsafe { - FixedSizeListArray::new_unchecked( - elements, - array.list_size(), - validity, - array.len(), - ) - } - .into_array(), - )) + .cast_nullability(dtype.nullability(), array.len(), ctx)?; + let elements = array.elements().cast((**target_element_type).clone())?; + + Ok(Some(build_with_validity(array, elements, validity))) } } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs index c28a3a2a805..319751070de 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs @@ -4,8 +4,11 @@ use crate::arrays::FixedSizeList; use crate::arrays::dict::TakeExecuteAdaptor; use crate::kernel::ParentKernelSet; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; impl FixedSizeList { - pub(crate) const PARENT_KERNELS: ParentKernelSet = - ParentKernelSet::new(&[ParentKernelSet::lift(&TakeExecuteAdaptor(FixedSizeList))]); + pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(FixedSizeList)), + ParentKernelSet::lift(&TakeExecuteAdaptor(FixedSizeList)), + ]); } diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index c9813cfe92d..e9b29b9b145 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -4,6 +4,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::List; @@ -11,6 +12,7 @@ use crate::arrays::ListArray; use crate::arrays::list::ListArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; impl CastReduce for List { @@ -19,24 +21,56 @@ impl CastReduce for List { return Ok(None); }; + let Some(validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; + + let new_elements = array.elements().cast((**target_element_type).clone())?; + + Ok(Some( + unsafe { ListArray::new_unchecked(new_elements, array.offsets().clone(), validity) } + .into_array(), + )) + } +} + +impl CastKernel for List { + fn cast( + array: ArrayView<'_, List>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let Some(target_element_type) = dtype.as_list_element_opt() else { + return Ok(None); + }; + let validity = array .validity()? - .cast_nullability(dtype.nullability(), array.len())?; + .cast_nullability(dtype.nullability(), array.len(), ctx)?; let new_elements = array.elements().cast((**target_element_type).clone())?; - ListArray::try_new(new_elements, array.offsets().clone(), validity) - .map(|a| Some(a.into_array())) + Ok(Some( + unsafe { ListArray::new_unchecked(new_elements, array.offsets().clone(), validity) } + .into_array(), + )) } } #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::LazyLock; use rstest::rstest; + use vortex_array::session::ArraySession; use vortex_buffer::buffer; + use vortex_session::VortexSession; + use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::RecursiveCanonical; @@ -52,6 +86,9 @@ mod tests { use crate::dtype::PType; use crate::validity::Validity; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_cast_list_success() { let list = ListArray::try_new( @@ -87,11 +124,11 @@ mod tests { let target_dtype = DType::Primitive(PType::U64, Nullability::NonNullable); // can't cast list to u64 - let result = list.into_array().cast(target_dtype).and_then(|a| { - #[expect(deprecated)] - let canonical = a.to_canonical().map(|c| c.into_array()); - canonical - }); + let result = list + .into_array() + .cast(target_dtype) + .and_then(|a| a.execute::(&mut SESSION.create_execution_ctx())) + .map(|c| c.into_array()); assert!(result.is_err()); } @@ -112,11 +149,11 @@ mod tests { Nullability::NonNullable, ); - let result = list.into_array().cast(target_dtype).and_then(|a| { - #[expect(deprecated)] - let canonical = a.to_canonical().map(|c| c.into_array()); - canonical - }); + let result = list + .into_array() + .cast(target_dtype) + .and_then(|a| a.execute::(&mut SESSION.create_execution_ctx())) + .map(|c| c.into_array()); assert!(result.is_err()); // Nulls in list element array — the inner cast error is deferred until diff --git a/vortex-array/src/arrays/list/compute/kernels.rs b/vortex-array/src/arrays/list/compute/kernels.rs index b4268642e82..188c83c1bf5 100644 --- a/vortex-array/src/arrays/list/compute/kernels.rs +++ b/vortex-array/src/arrays/list/compute/kernels.rs @@ -5,8 +5,10 @@ use crate::arrays::List; use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterExecuteAdaptor; use crate::kernel::ParentKernelSet; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(List)), ParentKernelSet::lift(&FilterExecuteAdaptor(List)), ParentKernelSet::lift(&TakeExecuteAdaptor(List)), ]); diff --git a/vortex-array/src/arrays/listview/compute/cast.rs b/vortex-array/src/arrays/listview/compute/cast.rs index 77cd7ab7736..473f76b12ee 100644 --- a/vortex-array/src/arrays/listview/compute/cast.rs +++ b/vortex-array/src/arrays/listview/compute/cast.rs @@ -4,6 +4,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ListView; @@ -11,7 +12,27 @@ use crate::arrays::ListViewArray; use crate::arrays::listview::ListViewArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; +use crate::validity::Validity; + +fn build_with_validity( + array: ArrayView<'_, ListView>, + new_elements: ArrayRef, + validity: Validity, +) -> ArrayRef { + // SAFETY: Since `cast` is length-preserving, all of the invariants remain the same. + unsafe { + ListViewArray::new_unchecked( + new_elements, + array.offsets().clone(), + array.sizes().clone(), + validity, + ) + .with_zero_copy_to_list(array.is_zero_copy_to_list()) + } + .into_array() +} impl CastReduce for ListView { fn cast(array: ArrayView<'_, ListView>, dtype: &DType) -> VortexResult> { @@ -19,25 +40,34 @@ impl CastReduce for ListView { let Some(target_element_type) = dtype.as_list_element_opt() else { return Ok(None); }; + let Some(validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { + return Ok(None); + }; // Cast the elements to the target element type. let new_elements = array.elements().cast((**target_element_type).clone())?; + Ok(Some(build_with_validity(array, new_elements, validity))) + } +} + +impl CastKernel for ListView { + fn cast( + array: ArrayView<'_, ListView>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let Some(target_element_type) = dtype.as_list_element_opt() else { + return Ok(None); + }; + let validity = array .validity()? - .cast_nullability(dtype.nullability(), array.len())?; - - // SAFETY: Since `cast` is length-preserving, all of the invariants remain the same. - Ok(Some( - unsafe { - ListViewArray::new_unchecked( - new_elements, - array.offsets().clone(), - array.sizes().clone(), - validity, - ) - .with_zero_copy_to_list(array.is_zero_copy_to_list()) - } - .into_array(), - )) + .cast_nullability(dtype.nullability(), array.len(), ctx)?; + let new_elements = array.elements().cast((**target_element_type).clone())?; + + Ok(Some(build_with_validity(array, new_elements, validity))) } } diff --git a/vortex-array/src/arrays/listview/vtable/kernel.rs b/vortex-array/src/arrays/listview/vtable/kernel.rs new file mode 100644 index 00000000000..f6ceca284bf --- /dev/null +++ b/vortex-array/src/arrays/listview/vtable/kernel.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use crate::arrays::ListView; +use crate::kernel::ParentKernelSet; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; + +pub(super) const PARENT_KERNELS: ParentKernelSet = + ParentKernelSet::new(&[ParentKernelSet::lift(&CastExecuteAdaptor(ListView))]); diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 893edb6bf71..f51fde706a4 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -29,12 +29,14 @@ use crate::arrays::listview::ListViewData; use crate::arrays::listview::array::NUM_SLOTS; use crate::arrays::listview::array::SLOT_NAMES; use crate::arrays::listview::compute::rules::PARENT_RULES; +use crate::arrays::listview::vtable::kernel::PARENT_KERNELS; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::serde::ArrayChildren; use crate::validity::Validity; +mod kernel; mod operations; mod validity; /// A [`ListView`]-encoded Vortex array. @@ -214,4 +216,13 @@ impl VTable for ListView { ) -> VortexResult> { PARENT_RULES.evaluate(array, parent, child_idx) } + + fn execute_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + PARENT_KERNELS.execute(array, parent, child_idx, ctx) + } } diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index abd58022ba6..92140bf9b8b 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -21,6 +21,37 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::scalar_fn::fns::cast::CastKernel; +use crate::scalar_fn::fns::cast::CastReduce; + +impl CastReduce for Primitive { + fn cast(array: ArrayView<'_, Primitive>, dtype: &DType) -> VortexResult> { + // Only the same ptype is reducible without execution; type changes need the kernel + // to verify values fit in the target range. + let DType::Primitive(new_ptype, new_nullability) = dtype else { + return Ok(None); + }; + if *new_ptype != array.ptype() { + return Ok(None); + } + + let Some(new_validity) = array + .validity()? + .trivial_cast_nullability(*new_nullability, array.len())? + else { + return Ok(None); + }; + + // SAFETY: validity and data buffer still have same length. + Ok(Some(unsafe { + PrimitiveArray::new_unchecked_from_handle( + array.buffer_handle().clone(), + array.ptype(), + new_validity, + ) + .into_array() + })) + } +} impl CastKernel for Primitive { fn cast( @@ -36,7 +67,7 @@ impl CastKernel for Primitive { // First, check that the cast is compatible with the source array's validity let new_validity = array .validity()? - .cast_nullability(new_nullability, array.len())?; + .cast_nullability(new_nullability, array.len(), ctx)?; // Same ptype: zero-copy, just update validity. if array.ptype() == new_ptype { diff --git a/vortex-array/src/arrays/primitive/compute/rules.rs b/vortex-array/src/arrays/primitive/compute/rules.rs index 99b0a7464d5..528e5e91520 100644 --- a/vortex-array/src/arrays/primitive/compute/rules.rs +++ b/vortex-array/src/arrays/primitive/compute/rules.rs @@ -12,10 +12,12 @@ use crate::arrays::PrimitiveArray; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; +use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&PrimitiveMaskedValidityRule), + ParentRuleSet::lift(&CastReduceAdaptor(Primitive)), ParentRuleSet::lift(&MaskReduceAdaptor(Primitive)), ParentRuleSet::lift(&SliceReduceAdaptor(Primitive)), ]); diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 6c9a5f1edca..ddb3cbeadf2 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -22,7 +22,7 @@ impl CastKernel for Struct { fn cast( array: ArrayView<'_, Struct>, dtype: &DType, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let Some(target_sdtype) = dtype.as_struct_fields_opt() else { return Ok(None); @@ -73,7 +73,7 @@ impl CastKernel for Struct { let validity = array .validity()? - .cast_nullability(dtype.nullability(), array.len())?; + .cast_nullability(dtype.nullability(), array.len(), ctx)?; StructArray::try_new( target_sdtype.names().clone(), diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index 8a802969d39..6e95e3eaff8 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -18,43 +18,46 @@ use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::arrays::slice::SliceReduceAdaptor; use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; +use crate::dtype::DType; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::fns::cast::Cast; +use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::get_item::GetItem; use crate::scalar_fn::fns::mask::Mask; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; use crate::validity::Validity; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ - ParentRuleSet::lift(&StructCastPushDownRule), + ParentRuleSet::lift(&CastReduceAdaptor(Struct)), ParentRuleSet::lift(&StructGetItemRule), ParentRuleSet::lift(&MaskReduceAdaptor(Struct)), ParentRuleSet::lift(&SliceReduceAdaptor(Struct)), ParentRuleSet::lift(&TakeReduceAdaptor(Struct)), ]); -/// Rule to push down cast into struct fields. +/// Push the cast into struct fields without execution. /// -/// TODO(joe/rob): should be have this in casts. +/// Supports schema evolution by allowing new nullable fields to be added during the cast, +/// filled with null values. For nullability changes, only handles the cheap path +/// (`try_cast_nullability`); when statistics computation is required to determine whether +/// the array contains invalid values, returns `Ok(None)` so [`CastKernel`] can run instead. /// -/// This rule supports schema evolution by allowing new nullable fields to be added -/// at the end of the struct, filled with null values. -#[derive(Debug)] -struct StructCastPushDownRule; -impl ArrayParentReduceRule for StructCastPushDownRule { - type Parent = ExactScalarFn; +/// [`CastKernel`]: crate::scalar_fn::fns::cast::CastKernel +impl CastReduce for Struct { + fn cast(array: ArrayView<'_, Struct>, dtype: &DType) -> VortexResult> { + let Some(target_fields) = dtype.as_struct_fields_opt() else { + return Ok(None); + }; - fn reduce_parent( - &self, - array: ArrayView<'_, Struct>, - parent: ScalarFnArrayView, - _child_idx: usize, - ) -> VortexResult> { - let Some(target_fields) = parent.options.as_struct_fields_opt() else { + let Some(validity) = array + .validity()? + .trivial_cast_nullability(dtype.nullability(), array.len())? + else { return Ok(None); }; + let mut new_fields = Vec::with_capacity(target_fields.nfields()); for (target_name, target_dtype) in target_fields.names().iter().zip(target_fields.fields()) @@ -78,20 +81,12 @@ impl ArrayParentReduceRule for StructCastPushDownRule { } } - let validity = if parent.options.is_nullable() { - array.validity()?.into_nullable() - } else { - array - .validity()? - .into_non_nullable(array.len()) - .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))? - }; - - let new_struct = unsafe { - StructArray::new_unchecked(new_fields, target_fields.clone(), array.len(), validity) - }; - - Ok(Some(new_struct.into_array())) + Ok(Some( + unsafe { + StructArray::new_unchecked(new_fields, target_fields.clone(), array.len(), validity) + } + .into_array(), + )) } } @@ -144,25 +139,31 @@ impl ArrayParentReduceRule for StructGetItemRule { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use vortex_buffer::buffer; + use vortex_session::VortexSession; use crate::IntoArray; + use crate::VortexSessionExecute; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; use crate::arrays::struct_::StructArrayExt; use crate::arrays::struct_::compute::rules::ConstantArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::FieldNames; use crate::dtype::Nullability; use crate::dtype::PType; use crate::dtype::StructFields; use crate::scalar::Scalar; + use crate::session::ArraySession; use crate::validity::Validity; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_struct_cast_field_reorder() { // Source: {a, b}, Target: {c, b, a} - reordered + new null field @@ -188,8 +189,12 @@ mod tests { // Use `ArrayBuiltins::cast` which goes through the optimizer and applies // `StructCastPushDownRule`. - #[expect(deprecated)] - let result = source.into_array().cast(target).unwrap().to_struct(); + let result = source + .into_array() + .cast(target) + .unwrap() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); assert_arrays_eq!( result.unmasked_field_by_name("a").unwrap(), VarBinViewArray::from_iter_nullable_str([Some("A")]) @@ -257,8 +262,12 @@ mod tests { Nullability::NonNullable, ); - #[expect(deprecated)] - let result = source.into_array().cast(target).unwrap().to_struct(); + let result = source + .into_array() + .cast(target) + .unwrap() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); assert_eq!(result.unmasked_fields().len(), 2); assert_arrays_eq!( result.unmasked_field_by_name("a").unwrap(), @@ -289,8 +298,12 @@ mod tests { Nullability::NonNullable, ); - #[expect(deprecated)] - let result = source.into_array().cast(target).unwrap().to_struct(); + let result = source + .into_array() + .cast(target) + .unwrap() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( result.unmasked_field_by_name("val").unwrap().dtype(), &DType::Primitive(PType::I64, Nullability::NonNullable) diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index 6b02283aec8..e6403535e2d 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -4,13 +4,30 @@ use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::varbin::VarBinArrayExt; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; +use crate::validity::Validity; + +fn build_with_validity( + array: ArrayView<'_, VarBin>, + new_dtype: DType, + new_validity: Validity, +) -> VortexResult { + Ok(VarBinArray::try_new( + array.offsets().clone(), + array.bytes().clone(), + new_dtype, + new_validity, + )? + .into_array()) +} impl CastReduce for VarBin { fn cast(array: ArrayView<'_, VarBin>, dtype: &DType) -> VortexResult> { @@ -18,33 +35,56 @@ impl CastReduce for VarBin { return Ok(None); } + let new_nullability = dtype.nullability(); + let Some(new_validity) = array + .validity()? + .trivial_cast_nullability(new_nullability, array.len())? + else { + return Ok(None); + }; + let new_dtype = array.dtype().with_nullability(new_nullability); + Ok(Some(build_with_validity(array, new_dtype, new_validity)?)) + } +} + +impl CastKernel for VarBin { + fn cast( + array: ArrayView<'_, VarBin>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); + } + let new_nullability = dtype.nullability(); let new_validity = array .validity()? - .cast_nullability(new_nullability, array.len())?; + .cast_nullability(new_nullability, array.len(), ctx)?; let new_dtype = array.dtype().with_nullability(new_nullability); - Ok(Some( - VarBinArray::try_new( - array.offsets().clone(), - array.bytes().clone(), - new_dtype, - new_validity, - )? - .into_array(), - )) + Ok(Some(build_with_validity(array, new_dtype, new_validity)?)) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; + use vortex_session::VortexSession; + use crate::Canonical; use crate::IntoArray; + use crate::VortexSessionExecute; use crate::arrays::VarBinArray; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; + use crate::session::ArraySession; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); #[rstest] #[case( @@ -71,14 +111,18 @@ mod tests { } #[rstest] - #[should_panic] #[case(DType::Utf8(Nullability::Nullable))] - #[should_panic] #[case(DType::Binary(Nullability::Nullable))] fn try_cast_varbin_fail(#[case] source: DType) { + // Failure surfaces during execution via the kernel. let non_nullable_source = source.as_nonnullable(); let varbin = VarBinArray::from_iter(vec![Some("a"), Some("b"), None], source); - varbin.into_array().cast(non_nullable_source).unwrap(); + let mut ctx = SESSION.create_execution_ctx(); + let result = varbin + .into_array() + .cast(non_nullable_source) + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); + assert!(result.is_err(), "Expected error, got: {result:?}"); } #[rstest] diff --git a/vortex-array/src/arrays/varbin/vtable/kernel.rs b/vortex-array/src/arrays/varbin/vtable/kernel.rs index 8cb6a2ca377..9258e677933 100644 --- a/vortex-array/src/arrays/varbin/vtable/kernel.rs +++ b/vortex-array/src/arrays/varbin/vtable/kernel.rs @@ -6,8 +6,10 @@ use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterExecuteAdaptor; use crate::kernel::ParentKernelSet; use crate::scalar_fn::fns::binary::CompareExecuteAdaptor; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(VarBin)), ParentKernelSet::lift(&CompareExecuteAdaptor(VarBin)), ParentKernelSet::lift(&FilterExecuteAdaptor(VarBin)), ParentKernelSet::lift(&TakeExecuteAdaptor(VarBin)), diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs index a8e649f52e4..9b3d6b45dc7 100644 --- a/vortex-array/src/arrays/varbinview/compute/cast.rs +++ b/vortex-array/src/arrays/varbinview/compute/cast.rs @@ -6,12 +6,32 @@ use std::sync::Arc; use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::dtype::DType; +use crate::scalar_fn::fns::cast::CastKernel; use crate::scalar_fn::fns::cast::CastReduce; +use crate::validity::Validity; + +fn build_with_validity( + array: ArrayView<'_, VarBinView>, + new_dtype: DType, + new_validity: Validity, +) -> ArrayRef { + // SAFETY: casting just changes the DType, does not affect invariants on views/buffers. + unsafe { + VarBinViewArray::new_handle_unchecked( + array.views_handle().clone(), + Arc::clone(array.data_buffers()), + new_dtype, + new_validity, + ) + .into_array() + } +} impl CastReduce for VarBinView { fn cast(array: ArrayView<'_, VarBinView>, dtype: &DType) -> VortexResult> { @@ -20,36 +40,55 @@ impl CastReduce for VarBinView { } let new_nullability = dtype.nullability(); - let new_validity = array + let Some(new_validity) = array .validity()? - .cast_nullability(new_nullability, array.len())?; + .trivial_cast_nullability(new_nullability, array.len())? + else { + return Ok(None); + }; let new_dtype = array.dtype().with_nullability(new_nullability); + Ok(Some(build_with_validity(array, new_dtype, new_validity))) + } +} - // SAFETY: casting just changes the DType, does not affect invariants on views/buffers. - unsafe { - Ok(Some( - VarBinViewArray::new_handle_unchecked( - array.views_handle().clone(), - Arc::clone(array.data_buffers()), - new_dtype, - new_validity, - ) - .into_array(), - )) +impl CastKernel for VarBinView { + fn cast( + array: ArrayView<'_, VarBinView>, + dtype: &DType, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if !array.dtype().eq_ignore_nullability(dtype) { + return Ok(None); } + + let new_nullability = dtype.nullability(); + let new_validity = array + .validity()? + .cast_nullability(new_nullability, array.len(), ctx)?; + let new_dtype = array.dtype().with_nullability(new_nullability); + Ok(Some(build_with_validity(array, new_dtype, new_validity))) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; + use vortex_session::VortexSession; + use crate::Canonical; use crate::IntoArray; + use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; + use crate::session::ArraySession; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); #[rstest] #[case( @@ -76,14 +115,18 @@ mod tests { } #[rstest] - #[should_panic] #[case(DType::Utf8(Nullability::Nullable))] - #[should_panic] #[case(DType::Binary(Nullability::Nullable))] fn try_cast_varbin_fail(#[case] source: DType) { + // Failure surfaces during execution via the kernel. let non_nullable_source = source.as_nonnullable(); let varbin = VarBinViewArray::from_iter(vec![Some("a"), Some("b"), None], source); - varbin.into_array().cast(non_nullable_source).unwrap(); + let mut ctx = SESSION.create_execution_ctx(); + let result = varbin + .into_array() + .cast(non_nullable_source) + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); + assert!(result.is_err(), "Expected error, got: {result:?}"); } #[rstest] diff --git a/vortex-array/src/arrays/varbinview/vtable/kernel.rs b/vortex-array/src/arrays/varbinview/vtable/kernel.rs index 8486f0959f7..cd9d68010af 100644 --- a/vortex-array/src/arrays/varbinview/vtable/kernel.rs +++ b/vortex-array/src/arrays/varbinview/vtable/kernel.rs @@ -4,9 +4,11 @@ use crate::arrays::VarBinView; use crate::arrays::dict::TakeExecuteAdaptor; use crate::kernel::ParentKernelSet; +use crate::scalar_fn::fns::cast::CastExecuteAdaptor; use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&CastExecuteAdaptor(VarBinView)), ParentKernelSet::lift(&TakeExecuteAdaptor(VarBinView)), ParentKernelSet::lift(&ZipExecuteAdaptor(VarBinView)), ]); diff --git a/vortex-array/src/scalar_fn/fns/cast/mod.rs b/vortex-array/src/scalar_fn/fns/cast/mod.rs index 4bb21961552..cbab9615e97 100644 --- a/vortex-array/src/scalar_fn/fns/cast/mod.rs +++ b/vortex-array/src/scalar_fn/fns/cast/mod.rs @@ -198,7 +198,13 @@ impl ScalarFnVTable for Cast { } /// Cast a canonical array to the target dtype by dispatching to the appropriate -/// [`CastReduce`] or [`CastKernel`] for each canonical encoding. +/// [`CastKernel`] for each canonical encoding. +/// +/// Canonical encodings that can manipulate validity directly all implement [`CastKernel`] — +/// the kernel is the execution-time complement of their [`CastReduce`] rule and can compute +/// statistics (e.g. min of the validity array) when the reduce rule had to give up. +/// Encodings that delegate to scalars or storage (e.g. [`Null`], [`Constant`], [`Extension`]) +/// only implement [`CastReduce`] because they never need execution-level information. fn cast_canonical( canonical: CanonicalView<'_>, dtype: &DType, @@ -206,12 +212,12 @@ fn cast_canonical( ) -> VortexResult> { match canonical { CanonicalView::Null(a) => ::cast(a, dtype), - CanonicalView::Bool(a) => ::cast(a, dtype), + CanonicalView::Bool(a) => ::cast(a, dtype, ctx), CanonicalView::Primitive(a) => ::cast(a, dtype, ctx), CanonicalView::Decimal(a) => ::cast(a, dtype, ctx), - CanonicalView::VarBinView(a) => ::cast(a, dtype), - CanonicalView::List(a) => ::cast(a, dtype), - CanonicalView::FixedSizeList(a) => ::cast(a, dtype), + CanonicalView::VarBinView(a) => ::cast(a, dtype, ctx), + CanonicalView::List(a) => ::cast(a, dtype, ctx), + CanonicalView::FixedSizeList(a) => ::cast(a, dtype, ctx), CanonicalView::Struct(a) => ::cast(a, dtype, ctx), CanonicalView::Extension(a) => ::cast(a, dtype), CanonicalView::Variant(_) => { diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index 0389566bbba..b081783b809 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -12,7 +12,6 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; -use vortex_error::vortex_panic; use vortex_mask::Mask; use vortex_mask::MaskValues; @@ -295,12 +294,11 @@ impl Validity { _ => {} }; - let own_nullability = if matches!(self, Validity::NonNullable) { - Nullability::NonNullable - } else { - Nullability::Nullable - }; + if matches!(self, Validity::NonNullable) { + return Ok(Self::NonNullable); + } + // From here on we know that the validity is nullable let source = match self { Validity::NonNullable => BoolArray::from(BitBuffer::new_set(len)), Validity::AllValid => BoolArray::from(BitBuffer::new_set(len)), @@ -324,13 +322,10 @@ impl Validity { None, )?; - Ok(Self::from_array( - source.patch(&patches, ctx)?.into_array(), - own_nullability, - )) + Ok(Self::Array(source.patch(&patches, ctx)?.into_array())) } - /// Convert into a nullable variant + /// Convert into a nullable variant. #[inline] pub fn into_nullable(self) -> Validity { match self { @@ -339,9 +334,13 @@ impl Validity { } } - /// Convert into a non-nullable variant + /// Convert into a non-nullable variant, computing statistics if necessary. + /// + /// Returns `None` when the array contains invalid values (so the cast cannot be performed), + /// either because it is [`Validity::AllInvalid`] or because the validity array's minimum is + /// `false`. #[inline] - pub fn into_non_nullable(self, len: usize) -> Option { + pub fn into_non_nullable(self, len: usize, ctx: &mut ExecutionCtx) -> Option { match self { _ if len == 0 => Some(Validity::NonNullable), Self::NonNullable => Some(Self::NonNullable), @@ -350,7 +349,7 @@ impl Validity { Self::Array(is_valid) => { is_valid .statistics() - .compute_min::(&mut LEGACY_SESSION.create_execution_ctx()) + .compute_min::(ctx) .vortex_expect("validity array must support min") .then(|| { // min true => all true @@ -360,28 +359,92 @@ impl Validity { } } - /// Convert into a variant compatible with the given nullability, if possible. + /// Convert into a non-nullable variant without running execution. + /// + /// This is the cheap counterpart to [`Self::into_non_nullable`]: it inspects already-computed + /// statistics rather than triggering execution. + /// + /// Return values: + /// - `Ok(Some(NonNullable))` — the cast is provably safe. + /// - `Ok(None)` — We need to perform compute to determine whether cast is valid. Callers should fall back to [`Self::into_non_nullable`], typically by + /// returning `Ok(None)` from a `CastReduce` rule so the corresponding `CastKernel` runs. + /// - `Err(_)` — we know the cast must fail (e.g. [`Validity::AllInvalid`]). + #[inline] + pub fn trivial_into_non_nullable(self, len: usize) -> VortexResult> { + match self { + _ if len == 0 => Ok(Some(Validity::NonNullable)), + Self::NonNullable => Ok(Some(Self::NonNullable)), + Self::AllValid => Ok(Some(Self::NonNullable)), + Self::AllInvalid => { + Err(vortex_err!(InvalidArgument: "Cannot cast AllInvalid to NonNullable")) + } + Self::Array(_) => Ok(None), + } + } + + /// Convert into a variant compatible with the given nullability. + /// + /// This is the execution-time half of the nullability-cast pair. It is paired with + /// [`Self::trivial_cast_nullability`], which is used by `CastReduce` rules. The pattern is: + /// + /// - **`CastReduce` rules** (metadata-only rewrites in the optimizer) call + /// [`Self::trivial_cast_nullability`]. If it returns `Ok(None)`, the rule returns `Ok(None)` + /// and the cast is deferred to execution. + /// - **`CastKernel` impls** (executed via [`ExecuteParentKernel`]) call this method, which + /// may run the underlying validity array to compute statistics. + /// + /// Returns `Err` when nullability cannot be cast (for example, casting to non-nullable while + /// invalid values are present). + /// + /// [`ExecuteParentKernel`]: crate::kernel::ExecuteParentKernel #[inline] - pub fn cast_nullability(self, nullability: Nullability, len: usize) -> VortexResult { + pub fn cast_nullability( + self, + nullability: Nullability, + len: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { match nullability { - Nullability::NonNullable => self.into_non_nullable(len).ok_or_else(|| { + Nullability::NonNullable => self.into_non_nullable(len, ctx).ok_or_else(|| { vortex_err!(InvalidArgument: "Cannot cast array with invalid values to non-nullable type.") }), Nullability::Nullable => Ok(self.into_nullable()), } } - /// Create Validity from boolean array with given nullability of the array. + /// Best-effort, non-executing variant of [`Self::cast_nullability`]. /// - /// Note: You want to pass the nullability of parent array and not the nullability of the validity array itself - /// as that is always nonnullable - fn from_array(value: ArrayRef, nullability: Nullability) -> Self { - if !matches!(value.dtype(), DType::Bool(Nullability::NonNullable)) { - vortex_panic!("Expected a non-nullable boolean array") - } + /// Use this from `CastReduce` rules — they run inside the optimizer where execution is not + /// available. The pairing with [`Self::cast_nullability`] is symmetric: every encoding that + /// implements `CastReduce` and inspects validity should also implement `CastKernel` so that + /// the harder cases (where statistics are not yet cached) can still be handled at execution + /// time. + /// + /// Return values: + /// - `Ok(Some(_))` — the cast is provably safe and the new [`Validity`] is returned. + /// - `Ok(None)` — the cast cannot be reduced cheaply (the `CastKernel` should be tried via + /// [`Self::cast_nullability`]). + /// - `Err(_)` — the cast is provably impossible. + /// + /// Typical usage inside a `CastReduce`: + /// + /// ```ignore + /// let Some(new_validity) = array + /// .validity()? + /// .trivial_cast_nullability(dtype.nullability(), array.len())? + /// else { + /// return Ok(None); + /// }; + /// ``` + #[inline] + pub fn trivial_cast_nullability( + self, + nullability: Nullability, + len: usize, + ) -> VortexResult> { match nullability { - Nullability::NonNullable => Self::NonNullable, - Nullability::Nullable => Self::Array(value), + Nullability::NonNullable => self.trivial_into_non_nullable(len), + Nullability::Nullable => Ok(Some(self.into_nullable())), } } @@ -393,15 +456,6 @@ impl Validity { Self::Array(a) => Some(a.len()), } } - - #[inline] - pub fn uncompressed_size(&self) -> usize { - if let Validity::Array(a) = self { - a.len().div_ceil(8) - } else { - 0 - } - } } impl From for Validity { diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 7dc1fccad80..dfd61deb80b 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -670,14 +670,8 @@ impl Scheme for RunEndScheme { // SAFETY: compression doesn't affect invariants. Ok(unsafe { - RunEnd::new_unchecked( - compressed_ends, - compressed_values, - 0, - data.array_len(), - exec_ctx, - ) - .into_array() + RunEnd::new_unchecked(compressed_ends, compressed_values, 0, data.array_len()) + .into_array() }) } } diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 485fff7ee30..16360ff287d 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; @@ -103,7 +102,7 @@ impl ZoneMap { return applied.execute::(&mut ctx); } - let row_count_array = row_count_array(self.zone_len, self.row_count, num_zones, &mut ctx)?; + let row_count_array = row_count_array(self.zone_len, self.row_count, num_zones)?; let substituted = substitute_row_count(applied, &row_count_array)?; substituted.execute::(&mut ctx) } @@ -114,12 +113,7 @@ impl ZoneMap { /// `zone_len` is the nominal zone size; only the final zone may be shorter. The /// result is a [`ConstantArray`] for uniform zone sizes, otherwise a two-run /// run-end encoded array whose trailing run carries the final zone length. -fn row_count_array( - zone_len: u64, - row_count: u64, - num_zones: usize, - ctx: &mut ExecutionCtx, -) -> VortexResult { +fn row_count_array(zone_len: u64, row_count: u64, num_zones: usize) -> VortexResult { let last_zone_len = row_count - zone_len.saturating_mul((num_zones as u64) - 1); if num_zones == 1 || last_zone_len == zone_len { return Ok(ConstantArray::new(last_zone_len, num_zones).into_array()); @@ -139,7 +133,7 @@ fn row_count_array( // SAFETY: `ends` are strictly increasing, terminate at `num_zones`, and align one-to-one // with the non-null run values. - Ok(unsafe { RunEnd::new_unchecked(ends, values, 0, num_zones, ctx) }.into_array()) + Ok(unsafe { RunEnd::new_unchecked(ends, values, 0, num_zones) }.into_array()) } #[cfg(test)]