diff --git a/AGENTS.md b/AGENTS.md
index 569fac7cc42..6ba6006251f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -64,11 +64,14 @@ Examples:
cargo nextest run -p vortex-array
make -C docs doctest
uv run --all-packages pytest vortex-python/test
+cargo test --doc
```
Run docs doctests from the docs directory with `make -C docs doctest` so the correct Sphinx
Makefile target is used.
+If you touch documentation run doc tests via `cargo test --doc`.
+
## Linting, Formatting, and Generated Files
Run verification that matches the files changed. Do not run expensive Rust checks for changes that
diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs
index 45adf5a46cd..71b51ecf9af 100644
--- a/encodings/alp/src/alp/compute/cast.rs
+++ b/encodings/alp/src/alp/compute/cast.rs
@@ -6,7 +6,6 @@ use vortex_array::ArrayView;
use vortex_array::IntoArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
-use vortex_array::patches::Patches;
use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_error::VortexResult;
@@ -17,41 +16,29 @@ use crate::alp::ALP;
impl CastReduce for ALP {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
// Check if this is just a nullability change
- if array.dtype().eq_ignore_nullability(dtype) {
- // For nullability-only changes, we can avoid decoding
- // Cast the encoded array (integers) to handle nullability
- let new_encoded = array.encoded().cast(
- array
- .encoded()
- .dtype()
- .with_nullability(dtype.nullability()),
- )?;
-
- let new_patches = array
- .patches()
- .map(|p| {
- if p.values().dtype() == dtype {
- Ok(p)
- } else {
- Patches::new(
- p.array_len(),
- p.offset(),
- p.indices().clone(),
- p.values().cast(dtype.clone())?,
- p.chunk_offsets().clone(),
- )
- }
- })
- .transpose()?;
-
- // SAFETY: casting nullability doesn't alter the invariants
- unsafe {
- Ok(Some(
- ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(),
- ))
- }
- } else {
- Ok(None)
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
+ }
+
+ // For nullability-only changes, we can avoid decoding
+ // Cast the encoded array (integers) to handle nullability
+ let new_encoded = array.encoded().cast(
+ array
+ .encoded()
+ .dtype()
+ .with_nullability(dtype.nullability()),
+ )?;
+
+ let new_patches = array
+ .patches()
+ .map(|p| p.map_values(|v| v.cast(dtype.clone())))
+ .transpose()?;
+
+ // SAFETY: casting nullability doesn't alter the invariants
+ unsafe {
+ Ok(Some(
+ ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(),
+ ))
}
}
}
diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs
index 9086877c2ce..c110475b880 100644
--- a/encodings/alp/src/alp_rd/compute/cast.rs
+++ b/encodings/alp/src/alp_rd/compute/cast.rs
@@ -4,8 +4,6 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::scalar_fn::fns::cast::CastReduce;
@@ -16,38 +14,35 @@ use crate::alp_rd::ALPRD;
impl CastReduce for ALPRD {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult > {
- // ALPRDArray stores floating-point values, so only cast between float types
- // or if just changing nullability
-
// Check if this is just a nullability change
- if array.dtype().eq_ignore_nullability(dtype) {
- // For nullability-only changes, we need to cast the left_parts array
- // since it carries the validity information
- let new_left_parts = array.left_parts().cast(
- array
- .left_parts()
- .dtype()
- .with_nullability(dtype.nullability()),
- )?;
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
+ }
+
+ // For nullability-only changes, we need to cast the left_parts array
+ // since it carries the validity information
+ let new_left_parts = array.left_parts().cast(
+ array
+ .left_parts()
+ .dtype()
+ .with_nullability(dtype.nullability()),
+ )?;
- // NOTE: `CastReduce::cast` has a fixed trait signature without `ExecutionCtx`, so we
- // construct a legacy ctx locally at this trait boundary.
- return Ok(Some(
- ALPRD::try_new(
+ // NOTE: `CastReduce::cast` has a fixed trait signature without `ExecutionCtx`, so we
+ // construct a legacy ctx locally at this trait boundary.
+ Ok(Some(
+ unsafe {
+ ALPRD::new_unchecked(
dtype.clone(),
new_left_parts,
array.left_parts_dictionary().clone(),
array.right_parts().clone(),
array.right_bit_width(),
array.left_parts_patches(),
- &mut LEGACY_SESSION.create_execution_ctx(),
- )?
- .into_array(),
- ));
- }
-
- // For other casts (e.g., f32 to f64), decode to canonical and let PrimitiveArray handle it
- Ok(None)
+ )
+ }
+ .into_array(),
+ ))
}
}
@@ -99,12 +94,17 @@ mod tests {
let encoder = RDEncoder::new(&values);
let alprd = encoder.encode(arr.as_view(), &mut ctx);
- // Cast to NonNullable should fail since we have nulls
+ // Cast to NonNullable should fail since we have nulls. The failure surfaces during
+ // execution since the reduce path defers when the validity stat is not cached.
let result = alprd
.clone()
.into_array()
- .cast(DType::Primitive(PType::F64, Nullability::NonNullable));
- assert!(result.is_err());
+ .cast(DType::Primitive(PType::F64, Nullability::NonNullable))
+ .and_then(|a| {
+ a.execute::(&mut ctx)
+ .map(|p| p.into_array())
+ });
+ assert!(result.is_err(), "Expected error, got: {result:?}");
// Cast to same type with Nullable should succeed
let casted = alprd
diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock
index 53c71e3bfb6..e9f4944310f 100644
--- a/encodings/bytebool/public-api.lock
+++ b/encodings/bytebool/public-api.lock
@@ -64,6 +64,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_bytebool::ByteBool
pub fn vortex_bytebool::ByteBool::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_bytebool::ByteBool
+
+pub fn vortex_bytebool::ByteBool::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_bytebool::ByteBool
pub fn vortex_bytebool::ByteBool::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs
index 609a7bc553f..629a7ebe3e2 100644
--- a/encodings/bytebool/src/compute.rs
+++ b/encodings/bytebool/src/compute.rs
@@ -11,6 +11,7 @@ use vortex_array::arrays::dict::TakeExecute;
use vortex_array::buffer::BufferHandle;
use vortex_array::dtype::DType;
use vortex_array::match_each_integer_ptype;
+use vortex_array::scalar_fn::fns::cast::CastKernel;
use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_array::scalar_fn::fns::mask::MaskReduce;
use vortex_array::validity::Validity;
@@ -24,20 +25,43 @@ impl CastReduce for ByteBool {
// ByteBool is essentially a bool array stored as bytes
// The main difference from BoolArray is the storage format
// For casting, we can decode to canonical (BoolArray) and let it handle the cast
-
// If just changing nullability, we can optimize
- if array.dtype().eq_ignore_nullability(dtype) {
- let new_validity = array
- .validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
+ if !dtype.is_boolean() {
+ return Ok(None);
+ }
+
+ let Some(new_validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
+
+ Ok(Some(
+ ByteBool::new(array.buffer().clone(), new_validity).into_array(),
+ ))
+ }
+}
- return Ok(Some(
- ByteBool::new(array.buffer().clone(), new_validity).into_array(),
- ));
+impl CastKernel for ByteBool {
+ fn cast(
+ array: ArrayView<'_, Self>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult> {
+ // Only handle nullability changes here; non-bool targets fall through to canonicalization.
+ if !dtype.is_boolean() {
+ return Ok(None);
}
- // For other casts, decode to canonical and let BoolArray handle it
- Ok(None)
+ let new_validity =
+ array
+ .validity()?
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
+
+ Ok(Some(
+ ByteBool::new(array.buffer().clone(), new_validity).into_array(),
+ ))
}
}
diff --git a/encodings/bytebool/src/kernel.rs b/encodings/bytebool/src/kernel.rs
index 4373520039a..ddf9679b3d5 100644
--- a/encodings/bytebool/src/kernel.rs
+++ b/encodings/bytebool/src/kernel.rs
@@ -3,8 +3,11 @@
use vortex_array::arrays::dict::TakeExecuteAdaptor;
use vortex_array::kernel::ParentKernelSet;
+use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor;
use crate::ByteBool;
-pub(crate) const PARENT_KERNELS: ParentKernelSet =
- ParentKernelSet::new(&[ParentKernelSet::lift(&TakeExecuteAdaptor(ByteBool))]);
+pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(ByteBool)),
+ ParentKernelSet::lift(&TakeExecuteAdaptor(ByteBool)),
+]);
diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs
index b313a165b79..882d07bbaef 100644
--- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs
+++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs
@@ -7,39 +7,31 @@ use vortex_array::IntoArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::scalar_fn::fns::cast::CastReduce;
-use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use crate::DecimalByteParts;
use crate::decimal_byte_parts::DecimalBytePartsArrayExt;
+
impl CastReduce for DecimalByteParts {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
+ // Check if this is just a nullability change
+ if !dtype.eq_ignore_nullability(array.dtype()) {
+ return Ok(None);
+ }
// DecimalBytePartsArray can only have Decimal dtype, so we only handle decimal-to-decimal casts
let DType::Decimal(target_decimal, target_nullability) = dtype else {
// Cannot cast decimal to non-decimal types - delegate to canonical form
return Ok(None);
};
- // Check if this is just a nullability change
- if array
- .dtype()
- .as_decimal_opt()
- .vortex_expect("must be a decimal dtype")
- == target_decimal
- && array.dtype().nullability() != *target_nullability
- {
- // Cast the msp array to handle nullability change
- let new_msp = array
- .msp()
- .cast(array.msp().dtype().with_nullability(*target_nullability))?;
-
- return Ok(Some(
- DecimalByteParts::try_new(new_msp, *target_decimal)?.into_array(),
- ));
- }
+ // Cast the msp array to handle nullability change
+ let new_msp = array
+ .msp()
+ .cast(array.msp().dtype().with_nullability(*target_nullability))?;
- // For precision/scale changes, decode to canonical and let DecimalArray handle it
- Ok(None)
+ Ok(Some(
+ DecimalByteParts::try_new(new_msp, *target_decimal)?.into_array(),
+ ))
}
}
diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock
index 59dfbb4ea4f..1f800cf8b22 100644
--- a/encodings/fastlanes/public-api.lock
+++ b/encodings/fastlanes/public-api.lock
@@ -186,6 +186,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_fastlanes::BitPacked
pub fn vortex_fastlanes::BitPacked::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fastlanes::BitPacked
+
+pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes::BitPacked
pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs
index 9f23750398a..75b75a51e99 100644
--- a/encodings/fastlanes/src/bitpacking/array/mod.rs
+++ b/encodings/fastlanes/src/bitpacking/array/mod.rs
@@ -334,19 +334,25 @@ impl> BitPackedArrayExt for T {}
#[cfg(test)]
mod test {
+ use std::sync::LazyLock;
+
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
+ use vortex_array::session::ArraySession;
use vortex_buffer::Buffer;
+ use vortex_session::VortexSession;
use crate::BitPackedData;
use crate::bitpacking::array::BitPackedArrayExt;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_encode() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = [
Some(1u64),
None,
@@ -369,7 +375,7 @@ mod test {
#[test]
fn test_encode_too_wide() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = [Some(1u8), None, Some(1), None, Some(1), None];
let uncompressed = PrimitiveArray::from_option_iter(values);
let _packed = BitPackedData::encode(&uncompressed.clone().into_array(), 8, &mut ctx)
@@ -380,7 +386,7 @@ mod test {
#[test]
fn signed_with_patches() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values: Buffer = (0i32..=512).collect();
let parray = values.clone().into_array();
diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs
index 8843c8e3013..86bd701e2ee 100644
--- a/encodings/fastlanes/src/bitpacking/compute/cast.rs
+++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs
@@ -3,48 +3,67 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
+use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
-use vortex_array::patches::Patches;
+use vortex_array::scalar_fn::fns::cast::CastKernel;
use vortex_array::scalar_fn::fns::cast::CastReduce;
+use vortex_array::validity::Validity;
use vortex_error::VortexResult;
use crate::bitpacking::BitPacked;
use crate::bitpacking::array::BitPackedArrayExt;
+
+fn build_with_validity(
+ array: ArrayView<'_, BitPacked>,
+ dtype: &DType,
+ new_validity: Validity,
+) -> VortexResult {
+ Ok(BitPacked::try_new(
+ array.packed().clone(),
+ dtype.as_ptype(),
+ new_validity,
+ array
+ .patches()
+ .map(|patches| patches.map_values(|values| values.cast(dtype.clone())))
+ .transpose()?,
+ array.bit_width(),
+ array.len(),
+ array.offset(),
+ )?
+ .into_array())
+}
+
impl CastReduce for BitPacked {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
- if array.dtype().eq_ignore_nullability(dtype) {
- let new_validity = array
- .validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
- return Ok(Some(
- BitPacked::try_new(
- array.packed().clone(),
- dtype.as_ptype(),
- new_validity,
- array
- .patches()
- .map(|patches| {
- let new_values = patches.values().cast(dtype.clone())?;
- Patches::new(
- patches.array_len(),
- patches.offset(),
- patches.indices().clone(),
- new_values,
- patches.chunk_offsets().clone(),
- )
- })
- .transpose()?,
- array.bit_width(),
- array.len(),
- array.offset(),
- )?
- .into_array(),
- ));
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
}
+ let Some(new_validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
+ build_with_validity(array, dtype, new_validity).map(Some)
+ }
+}
- Ok(None)
+impl CastKernel for BitPacked {
+ fn cast(
+ array: ArrayView<'_, Self>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
+ }
+ let new_validity =
+ array
+ .validity()?
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
+ build_with_validity(array, dtype, new_validity).map(Some)
}
}
diff --git a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs
index 128ff77d99c..cf975602179 100644
--- a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs
+++ b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs
@@ -4,10 +4,12 @@
use vortex_array::arrays::dict::TakeExecuteAdaptor;
use vortex_array::arrays::filter::FilterExecuteAdaptor;
use vortex_array::kernel::ParentKernelSet;
+use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor;
use crate::BitPacked;
pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(BitPacked)),
ParentKernelSet::lift(&FilterExecuteAdaptor(BitPacked)),
ParentKernelSet::lift(&TakeExecuteAdaptor(BitPacked)),
]);
diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs
index 9289e0a83fc..c11a6c4ec51 100644
--- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs
+++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs
@@ -153,18 +153,6 @@ impl VTable for BitPacked {
}
}
- fn reduce_parent(
- array: ArrayView<'_, Self>,
- parent: &ArrayRef,
- child_idx: usize,
- ) -> VortexResult> {
- RULES.evaluate(array, parent, child_idx)
- }
-
- fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
- BitPackedSlots::NAMES[idx].to_string()
- }
-
fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
@@ -283,6 +271,10 @@ impl VTable for BitPacked {
})
}
+ fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
+ BitPackedSlots::NAMES[idx].to_string()
+ }
+
fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult {
require_patches!(
array,
@@ -305,6 +297,14 @@ impl VTable for BitPacked {
) -> VortexResult> {
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
}
+
+ fn reduce_parent(
+ array: ArrayView<'_, Self>,
+ parent: &ArrayRef,
+ child_idx: usize,
+ ) -> VortexResult > {
+ RULES.evaluate(array, parent, child_idx)
+ }
}
#[derive(Clone, Debug)]
diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs
index 4beb558dca0..324e2fc9c22 100644
--- a/encodings/fastlanes/src/delta/compute/cast.rs
+++ b/encodings/fastlanes/src/delta/compute/cast.rs
@@ -9,24 +9,17 @@ use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability::NonNullable;
use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_error::VortexResult;
-use vortex_error::vortex_panic;
use crate::delta::Delta;
use crate::delta::array::DeltaArrayExt;
+
impl CastReduce for Delta {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult > {
- // Delta encoding stores differences between consecutive values, which requires
- // unsigned integers to avoid overflow issues. Signed integers could produce
- // negative deltas that wouldn't fit in the unsigned delta representation.
- // This encoding is optimized for monotonically increasing sequences.
let DType::Primitive(target_ptype, _) = dtype else {
return Ok(None);
};
- let DType::Primitive(source_ptype, _) = array.dtype() else {
- vortex_panic!("delta should be primitive typed");
- };
-
+ let source_ptype = array.dtype().as_ptype();
// TODO(DK): narrows can be safe but we must decompress to compute the maximum value.
if target_ptype.is_signed_int() || source_ptype.bit_width() > target_ptype.bit_width() {
return Ok(None);
diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs
index bb5add65aa0..17e571ba4f6 100644
--- a/encodings/fastlanes/src/delta/vtable/mod.rs
+++ b/encodings/fastlanes/src/delta/vtable/mod.rs
@@ -39,6 +39,7 @@ use crate::delta::array::DeltaArrayExt;
use crate::delta::array::SLOT_NAMES;
use crate::delta::array::delta_decompress::delta_decompress;
use crate::delta::array::lane_count;
+use crate::delta_compress;
mod operations;
mod rules;
@@ -200,7 +201,7 @@ impl Delta {
ctx: &mut ExecutionCtx,
) -> VortexResult {
let logical_len = array.len();
- let (bases, deltas) = crate::delta::array::delta_compress::delta_compress(array, ctx)?;
+ let (bases, deltas) = delta_compress(array, ctx)?;
Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len)
}
}
diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs
index 23231b10b12..e01d359e59b 100644
--- a/encodings/fastlanes/src/rle/compute/cast.rs
+++ b/encodings/fastlanes/src/rle/compute/cast.rs
@@ -12,6 +12,7 @@ use vortex_error::VortexResult;
use crate::rle::RLE;
use crate::rle::RLEArrayExt;
+
impl CastReduce for RLE {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
// Cast RLE values.
@@ -20,14 +21,12 @@ impl CastReduce for RLE {
.cast(DType::Primitive(dtype.as_ptype(), Nullability::NonNullable))?;
// Cast RLE indices such that validity matches the target dtype.
- let casted_indices = if array.indices().dtype().nullability() != dtype.nullability() {
- array.indices().cast(DType::Primitive(
- array.indices().dtype().as_ptype(),
- dtype.nullability(),
- ))?
- } else {
- array.indices().clone()
- };
+ let casted_indices = array.indices().cast(
+ array
+ .indices()
+ .dtype()
+ .with_nullability(dtype.nullability()),
+ )?;
Ok(Some(
RLE::try_new(
@@ -44,6 +43,8 @@ impl CastReduce for RLE {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
@@ -57,19 +58,24 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
use vortex_array::validity::Validity;
use vortex_buffer::Buffer;
+ use vortex_session::VortexSession;
use crate::RLEData;
use crate::rle::RLEArray;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
fn rle(primitive: &PrimitiveArray, ctx: &mut ExecutionCtx) -> RLEArray {
RLEData::encode(primitive.as_view(), ctx).unwrap()
}
#[test]
fn try_cast_rle_success() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let primitive = PrimitiveArray::new(
Buffer::from_iter([10u8, 20, 30, 40, 50]),
Validity::from_iter([true, true, true, true, true]),
@@ -86,7 +92,7 @@ mod tests {
#[test]
#[should_panic]
fn try_cast_rle_fail() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let primitive = PrimitiveArray::new(
Buffer::from_iter([10u8, 20, 30, 40, 50]),
Validity::from_iter([true, false, true, true, false]),
diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock
index 6e598e26c18..7f6bff24bcd 100644
--- a/encodings/fsst/public-api.lock
+++ b/encodings/fsst/public-api.lock
@@ -28,7 +28,7 @@ pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_,
pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option
-pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>
+pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>
pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult
@@ -70,6 +70,10 @@ impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_fss
pub fn vortex_fsst::FSST::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fsst::FSST
+
+pub fn vortex_fsst::FSST::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fsst::FSST
pub fn vortex_fsst::FSST::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs
index dd5214aadc0..36e57c51239 100644
--- a/encodings/fsst/src/array.rs
+++ b/encodings/fsst/src/array.rs
@@ -189,12 +189,13 @@ impl VTable for FSST {
metadata: &[u8],
buffers: &[BufferHandle],
children: &dyn ArrayChildren,
- _session: &VortexSession,
+ session: &VortexSession,
) -> VortexResult> {
let metadata = FSSTMetadata::decode(metadata)?;
let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?);
let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?);
+ let mut ctx = session.create_execution_ctx();
if buffers.len() == 2 {
return Self::deserialize_legacy(
self,
@@ -204,6 +205,7 @@ impl VTable for FSST {
&symbols,
&symbol_lengths,
children,
+ &mut ctx,
);
}
@@ -237,8 +239,6 @@ impl VTable for FSST {
vortex_bail!("Expected 2 or 3 children, got {}", children.len());
};
- // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
FSSTData::validate_parts(
&symbols,
&symbol_lengths,
@@ -413,6 +413,7 @@ impl FSST {
/// Legacy deserialization path (2 buffers): the codes were stored as a full
/// `VarBinArray` child. We decompose the VarBinArray into its bytes (stored in
/// FSSTData) and offsets/validity (stored in slots).
+ #[allow(clippy::too_many_arguments)]
fn deserialize_legacy(
&self,
dtype: &DType,
@@ -421,6 +422,7 @@ impl FSST {
symbols: &Buffer,
symbol_lengths: &Buffer,
children: &dyn ArrayChildren,
+ ctx: &mut ExecutionCtx,
) -> VortexResult> {
if children.len() != 2 {
vortex_bail!(InvalidArgument: "Expected 2 children, got {}", children.len());
@@ -444,8 +446,6 @@ impl FSST {
len,
)?;
- // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
FSSTData::validate_parts_from_codes(
symbols,
symbol_lengths,
@@ -453,7 +453,7 @@ impl FSST {
&uncompressed_lengths,
dtype,
len,
- &mut ctx,
+ ctx,
)?;
let slots = FSSTData::make_slots(&codes, &uncompressed_lengths);
let codes_bytes = codes.bytes_handle().clone();
diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs
index 550181af309..cd8c5e3a9c3 100644
--- a/encodings/fsst/src/compute/cast.rs
+++ b/encodings/fsst/src/compute/cast.rs
@@ -3,65 +3,114 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
+use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
-use vortex_array::arrays::VarBin;
-use vortex_array::builtins::ArrayBuiltins;
+use vortex_array::arrays::VarBinArray;
+use vortex_array::arrays::varbin::VarBinArrayExt;
use vortex_array::dtype::DType;
+use vortex_array::scalar_fn::fns::cast::CastKernel;
use vortex_array::scalar_fn::fns::cast::CastReduce;
+use vortex_array::validity::Validity;
use vortex_error::VortexResult;
use crate::FSST;
use crate::FSSTArrayExt;
+
+fn build_with_codes_validity(
+ array: ArrayView<'_, FSST>,
+ dtype: &DType,
+ new_codes_validity: Validity,
+) -> VortexResult {
+ let codes = array.codes();
+ let new_codes = VarBinArray::try_new(
+ codes.offsets().clone(),
+ codes.bytes().clone(),
+ codes.dtype().with_nullability(dtype.nullability()),
+ new_codes_validity,
+ )?;
+
+ Ok(unsafe {
+ FSST::new_unchecked(
+ dtype.clone(),
+ array.symbols().clone(),
+ array.symbol_lengths().clone(),
+ new_codes,
+ array.uncompressed_lengths().clone(),
+ )
+ }
+ .into_array())
+}
+
impl CastReduce for FSST {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
- // FSST is a string compression encoding.
- // For nullability changes, we can cast the codes and symbols arrays
- if array.dtype().eq_ignore_nullability(dtype) {
- // Cast codes array to handle nullability
- let new_codes = array
- .codes()
- .into_array()
- .cast(array.codes_dtype().with_nullability(dtype.nullability()))?;
-
- // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
- Ok(Some(
- FSST::try_new(
- dtype.clone(),
- array.symbols().clone(),
- array.symbol_lengths().clone(),
- new_codes.as_::().into_owned(),
- array.uncompressed_lengths().clone(),
- &mut ctx,
- )?
- .into_array(),
- ))
- } else {
- Ok(None)
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
}
+
+ let codes = array.codes();
+ let Some(new_codes_validity) = codes
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), codes.len())?
+ else {
+ return Ok(None);
+ };
+
+ Ok(Some(build_with_codes_validity(
+ array,
+ dtype,
+ new_codes_validity,
+ )?))
+ }
+}
+
+impl CastKernel for FSST {
+ fn cast(
+ array: ArrayView<'_, Self>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult> {
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
+ }
+
+ let codes = array.codes();
+ let new_codes_validity =
+ codes
+ .validity()?
+ .cast_nullability(dtype.nullability(), codes.len(), ctx)?;
+
+ Ok(Some(build_with_codes_validity(
+ array,
+ dtype,
+ new_codes_validity,
+ )?))
}
}
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::VarBinArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::compute::conformance::cast::test_cast_conformance;
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
+ use vortex_array::session::ArraySession;
+ use vortex_session::VortexSession;
use crate::fsst_compress;
use crate::fsst_train_compressor;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_fsst_nullability() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let strings = VarBinArray::from_iter(
vec![Some("hello"), Some("world"), Some("hello world")],
DType::Utf8(Nullability::NonNullable),
@@ -94,7 +143,7 @@ mod tests {
DType::Utf8(Nullability::NonNullable)
))]
fn test_cast_fsst_conformance(#[case] array: VarBinArray) {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let compressor = fsst_train_compressor(&array);
let fsst = fsst_compress(&array, array.len(), array.dtype(), &compressor, &mut ctx);
test_cast_conformance(&fsst.into_array());
diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs
index 079efcfecb0..7f455a06e16 100644
--- a/encodings/fsst/src/kernel.rs
+++ b/encodings/fsst/src/kernel.rs
@@ -5,11 +5,13 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor;
use vortex_array::arrays::filter::FilterExecuteAdaptor;
use vortex_array::kernel::ParentKernelSet;
use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor;
+use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor;
use vortex_array::scalar_fn::fns::like::LikeExecuteAdaptor;
use crate::FSST;
pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(FSST)),
ParentKernelSet::lift(&CompareExecuteAdaptor(FSST)),
ParentKernelSet::lift(&FilterExecuteAdaptor(FSST)),
ParentKernelSet::lift(&TakeExecuteAdaptor(FSST)),
diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs
index ad95badd7f3..7cb1a28bb9c 100644
--- a/encodings/pco/src/compute/cast.rs
+++ b/encodings/pco/src/compute/cast.rs
@@ -4,8 +4,6 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
use vortex_array::dtype::DType;
use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_array::vtable::child_to_validity;
@@ -13,53 +11,52 @@ use vortex_error::VortexResult;
use crate::Pco;
use crate::PcoData;
+
impl CastReduce for Pco {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
- if !dtype.is_nullable()
- || !array
- .array()
- .all_valid(&mut LEGACY_SESSION.create_execution_ctx())?
- {
- // TODO(joe): fixme
- // We cannot cast to non-nullable since the validity containing nulls is used to decode
- // the PCO array, this would require rewriting tables.
+ // PCO (Pcodec) stores compressed data and uses validity bits to decode (the validity
+ // tells PCO which logical positions correspond to compressed values). Casting away
+ // nullability would change the validity-to-compressed-value mapping, so we cannot
+ // construct a non-nullable Pco without re-encoding — we only handle nullability changes
+ // toward `Nullable`. Non-nullable targets fall through to canonicalization.
+ //
+ // No `CastKernel` is provided for the same reason: even with execution context, we
+ // cannot cast away nullability on a PCO array in place.
+ //
+ // PCO supports: F16, F32, F64, I16, I32, I64, U16, U32, U64.
+ if !array.dtype().eq_ignore_nullability(dtype) {
return Ok(None);
}
- // PCO (Pcodec) is a compression encoding that stores data in a compressed format.
- // It can efficiently handle nullability changes without decompression, but type changes
- // require decompression since the compression algorithm is type-specific.
- // PCO supports: F16, F32, F64, I16, I32, I64, U16, U32, U64
- if array.dtype().eq_ignore_nullability(dtype) {
- // Create a new validity with the target nullability
- let unsliced_validity =
- child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability());
- let new_validity =
- unsliced_validity.cast_nullability(dtype.nullability(), array.len())?;
-
- let data = PcoData::new(
- array.chunk_metas.clone(),
- array.pages.clone(),
- dtype.as_ptype(),
- array.metadata.clone(),
- array.unsliced_n_rows(),
- )
- ._slice(array.slice_start(), array.slice_stop());
-
- return Ok(Some(
- Pco::try_new(dtype.clone(), data, new_validity)?.into_array(),
- ));
- }
- // For other casts (e.g., numeric type changes), decode to canonical and let PrimitiveArray handle it
- Ok(None)
+ let unsliced_validity =
+ child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability());
+ let Some(new_validity) =
+ unsliced_validity.trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
+
+ let data = PcoData::new(
+ array.chunk_metas.clone(),
+ array.pages.clone(),
+ dtype.as_ptype(),
+ array.metadata.clone(),
+ array.unsliced_n_rows(),
+ )
+ ._slice(array.slice_start(), array.slice_stop());
+
+ Ok(Some(
+ Pco::try_new(dtype.clone(), data, new_validity)?.into_array(),
+ ))
}
}
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
@@ -68,14 +65,19 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
use vortex_array::validity::Validity;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::Pco;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_pco_f32_to_f64() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0]);
let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap();
@@ -96,7 +98,7 @@ mod tests {
#[test]
fn test_cast_pco_nullability_change() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
// Test casting from NonNullable to Nullable
let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]);
let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap();
@@ -113,7 +115,7 @@ mod tests {
#[test]
fn test_cast_sliced_pco_nullable_to_nonnullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::new(
buffer![10u32, 20, 30, 40, 50, 60],
Validity::from_iter([true, true, true, true, true, true]),
@@ -133,7 +135,7 @@ mod tests {
#[test]
fn test_cast_sliced_pco_part_valid_to_nonnullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::from_option_iter([
None,
Some(20u32),
@@ -176,7 +178,7 @@ mod tests {
Validity::NonNullable,
))]
fn test_cast_pco_conformance(#[case] values: PrimitiveArray) {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap();
test_cast_conformance(&pco.into_array());
}
diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock
index c768bb61f62..2f11d91d5ad 100644
--- a/encodings/runend/public-api.lock
+++ b/encodings/runend/public-api.lock
@@ -24,11 +24,11 @@ pub fn vortex_runend::RunEnd::encode(array: vortex_array::array::erased::ArrayRe
pub fn vortex_runend::RunEnd::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray
-pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray
+pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_runend::RunEndArray
pub fn vortex_runend::RunEnd::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult
-pub fn vortex_runend::RunEnd::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult
+pub fn vortex_runend::RunEnd::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult
impl core::clone::Clone for vortex_runend::RunEnd
diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs
index a8f99a42e94..7fe1c70461d 100644
--- a/encodings/runend/src/array.rs
+++ b/encodings/runend/src/array.rs
@@ -256,12 +256,9 @@ impl RunEnd {
values: ArrayRef,
offset: usize,
length: usize,
- ctx: &mut ExecutionCtx,
) -> RunEndArray {
let dtype = values.dtype().clone();
- let slots = vec![Some(ends.clone()), Some(values.clone())];
- RunEndData::validate_parts(&ends, &values, offset, length, ctx)
- .vortex_expect("RunEndArray validation failed");
+ let slots = vec![Some(ends), Some(values)];
let data = unsafe { RunEndData::new_unchecked(offset) };
unsafe {
Array::from_parts_unchecked(
@@ -277,6 +274,7 @@ impl RunEnd {
ctx: &mut ExecutionCtx,
) -> VortexResult {
let len = RunEndData::logical_len_from_ends(&ends, ctx)?;
+ RunEndData::validate_parts(&ends, &values, 0, len, ctx)?;
let dtype = values.dtype().clone();
let slots = vec![Some(ends), Some(values)];
let data = RunEndData::new(0);
@@ -289,7 +287,9 @@ impl RunEnd {
values: ArrayRef,
offset: usize,
length: usize,
+ ctx: &mut ExecutionCtx,
) -> VortexResult {
+ RunEndData::validate_parts(&ends, &values, offset, length, ctx)?;
let dtype = values.dtype().clone();
let slots = vec![Some(ends), Some(values)];
let data = RunEndData::new(offset);
@@ -469,20 +469,15 @@ impl ValidityVTable for RunEnd {
Ok(match array.values().validity()? {
Validity::NonNullable | Validity::AllValid => Validity::AllValid,
Validity::AllInvalid => Validity::AllInvalid,
- Validity::Array(values_validity) => {
- // TODO(ctx): trait fixes - ValidityVTable::validity has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
- Validity::Array(unsafe {
- RunEnd::new_unchecked(
- array.ends().clone(),
- values_validity,
- array.offset(),
- array.len(),
- &mut ctx,
- )
- .into_array()
- })
- }
+ Validity::Array(values_validity) => Validity::Array(unsafe {
+ RunEnd::new_unchecked(
+ array.ends().clone(),
+ values_validity,
+ array.offset(),
+ array.len(),
+ )
+ .into_array()
+ }),
})
}
}
@@ -515,8 +510,9 @@ pub(super) fn run_end_canonicalize(
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::DictArray;
use vortex_array::arrays::VarBinViewArray;
@@ -524,13 +520,18 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::RunEnd;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_runend_constructor() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let arr = RunEnd::new(
buffer![2u32, 5, 10].into_array(),
buffer![1i32, 2, 3].into_array(),
@@ -551,7 +552,7 @@ mod tests {
#[test]
fn test_runend_utf8() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = VarBinViewArray::from_iter_str(["a", "b", "c"]).into_array();
let arr = RunEnd::new(buffer![2u32, 5, 10].into_array(), values, &mut ctx);
assert_eq!(arr.len(), 10);
@@ -565,7 +566,7 @@ mod tests {
#[test]
fn test_runend_dict() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let dict_values = VarBinViewArray::from_iter_str(["x", "y", "z"]).into_array();
let dict_codes = buffer![0u32, 1, 2].into_array();
let dict = DictArray::try_new(dict_codes, dict_values).unwrap();
diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs
index f423dcaa60e..0b1d7c2ca3e 100644
--- a/encodings/runend/src/arrow.rs
+++ b/encodings/runend/src/arrow.rs
@@ -20,6 +20,7 @@ use vortex_error::VortexResult;
use crate::RunEndData;
use crate::ops::find_slice_end_index;
+
impl FromArrowArray<&RunArray> for RunEndData
where
R::Native: NativePType,
@@ -147,7 +148,13 @@ mod tests {
)
};
- RunEnd::try_new_offset_length(ends_slice, values_slice, offset, array.len())
+ RunEnd::try_new_offset_length(
+ ends_slice,
+ values_slice,
+ offset,
+ array.len(),
+ &mut SESSION.create_execution_ctx(),
+ )
}
#[test]
diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs
index 8b5ada03ff3..f70bb0becb5 100644
--- a/encodings/runend/src/compute/cast.rs
+++ b/encodings/runend/src/compute/cast.rs
@@ -4,8 +4,6 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::scalar_fn::fns::cast::CastReduce;
@@ -18,8 +16,6 @@ impl CastReduce for RunEnd {
// Cast the values array to the target type
let casted_values = array.values().cast(dtype.clone())?;
- // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
// SAFETY: casting does not affect the ends being valid
unsafe {
Ok(Some(
@@ -28,7 +24,6 @@ impl CastReduce for RunEnd {
casted_values,
array.offset(),
array.len(),
- &mut ctx,
)
.into_array(),
))
@@ -38,9 +33,10 @@ impl CastReduce for RunEnd {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::BoolArray;
use vortex_array::arrays::PrimitiveArray;
@@ -50,14 +46,19 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::RunEnd;
use crate::RunEndArray;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_runend_i32_to_i64() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let runend = RunEnd::try_new(
buffer![3u64, 5, 8, 10].into_array(),
buffer![100i32, 200, 100, 300].into_array(),
@@ -98,7 +99,7 @@ mod tests {
#[test]
fn test_cast_runend_nullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let runend = RunEnd::try_new(
buffer![2u64, 4, 7].into_array(),
PrimitiveArray::from_option_iter([Some(10i32), None, Some(20)]).into_array(),
@@ -118,7 +119,7 @@ mod tests {
#[test]
fn test_cast_runend_with_offset() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
// Create a RunEndArray: [100, 100, 100, 200, 200, 300, 300, 300, 300, 300]
let runend = RunEnd::try_new(
buffer![3u64, 5, 10].into_array(),
@@ -174,7 +175,7 @@ mod tests {
ctx,
).unwrap())]
fn test_cast_runend_conformance(#[case] build: RunEndBuilder) {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let array = build(&mut ctx);
test_cast_conformance(&array.into_array());
}
diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs
index a4597950b95..ddce31184eb 100644
--- a/encodings/runend/src/compute/fill_null.rs
+++ b/encodings/runend/src/compute/fill_null.rs
@@ -4,8 +4,6 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::scalar::Scalar;
use vortex_array::scalar_fn::fns::fill_null::FillNullReduce;
@@ -20,8 +18,6 @@ impl FillNullReduce for RunEnd {
fill_value: &Scalar,
) -> VortexResult> {
let new_values = array.values().fill_null(fill_value.clone())?;
- // TODO(ctx): trait fixes - FillNullReduce::fill_null has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
// SAFETY: modifying values only, does not affect ends
Ok(Some(
unsafe {
@@ -30,7 +26,6 @@ impl FillNullReduce for RunEnd {
new_values,
array.offset(),
array.len(),
- &mut ctx,
)
}
.into_array(),
diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs
index d537845ee2b..60644e2bced 100644
--- a/encodings/runend/src/compute/filter.rs
+++ b/encodings/runend/src/compute/filter.rs
@@ -65,7 +65,6 @@ impl FilterKernel for RunEnd {
values,
0,
mask_values.true_count(),
- ctx,
)
.into_array(),
))
diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs
index 57bc207a385..32b22eb96c5 100644
--- a/encodings/runend/src/compute/take_from.rs
+++ b/encodings/runend/src/compute/take_from.rs
@@ -25,7 +25,7 @@ impl ExecuteParentKernel for RunEndTakeFrom {
array: ArrayView<'_, RunEnd>,
dict: ArrayView<'_, Dict>,
child_idx: usize,
- ctx: &mut ExecutionCtx,
+ _ctx: &mut ExecutionCtx,
) -> VortexResult> {
if child_idx != 0 {
return Ok(None);
@@ -44,10 +44,8 @@ impl ExecuteParentKernel for RunEndTakeFrom {
dict.values().take(array.values().clone())?,
array.offset(),
array.len(),
- ctx,
)
};
- //
Ok(Some(ree_array.into_array()))
}
}
@@ -109,7 +107,6 @@ mod tests {
codes.values().clone(),
2, // offset
3, // len
- &mut ctx,
)
};
@@ -134,7 +131,6 @@ mod tests {
codes.values().clone(),
3, // offset at exact run boundary
4, // len
- &mut ctx,
)
};
@@ -159,7 +155,6 @@ mod tests {
codes.values().slice(1..3)?,
4, // offset
1, // len
- &mut ctx,
)
};
diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs
index f619dae3d67..73f6ebc524e 100644
--- a/encodings/runend/src/kernel.rs
+++ b/encodings/runend/src/kernel.rs
@@ -72,7 +72,6 @@ fn slice(
array.values().slice(slice_begin..slice_end)?,
range.start + array.offset(),
new_length,
- ctx,
)
.into_array()
})
diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs
index 64c3287ae79..cd80e42db46 100644
--- a/encodings/runend/src/rules.rs
+++ b/encodings/runend/src/rules.rs
@@ -4,8 +4,6 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
-use vortex_array::VortexSessionExecute;
use vortex_array::arrays::Constant;
use vortex_array::arrays::ConstantArray;
use vortex_array::arrays::ScalarFnArray;
@@ -82,8 +80,6 @@ impl ArrayParentReduceRule for RunEndScalarFnRule {
ScalarFnArray::try_new(parent.scalar_fn().clone(), new_children, values_len)?
.into_array();
- // TODO(ctx): trait fixes - ArrayParentReduceRule::reduce_parent has a fixed signature.
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
Ok(Some(
unsafe {
RunEnd::new_unchecked(
@@ -91,7 +87,6 @@ impl ArrayParentReduceRule for RunEndScalarFnRule {
new_values,
run_end.offset(),
run_end.len(),
- &mut ctx,
)
}
.into_array(),
diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs
index c532689a275..e6d64fdf7c5 100644
--- a/encodings/sequence/src/compute/cast.rs
+++ b/encodings/sequence/src/compute/cast.rs
@@ -87,9 +87,10 @@ impl CastReduce for Sequence {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
@@ -98,10 +99,15 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
+ use vortex_session::VortexSession;
use crate::Sequence;
use crate::SequenceArray;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_sequence_nullability() {
let sequence = Sequence::try_new_typed(0u32, 1u32, Nullability::NonNullable, 4).unwrap();
@@ -119,7 +125,7 @@ mod tests {
#[test]
fn test_cast_sequence_u32_to_i64() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let sequence = Sequence::try_new_typed(100u32, 10u32, Nullability::NonNullable, 4).unwrap();
let casted = sequence
@@ -138,7 +144,7 @@ mod tests {
#[test]
fn test_cast_sequence_i16_to_i32_nullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
// Test ptype change AND nullability change in one cast
let sequence = Sequence::try_new_typed(5i16, 3i16, Nullability::NonNullable, 3).unwrap();
@@ -161,7 +167,7 @@ mod tests {
#[test]
fn test_cast_sequence_to_float_delegates_to_canonical() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let sequence = Sequence::try_new_typed(0i32, 1i32, Nullability::NonNullable, 5).unwrap();
// Cast to float should delegate to canonical (SequenceArray doesn't support float)
diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs
index 8a27e72aed4..17455013720 100644
--- a/encodings/sparse/src/canonical.rs
+++ b/encodings/sparse/src/canonical.rs
@@ -52,7 +52,6 @@ use vortex_error::VortexError;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
-use vortex_error::vortex_panic;
use crate::ConstantArray;
use crate::Sparse;
@@ -83,7 +82,7 @@ pub(super) fn execute_sparse(
DType::Struct(struct_fields, ..) => execute_sparse_struct(
struct_fields,
array.fill_scalar().as_struct(),
- array.dtype(),
+ array.dtype().nullability(),
array.patches(),
array.len(),
ctx,
@@ -403,7 +402,7 @@ fn execute_sparse_primitives TryFrom<&'a Scalar, Error
fn execute_sparse_struct(
struct_fields: &StructFields,
fill_struct: StructScalar,
- dtype: &DType,
+ nullability: Nullability,
// Resolution is unnecessary b/c we're just pushing the patches into the fields.
unresolved_patches: &Patches,
len: usize,
@@ -425,28 +424,20 @@ fn execute_sparse_struct(
.execute::(ctx)?;
let columns_patch_values = patch_values_as_struct.unmasked_fields();
let names = patch_values_as_struct.names();
- let validity = if dtype.is_nullable() {
- top_level_fill_validity.patch(
- len,
- unresolved_patches.offset(),
- unresolved_patches.indices(),
- &Validity::from_mask(
- {
- let v = unresolved_patches.values();
- v.validity()
- .vortex_expect("validity_mask")
- .execute_mask(v.len(), ctx)
- .vortex_expect("Failed to compute validity mask")
- },
- Nullability::Nullable,
- ),
- ctx,
- )?
- } else {
- top_level_fill_validity
- .into_non_nullable(len)
- .unwrap_or_else(|| vortex_panic!("fill validity should match sparse array nullability"))
- };
+ let validity = top_level_fill_validity.patch(
+ len,
+ unresolved_patches.offset(),
+ unresolved_patches.indices(),
+ &Validity::from_mask(
+ patch_values_as_struct
+ .validity()
+ .vortex_expect("validity_mask")
+ .execute_mask(patch_values_as_struct.len(), ctx)
+ .vortex_expect("Failed to compute validity mask"),
+ nullability,
+ ),
+ ctx,
+ )?;
Ok(StructArray::try_from_iter_with_validity(
names.iter().zip_eq(
@@ -1215,35 +1206,35 @@ mod test {
let elements_slice = elements_array.as_slice::();
// List 0: [1]
- let list0_offset = result_listview.offset_at(0) as usize;
+ let list0_offset = result_listview.offset_at(0);
assert_eq!(elements_slice[list0_offset], 1);
// List 1: [5,6,7,8]
- let list1_offset = result_listview.offset_at(1) as usize;
- let list1_size = result_listview.size_at(1) as usize;
+ let list1_offset = result_listview.offset_at(1);
+ let list1_size = result_listview.size_at(1);
assert_eq!(
&elements_slice[list1_offset..list1_offset + list1_size],
&[5, 6, 7, 8]
);
// List 2: [5,6,7,8]
- let list2_offset = result_listview.offset_at(2) as usize;
- let list2_size = result_listview.size_at(2) as usize;
+ let list2_offset = result_listview.offset_at(2);
+ let list2_size = result_listview.size_at(2);
assert_eq!(
&elements_slice[list2_offset..list2_offset + list2_size],
&[5, 6, 7, 8]
);
// List 3: [2]
- let list3_offset = result_listview.offset_at(3) as usize;
+ let list3_offset = result_listview.offset_at(3);
assert_eq!(elements_slice[list3_offset], 2);
// List 4: [1]
- let list4_offset = result_listview.offset_at(4) as usize;
+ let list4_offset = result_listview.offset_at(4);
assert_eq!(elements_slice[list4_offset], 1);
// List 5: [2]
- let list5_offset = result_listview.offset_at(5) as usize;
+ let list5_offset = result_listview.offset_at(5);
assert_eq!(elements_slice[list5_offset], 2);
Ok(())
}
diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs
index 8ef09360e41..663539a6030 100644
--- a/encodings/sparse/src/compute/cast.rs
+++ b/encodings/sparse/src/compute/cast.rs
@@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_error::VortexResult;
use crate::Sparse;
+
impl CastReduce for Sparse {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
let casted_patches = array
@@ -34,9 +35,10 @@ impl CastReduce for Sparse {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
@@ -46,14 +48,19 @@ mod tests {
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
use vortex_array::scalar::Scalar;
+ use vortex_array::session::ArraySession;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::Sparse;
use crate::SparseArray;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_sparse_i32_to_i64() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let sparse = Sparse::try_new(
buffer![2u64, 5, 8].into_array(),
buffer![100i32, 200, 300].into_array(),
@@ -127,7 +134,7 @@ mod tests {
#[test]
fn test_cast_sparse_null_fill_all_patched_to_non_nullable() -> vortex_error::VortexResult<()> {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
// Regression test for https://github.com/vortex-data/vortex/issues/6932
//
// When all positions are patched the null fill is unused, so a cast to
diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs
index ae3032cdc03..e295e556566 100644
--- a/encodings/zstd/src/compute/cast.rs
+++ b/encodings/zstd/src/compute/cast.rs
@@ -7,12 +7,12 @@ use vortex_array::IntoArray;
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::scalar_fn::fns::cast::CastReduce;
-use vortex_array::validity::Validity;
use vortex_array::vtable::child_to_validity;
use vortex_error::VortexResult;
use crate::Zstd;
use crate::ZstdData;
+
impl CastReduce for Zstd {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> {
if !dtype.eq_ignore_nullability(array.dtype()) {
@@ -24,71 +24,57 @@ impl CastReduce for Zstd {
let src_nullability = array.dtype().nullability();
let target_nullability = dtype.nullability();
- match (src_nullability, target_nullability) {
+ let new_validity = match (src_nullability, target_nullability) {
// Same type case. This should be handled in the layer above but for
// completeness of the match arms we also handle it here.
(Nullability::Nullable, Nullability::Nullable)
| (Nullability::NonNullable, Nullability::NonNullable) => {
- Ok(Some(array.array().clone()))
+ return Ok(Some(array.array().clone()));
}
(Nullability::NonNullable, Nullability::Nullable) => {
// nonnull => null, trivial cast by altering the validity
- let unsliced_validity =
- child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability());
- Ok(Some(
- Zstd::try_new(
- dtype.clone(),
- ZstdData::new(
- array.dictionary.clone(),
- array.frames.clone(),
- array.metadata.clone(),
- array.unsliced_n_rows(),
- ),
- unsliced_validity,
- )?
- .into_array()
- .slice(array.slice_start()..array.slice_stop())?,
- ))
+ child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability())
}
(Nullability::Nullable, Nullability::NonNullable) => {
// null => non-null works if there are no nulls in the sliced range
let unsliced_validity =
child_to_validity(array.slots()[0].as_ref(), array.dtype().nullability());
- let has_nulls = !matches!(
- unsliced_validity.slice(array.slice_start()..array.slice_stop())?,
- Validity::AllValid | Validity::NonNullable
- );
+ let has_nulls = !unsliced_validity
+ .slice(array.slice_start()..array.slice_stop())?
+ .no_nulls();
// We don't attempt to handle casting when there are nulls.
if has_nulls {
return Ok(None);
}
-
- // If there are no nulls, the cast is trivial
- Ok(Some(
- Zstd::try_new(
- dtype.clone(),
- ZstdData::new(
- array.dictionary.clone(),
- array.frames.clone(),
- array.metadata.clone(),
- array.unsliced_n_rows(),
- ),
- unsliced_validity,
- )?
- .into_array()
- .slice(array.slice_start()..array.slice_stop())?,
- ))
+ unsliced_validity
}
- }
+ };
+
+ // If there are no nulls, the cast is trivial
+ Ok(Some(
+ Zstd::try_new(
+ dtype.clone(),
+ ZstdData::new(
+ array.dictionary.clone(),
+ array.frames.clone(),
+ array.metadata.clone(),
+ array.unsliced_n_rows(),
+ ),
+ new_validity,
+ )?
+ .into_array()
+ .slice(array.slice_start()..array.slice_stop())?,
+ ))
}
}
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
use vortex_array::IntoArray;
- use vortex_array::LEGACY_SESSION;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
@@ -97,14 +83,19 @@ mod tests {
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
+ use vortex_array::session::ArraySession;
use vortex_array::validity::Validity;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::Zstd;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_zstd_i32_to_i64() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]);
let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap();
@@ -123,7 +114,7 @@ mod tests {
#[test]
fn test_cast_zstd_nullability_change() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]);
let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap();
@@ -139,7 +130,7 @@ mod tests {
#[test]
fn test_cast_sliced_zstd_nullable_to_nonnullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::new(
buffer![10u32, 20, 30, 40, 50, 60],
Validity::from_iter([true, true, true, true, true, true]),
@@ -160,7 +151,7 @@ mod tests {
#[test]
fn test_cast_sliced_zstd_part_valid_to_nonnullable() {
- let mut ctx = LEGACY_SESSION.create_execution_ctx();
+ let mut ctx = SESSION.create_execution_ctx();
let values = PrimitiveArray::from_option_iter([
None,
Some(20u32),
@@ -201,8 +192,8 @@ mod tests {
Validity::NonNullable,
))]
fn test_cast_zstd_conformance(#[case] values: PrimitiveArray) {
- let zstd = Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx())
- .unwrap();
+ let zstd =
+ Zstd::from_primitive(&values, 0, 0, &mut SESSION.create_execution_ctx()).unwrap();
test_cast_conformance(&zstd.into_array());
}
}
diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock
index 903b9d633b9..0d52ce0a6ec 100644
--- a/vortex-array/public-api.lock
+++ b/vortex-array/public-api.lock
@@ -1432,6 +1432,10 @@ pub type vortex_array::arrays::bool::BoolMaskedValidityRule::Parent = vortex_arr
pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, child_idx: usize) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool
+
+pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Bool
pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -1948,7 +1952,11 @@ pub fn vortex_array::arrays::Decimal::between(arr: vortex_array::ArrayView<'_, v
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal
-pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal
+
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Decimal
@@ -2882,6 +2890,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSiz
pub fn vortex_array::arrays::FixedSizeList::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList
+
+pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::FixedSizeList
pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -3026,6 +3038,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::List
pub fn vortex_array::arrays::List::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List
+
+pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::List
pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -3202,6 +3218,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::ListView
pub fn vortex_array::arrays::ListView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView
+
+pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::ListView
pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -3914,6 +3934,10 @@ impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Pr
pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive
+
+pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Primitive
pub fn vortex_array::arrays::Primitive::fill_null(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, fill_value: &vortex_array::scalar::Scalar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
@@ -4640,7 +4664,11 @@ pub fn vortex_array::arrays::Struct::slice(array: vortex_array::ArrayView<'_, Se
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct
-pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct
+
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Struct
@@ -4806,6 +4834,10 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array
pub fn vortex_array::arrays::VarBin::compare(lhs: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin
+
+pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin
pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -5160,6 +5192,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinVi
pub fn vortex_array::arrays::VarBinView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView
+
+pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBinView
pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -5420,6 +5456,10 @@ pub type vortex_array::arrays::bool::BoolMaskedValidityRule::Parent = vortex_arr
pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, child_idx: usize) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool
+
+pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Bool
pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -5686,7 +5726,11 @@ pub fn vortex_array::arrays::Decimal::between(arr: vortex_array::ArrayView<'_, v
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal
-pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal
+
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Decimal
@@ -5992,6 +6036,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::FixedSiz
pub fn vortex_array::arrays::FixedSizeList::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList
+
+pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::FixedSizeList
pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -6070,6 +6118,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::List
pub fn vortex_array::arrays::List::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List
+
+pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::List
pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -6148,6 +6200,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::ListView
pub fn vortex_array::arrays::ListView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView
+
+pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::ListView
pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -6470,6 +6526,10 @@ impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Pr
pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive
+
+pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::fill_null::FillNullKernel for vortex_array::arrays::Primitive
pub fn vortex_array::arrays::Primitive::fill_null(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, fill_value: &vortex_array::scalar::Scalar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
@@ -6724,7 +6784,11 @@ pub fn vortex_array::arrays::Struct::slice(array: vortex_array::ArrayView<'_, Se
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct
-pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct
+
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Struct
@@ -6812,6 +6876,10 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array
pub fn vortex_array::arrays::VarBin::compare(lhs: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin
+
+pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin
pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -6886,6 +6954,10 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::VarBinVi
pub fn vortex_array::arrays::VarBinView::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView
+
+pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBinView
pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -16150,9 +16222,25 @@ pub trait vortex_array::scalar_fn::fns::cast::CastKernel: vortex_array::VTable
pub fn vortex_array::scalar_fn::fns::cast::CastKernel::cast(array: vortex_array::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Bool
+
+pub fn vortex_array::arrays::Bool::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Decimal
-pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::FixedSizeList
+
+pub fn vortex_array::arrays::FixedSizeList::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::List
+
+pub fn vortex_array::arrays::List::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::ListView
+
+pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Primitive
@@ -16160,7 +16248,15 @@ pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_,
impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::Struct
-pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBin
+
+pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastKernel for vortex_array::arrays::VarBinView
+
+pub fn vortex_array::arrays::VarBinView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, dtype: &vortex_array::dtype::DType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult>
pub trait vortex_array::scalar_fn::fns::cast::CastReduce: vortex_array::VTable
@@ -16178,6 +16274,10 @@ impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Co
pub fn vortex_array::arrays::Constant::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Constant>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Decimal
+
+pub fn vortex_array::arrays::Decimal::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Extension
pub fn vortex_array::arrays::Extension::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -16194,6 +16294,14 @@ impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Li
pub fn vortex_array::arrays::ListView::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Primitive
+
+pub fn vortex_array::arrays::Primitive::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+
+impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::Struct
+
+pub fn vortex_array::arrays::Struct::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
+
impl vortex_array::scalar_fn::fns::cast::CastReduce for vortex_array::arrays::VarBin
pub fn vortex_array::arrays::VarBin::cast(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult>
@@ -19438,7 +19546,7 @@ pub fn vortex_array::validity::Validity::and(self, rhs: vortex_array::validity::
pub fn vortex_array::validity::Validity::as_array(&self) -> core::option::Option<&vortex_array::ArrayRef>
-pub fn vortex_array::validity::Validity::cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize) -> vortex_error::VortexResult
+pub fn vortex_array::validity::Validity::cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult
pub fn vortex_array::validity::Validity::execute_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult
@@ -19446,7 +19554,7 @@ pub fn vortex_array::validity::Validity::filter(&self, mask: &vortex_mask::Mask)
pub fn vortex_array::validity::Validity::into_array(self) -> core::option::Option
-pub fn vortex_array::validity::Validity::into_non_nullable(self, len: usize) -> core::option::Option
+pub fn vortex_array::validity::Validity::into_non_nullable(self, len: usize, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option
pub fn vortex_array::validity::Validity::into_nullable(self) -> vortex_array::validity::Validity
@@ -19474,7 +19582,9 @@ pub fn vortex_array::validity::Validity::to_array(&self, len: usize) -> vortex_a
pub fn vortex_array::validity::Validity::to_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult
-pub fn vortex_array::validity::Validity::uncompressed_size(&self) -> usize
+pub fn vortex_array::validity::Validity::trivial_cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize) -> vortex_error::VortexResult>
+
+pub fn vortex_array::validity::Validity::trivial_into_non_nullable(self, len: usize) -> vortex_error::VortexResult>
pub fn vortex_array::validity::Validity::union_nullability(self, nullability: vortex_array::dtype::Nullability) -> Self
diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs
index 3b5fbea7607..6bfaa03b9db 100644
--- a/vortex-array/src/arrays/bool/compute/cast.rs
+++ b/vortex-array/src/arrays/bool/compute/cast.rs
@@ -4,24 +4,48 @@
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::Bool;
use crate::arrays::BoolArray;
use crate::arrays::bool::BoolArrayExt;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
impl CastReduce for Bool {
fn cast(array: ArrayView<'_, Bool>, dtype: &DType) -> VortexResult> {
- if !matches!(dtype, DType::Bool(_)) {
+ if !dtype.is_boolean() {
return Ok(None);
}
- let new_nullability = dtype.nullability();
- let new_validity = array
+ let Some(new_validity) = array
.validity()?
- .cast_nullability(new_nullability, array.len())?;
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
+ Ok(Some(
+ BoolArray::new(array.to_bit_buffer(), new_validity).into_array(),
+ ))
+ }
+}
+
+impl CastKernel for Bool {
+ fn cast(
+ array: ArrayView<'_, Bool>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ if !dtype.is_boolean() {
+ return Ok(None);
+ }
+
+ let new_validity =
+ array
+ .validity()?
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
Ok(Some(
BoolArray::new(array.to_bit_buffer(), new_validity).into_array(),
))
@@ -30,14 +54,23 @@ impl CastReduce for Bool {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
+ use vortex_session::VortexSession;
+ use crate::Canonical;
use crate::IntoArray;
+ use crate::VortexSessionExecute;
use crate::arrays::BoolArray;
use crate::builtins::ArrayBuiltins;
use crate::compute::conformance::cast::test_cast_conformance;
use crate::dtype::DType;
use crate::dtype::Nullability;
+ use crate::session::ArraySession;
+
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
#[test]
fn try_cast_bool_success() {
@@ -51,12 +84,16 @@ mod tests {
}
#[test]
- #[should_panic]
fn try_cast_bool_fail() {
+ // When the validity array's min stat is not cached, the reduce rule defers and the
+ // failure surfaces during execution via the kernel (cast_nullability -> compute_min).
let bool = BoolArray::from_iter(vec![Some(true), Some(false), None]);
- bool.into_array()
+ let mut ctx = SESSION.create_execution_ctx();
+ let result = bool
+ .into_array()
.cast(DType::Bool(Nullability::NonNullable))
- .unwrap();
+ .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array()));
+ assert!(result.is_err(), "Expected error, got: {result:?}");
}
#[rstest]
diff --git a/vortex-array/src/arrays/bool/vtable/kernel.rs b/vortex-array/src/arrays/bool/vtable/kernel.rs
index e8fa5f30e9c..4f1047a5132 100644
--- a/vortex-array/src/arrays/bool/vtable/kernel.rs
+++ b/vortex-array/src/arrays/bool/vtable/kernel.rs
@@ -4,9 +4,11 @@
use crate::arrays::Bool;
use crate::arrays::dict::TakeExecuteAdaptor;
use crate::kernel::ParentKernelSet;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor;
pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(Bool)),
ParentKernelSet::lift(&FillNullExecuteAdaptor(Bool)),
ParentKernelSet::lift(&TakeExecuteAdaptor(Bool)),
]);
diff --git a/vortex-array/src/arrays/constant/compute/cast.rs b/vortex-array/src/arrays/constant/compute/cast.rs
index 68c6dbeb758..2fd9b3ded32 100644
--- a/vortex-array/src/arrays/constant/compute/cast.rs
+++ b/vortex-array/src/arrays/constant/compute/cast.rs
@@ -15,7 +15,7 @@ impl CastReduce for Constant {
fn cast(array: ArrayView<'_, Constant>, dtype: &DType) -> VortexResult> {
match array.scalar().cast(dtype) {
Ok(scalar) => Ok(Some(ConstantArray::new(scalar, array.len()).into_array())),
- Err(_e) => Ok(None),
+ Err(_) => Ok(None),
}
}
}
diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs
index 75446b17e4e..34ccc27d094 100644
--- a/vortex-array/src/arrays/decimal/compute/cast.rs
+++ b/vortex-array/src/arrays/decimal/compute/cast.rs
@@ -18,12 +18,53 @@ use crate::dtype::DecimalType;
use crate::dtype::NativeDecimalType;
use crate::match_each_decimal_value_type;
use crate::scalar_fn::fns::cast::CastKernel;
+use crate::scalar_fn::fns::cast::CastReduce;
+
+impl CastReduce for Decimal {
+ fn cast(array: ArrayView<'_, Decimal>, dtype: &DType) -> VortexResult > {
+ // Only nullability changes within the same decimal dtype are reducible without execution.
+ // Precision/scale changes need the kernel.
+ let DType::Decimal(to_decimal_dtype, to_nullability) = dtype else {
+ return Ok(None);
+ };
+ let DType::Decimal(from_decimal_dtype, _) = array.dtype() else {
+ vortex_panic!(
+ "DecimalArray must have decimal dtype, got {:?}",
+ array.dtype()
+ );
+ };
+
+ if from_decimal_dtype != to_decimal_dtype {
+ return Ok(None);
+ }
+
+ let Some(new_validity) = array
+ .validity()?
+ .trivial_cast_nullability(*to_nullability, array.len())?
+ else {
+ return Ok(None);
+ };
+
+ // SAFETY: validity has the same length, only its nullability tag changes.
+ unsafe {
+ Ok(Some(
+ DecimalArray::new_unchecked_handle(
+ array.buffer_handle().clone(),
+ array.values_type(),
+ *to_decimal_dtype,
+ new_validity,
+ )
+ .into_array(),
+ ))
+ }
+ }
+}
impl CastKernel for Decimal {
fn cast(
array: ArrayView<'_, Decimal>,
dtype: &DType,
- _ctx: &mut ExecutionCtx,
+ ctx: &mut ExecutionCtx,
) -> VortexResult > {
// Early return if not casting to decimal
let DType::Decimal(to_decimal_dtype, to_nullability) = dtype else {
@@ -62,7 +103,7 @@ impl CastKernel for Decimal {
// Cast the validity to the new nullability
let new_validity = array
.validity()?
- .cast_nullability(*to_nullability, array.len())?;
+ .cast_nullability(*to_nullability, array.len(), ctx)?;
// If the target needs a wider physical type, upcast the values
let target_values_type = DecimalType::smallest_decimal_value_type(to_decimal_dtype);
diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs
index fae0c3dd866..31df664e5c7 100644
--- a/vortex-array/src/arrays/decimal/compute/rules.rs
+++ b/vortex-array/src/arrays/decimal/compute/rules.rs
@@ -16,10 +16,12 @@ use crate::arrays::slice::SliceReduceAdaptor;
use crate::match_each_decimal_value_type;
use crate::optimizer::rules::ArrayParentReduceRule;
use crate::optimizer::rules::ParentRuleSet;
+use crate::scalar_fn::fns::cast::CastReduceAdaptor;
use crate::scalar_fn::fns::mask::MaskReduceAdaptor;
pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[
ParentRuleSet::lift(&DecimalMaskedValidityRule),
+ ParentRuleSet::lift(&CastReduceAdaptor(Decimal)),
ParentRuleSet::lift(&MaskReduceAdaptor(Decimal)),
ParentRuleSet::lift(&SliceReduceAdaptor(Decimal)),
]);
diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs
index c9575c32ae2..ee80a1140b4 100644
--- a/vortex-array/src/arrays/dict/compute/cast.rs
+++ b/vortex-array/src/arrays/dict/compute/cast.rs
@@ -13,19 +13,12 @@ use crate::arrays::dict::DictArraySlotsExt;
use crate::builtins::ArrayBuiltins;
use crate::dtype::DType;
use crate::scalar_fn::fns::cast::CastReduce;
-use crate::validity::Validity;
impl CastReduce for Dict {
fn cast(array: ArrayView<'_, Dict>, dtype: &DType) -> VortexResult> {
// Can have un-reference null values making the cast of values fail without a possible mask.
// TODO(joe): optimize this, could look at accessible values and fill_null not those?
- if !dtype.is_nullable()
- && array.values().dtype().is_nullable()
- && !matches!(
- array.values().validity()?,
- Validity::NonNullable | Validity::AllValid
- )
- {
+ if !dtype.is_nullable() && !array.values().validity()?.no_nulls() {
return Ok(None);
}
// Cast the dictionary values to the target type
diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs
index af42fd5c9ea..c8e7fa17c36 100644
--- a/vortex-array/src/arrays/extension/compute/cast.rs
+++ b/vortex-array/src/arrays/extension/compute/cast.rs
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors
+use vortex_error::VortexResult;
+
use crate::ArrayRef;
use crate::IntoArray;
use crate::array::ArrayView;
@@ -12,10 +14,7 @@ use crate::dtype::DType;
use crate::scalar_fn::fns::cast::CastReduce;
impl CastReduce for Extension {
- fn cast(
- array: ArrayView<'_, Extension>,
- dtype: &DType,
- ) -> vortex_error::VortexResult > {
+ fn cast(array: ArrayView<'_, Extension>, dtype: &DType) -> VortexResult > {
if !array.dtype().eq_ignore_nullability(dtype) {
// Target is not the same extension type.
// Delegate to the storage array's cast.
@@ -45,10 +44,12 @@ impl CastReduce for Extension {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
use rstest::rstest;
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use super::*;
use crate::IntoArray;
@@ -59,8 +60,13 @@ mod tests {
use crate::dtype::DType;
use crate::dtype::Nullability;
use crate::dtype::PType;
+ use crate::executor::VortexSessionExecute;
use crate::extension::datetime::TimeUnit;
use crate::extension::datetime::Timestamp;
+ use crate::session::ArraySession;
+
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
#[test]
fn cast_same_ext_dtype() {
@@ -105,16 +111,18 @@ mod tests {
let storage = buffer![1i64].into_array();
let arr = ExtensionArray::new(original_dtype, storage);
- #[expect(deprecated)]
let result = arr
.into_array()
.cast(DType::Extension(target_dtype))
- .and_then(|a| a.to_canonical().map(|c| c.into_array()));
+ .and_then(|a| {
+ a.execute::(&mut SESSION.create_execution_ctx())
+ .map(|c| c.into_array())
+ });
assert!(result.is_err());
}
#[test]
- fn cast_timestamp_to_i64() -> vortex_error::VortexResult<()> {
+ fn cast_timestamp_to_i64() -> VortexResult<()> {
let ext_dtype = Timestamp::new_with_tz(
TimeUnit::Nanoseconds,
Some("UTC".into()),
diff --git a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs
index 791462d51c9..a54fe5c7c4d 100644
--- a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs
+++ b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs
@@ -4,6 +4,7 @@
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::FixedSizeList;
@@ -11,7 +12,20 @@ use crate::arrays::FixedSizeListArray;
use crate::arrays::fixed_size_list::FixedSizeListArrayExt;
use crate::builtins::ArrayBuiltins;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
+use crate::validity::Validity;
+
+fn build_with_validity(
+ array: ArrayView<'_, FixedSizeList>,
+ elements: ArrayRef,
+ validity: Validity,
+) -> ArrayRef {
+ // SAFETY: The only requirements for safety here are related to lengths, and no lengths have
+ // changed here. So as long as the original array is valid, this is also valid.
+ unsafe { FixedSizeListArray::new_unchecked(elements, array.list_size(), validity, array.len()) }
+ .into_array()
+}
/// Cast implementation for [`FixedSizeListArray`].
///
@@ -23,23 +37,33 @@ impl CastReduce for FixedSizeList {
return Ok(None);
};
+ let Some(validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
let elements = array.elements().cast((**target_element_type).clone())?;
+
+ Ok(Some(build_with_validity(array, elements, validity)))
+ }
+}
+
+impl CastKernel for FixedSizeList {
+ fn cast(
+ array: ArrayView<'_, FixedSizeList>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult> {
+ let Some(target_element_type) = dtype.as_fixed_size_list_element_opt() else {
+ return Ok(None);
+ };
+
let validity = array
.validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
-
- Ok(Some(
- // SAFETY: The only requirements for safety here are related to lengths, and no lengths
- // have changed here. So as long as the original array is valid, this is also valid.
- unsafe {
- FixedSizeListArray::new_unchecked(
- elements,
- array.list_size(),
- validity,
- array.len(),
- )
- }
- .into_array(),
- ))
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
+ let elements = array.elements().cast((**target_element_type).clone())?;
+
+ Ok(Some(build_with_validity(array, elements, validity)))
}
}
diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs
index c28a3a2a805..319751070de 100644
--- a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs
+++ b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs
@@ -4,8 +4,11 @@
use crate::arrays::FixedSizeList;
use crate::arrays::dict::TakeExecuteAdaptor;
use crate::kernel::ParentKernelSet;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
impl FixedSizeList {
- pub(crate) const PARENT_KERNELS: ParentKernelSet =
- ParentKernelSet::new(&[ParentKernelSet::lift(&TakeExecuteAdaptor(FixedSizeList))]);
+ pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(FixedSizeList)),
+ ParentKernelSet::lift(&TakeExecuteAdaptor(FixedSizeList)),
+ ]);
}
diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs
index c9813cfe92d..e9b29b9b145 100644
--- a/vortex-array/src/arrays/list/compute/cast.rs
+++ b/vortex-array/src/arrays/list/compute/cast.rs
@@ -4,6 +4,7 @@
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::List;
@@ -11,6 +12,7 @@ use crate::arrays::ListArray;
use crate::arrays::list::ListArrayExt;
use crate::builtins::ArrayBuiltins;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
impl CastReduce for List {
@@ -19,24 +21,56 @@ impl CastReduce for List {
return Ok(None);
};
+ let Some(validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
+
+ let new_elements = array.elements().cast((**target_element_type).clone())?;
+
+ Ok(Some(
+ unsafe { ListArray::new_unchecked(new_elements, array.offsets().clone(), validity) }
+ .into_array(),
+ ))
+ }
+}
+
+impl CastKernel for List {
+ fn cast(
+ array: ArrayView<'_, List>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult> {
+ let Some(target_element_type) = dtype.as_list_element_opt() else {
+ return Ok(None);
+ };
+
let validity = array
.validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
let new_elements = array.elements().cast((**target_element_type).clone())?;
- ListArray::try_new(new_elements, array.offsets().clone(), validity)
- .map(|a| Some(a.into_array()))
+ Ok(Some(
+ unsafe { ListArray::new_unchecked(new_elements, array.offsets().clone(), validity) }
+ .into_array(),
+ ))
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
+ use std::sync::LazyLock;
use rstest::rstest;
+ use vortex_array::session::ArraySession;
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
+ use crate::Canonical;
use crate::IntoArray;
use crate::LEGACY_SESSION;
use crate::RecursiveCanonical;
@@ -52,6 +86,9 @@ mod tests {
use crate::dtype::PType;
use crate::validity::Validity;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_cast_list_success() {
let list = ListArray::try_new(
@@ -87,11 +124,11 @@ mod tests {
let target_dtype = DType::Primitive(PType::U64, Nullability::NonNullable);
// can't cast list to u64
- let result = list.into_array().cast(target_dtype).and_then(|a| {
- #[expect(deprecated)]
- let canonical = a.to_canonical().map(|c| c.into_array());
- canonical
- });
+ let result = list
+ .into_array()
+ .cast(target_dtype)
+ .and_then(|a| a.execute::(&mut SESSION.create_execution_ctx()))
+ .map(|c| c.into_array());
assert!(result.is_err());
}
@@ -112,11 +149,11 @@ mod tests {
Nullability::NonNullable,
);
- let result = list.into_array().cast(target_dtype).and_then(|a| {
- #[expect(deprecated)]
- let canonical = a.to_canonical().map(|c| c.into_array());
- canonical
- });
+ let result = list
+ .into_array()
+ .cast(target_dtype)
+ .and_then(|a| a.execute::(&mut SESSION.create_execution_ctx()))
+ .map(|c| c.into_array());
assert!(result.is_err());
// Nulls in list element array — the inner cast error is deferred until
diff --git a/vortex-array/src/arrays/list/compute/kernels.rs b/vortex-array/src/arrays/list/compute/kernels.rs
index b4268642e82..188c83c1bf5 100644
--- a/vortex-array/src/arrays/list/compute/kernels.rs
+++ b/vortex-array/src/arrays/list/compute/kernels.rs
@@ -5,8 +5,10 @@ use crate::arrays::List;
use crate::arrays::dict::TakeExecuteAdaptor;
use crate::arrays::filter::FilterExecuteAdaptor;
use crate::kernel::ParentKernelSet;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(List)),
ParentKernelSet::lift(&FilterExecuteAdaptor(List)),
ParentKernelSet::lift(&TakeExecuteAdaptor(List)),
]);
diff --git a/vortex-array/src/arrays/listview/compute/cast.rs b/vortex-array/src/arrays/listview/compute/cast.rs
index 77cd7ab7736..473f76b12ee 100644
--- a/vortex-array/src/arrays/listview/compute/cast.rs
+++ b/vortex-array/src/arrays/listview/compute/cast.rs
@@ -4,6 +4,7 @@
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::ListView;
@@ -11,7 +12,27 @@ use crate::arrays::ListViewArray;
use crate::arrays::listview::ListViewArrayExt;
use crate::builtins::ArrayBuiltins;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
+use crate::validity::Validity;
+
+fn build_with_validity(
+ array: ArrayView<'_, ListView>,
+ new_elements: ArrayRef,
+ validity: Validity,
+) -> ArrayRef {
+ // SAFETY: Since `cast` is length-preserving, all of the invariants remain the same.
+ unsafe {
+ ListViewArray::new_unchecked(
+ new_elements,
+ array.offsets().clone(),
+ array.sizes().clone(),
+ validity,
+ )
+ .with_zero_copy_to_list(array.is_zero_copy_to_list())
+ }
+ .into_array()
+}
impl CastReduce for ListView {
fn cast(array: ArrayView<'_, ListView>, dtype: &DType) -> VortexResult> {
@@ -19,25 +40,34 @@ impl CastReduce for ListView {
let Some(target_element_type) = dtype.as_list_element_opt() else {
return Ok(None);
};
+ let Some(validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
+ return Ok(None);
+ };
// Cast the elements to the target element type.
let new_elements = array.elements().cast((**target_element_type).clone())?;
+ Ok(Some(build_with_validity(array, new_elements, validity)))
+ }
+}
+
+impl CastKernel for ListView {
+ fn cast(
+ array: ArrayView<'_, ListView>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ let Some(target_element_type) = dtype.as_list_element_opt() else {
+ return Ok(None);
+ };
+
let validity = array
.validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
-
- // SAFETY: Since `cast` is length-preserving, all of the invariants remain the same.
- Ok(Some(
- unsafe {
- ListViewArray::new_unchecked(
- new_elements,
- array.offsets().clone(),
- array.sizes().clone(),
- validity,
- )
- .with_zero_copy_to_list(array.is_zero_copy_to_list())
- }
- .into_array(),
- ))
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
+ let new_elements = array.elements().cast((**target_element_type).clone())?;
+
+ Ok(Some(build_with_validity(array, new_elements, validity)))
}
}
diff --git a/vortex-array/src/arrays/listview/vtable/kernel.rs b/vortex-array/src/arrays/listview/vtable/kernel.rs
new file mode 100644
index 00000000000..f6ceca284bf
--- /dev/null
+++ b/vortex-array/src/arrays/listview/vtable/kernel.rs
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use crate::arrays::ListView;
+use crate::kernel::ParentKernelSet;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
+
+pub(super) const PARENT_KERNELS: ParentKernelSet =
+ ParentKernelSet::new(&[ParentKernelSet::lift(&CastExecuteAdaptor(ListView))]);
diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs
index 893edb6bf71..f51fde706a4 100644
--- a/vortex-array/src/arrays/listview/vtable/mod.rs
+++ b/vortex-array/src/arrays/listview/vtable/mod.rs
@@ -29,12 +29,14 @@ use crate::arrays::listview::ListViewData;
use crate::arrays::listview::array::NUM_SLOTS;
use crate::arrays::listview::array::SLOT_NAMES;
use crate::arrays::listview::compute::rules::PARENT_RULES;
+use crate::arrays::listview::vtable::kernel::PARENT_KERNELS;
use crate::buffer::BufferHandle;
use crate::dtype::DType;
use crate::dtype::Nullability;
use crate::dtype::PType;
use crate::serde::ArrayChildren;
use crate::validity::Validity;
+mod kernel;
mod operations;
mod validity;
/// A [`ListView`]-encoded Vortex array.
@@ -214,4 +216,13 @@ impl VTable for ListView {
) -> VortexResult> {
PARENT_RULES.evaluate(array, parent, child_idx)
}
+
+ fn execute_parent(
+ array: ArrayView<'_, Self>,
+ parent: &ArrayRef,
+ child_idx: usize,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ PARENT_KERNELS.execute(array, parent, child_idx, ctx)
+ }
}
diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs
index abd58022ba6..92140bf9b8b 100644
--- a/vortex-array/src/arrays/primitive/compute/cast.rs
+++ b/vortex-array/src/arrays/primitive/compute/cast.rs
@@ -21,6 +21,37 @@ use crate::dtype::Nullability;
use crate::dtype::PType;
use crate::match_each_native_ptype;
use crate::scalar_fn::fns::cast::CastKernel;
+use crate::scalar_fn::fns::cast::CastReduce;
+
+impl CastReduce for Primitive {
+ fn cast(array: ArrayView<'_, Primitive>, dtype: &DType) -> VortexResult > {
+ // Only the same ptype is reducible without execution; type changes need the kernel
+ // to verify values fit in the target range.
+ let DType::Primitive(new_ptype, new_nullability) = dtype else {
+ return Ok(None);
+ };
+ if *new_ptype != array.ptype() {
+ return Ok(None);
+ }
+
+ let Some(new_validity) = array
+ .validity()?
+ .trivial_cast_nullability(*new_nullability, array.len())?
+ else {
+ return Ok(None);
+ };
+
+ // SAFETY: validity and data buffer still have same length.
+ Ok(Some(unsafe {
+ PrimitiveArray::new_unchecked_from_handle(
+ array.buffer_handle().clone(),
+ array.ptype(),
+ new_validity,
+ )
+ .into_array()
+ }))
+ }
+}
impl CastKernel for Primitive {
fn cast(
@@ -36,7 +67,7 @@ impl CastKernel for Primitive {
// First, check that the cast is compatible with the source array's validity
let new_validity = array
.validity()?
- .cast_nullability(new_nullability, array.len())?;
+ .cast_nullability(new_nullability, array.len(), ctx)?;
// Same ptype: zero-copy, just update validity.
if array.ptype() == new_ptype {
diff --git a/vortex-array/src/arrays/primitive/compute/rules.rs b/vortex-array/src/arrays/primitive/compute/rules.rs
index 99b0a7464d5..528e5e91520 100644
--- a/vortex-array/src/arrays/primitive/compute/rules.rs
+++ b/vortex-array/src/arrays/primitive/compute/rules.rs
@@ -12,10 +12,12 @@ use crate::arrays::PrimitiveArray;
use crate::arrays::slice::SliceReduceAdaptor;
use crate::optimizer::rules::ArrayParentReduceRule;
use crate::optimizer::rules::ParentRuleSet;
+use crate::scalar_fn::fns::cast::CastReduceAdaptor;
use crate::scalar_fn::fns::mask::MaskReduceAdaptor;
pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[
ParentRuleSet::lift(&PrimitiveMaskedValidityRule),
+ ParentRuleSet::lift(&CastReduceAdaptor(Primitive)),
ParentRuleSet::lift(&MaskReduceAdaptor(Primitive)),
ParentRuleSet::lift(&SliceReduceAdaptor(Primitive)),
]);
diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs
index 6c9a5f1edca..ddb3cbeadf2 100644
--- a/vortex-array/src/arrays/struct_/compute/cast.rs
+++ b/vortex-array/src/arrays/struct_/compute/cast.rs
@@ -22,7 +22,7 @@ impl CastKernel for Struct {
fn cast(
array: ArrayView<'_, Struct>,
dtype: &DType,
- _ctx: &mut ExecutionCtx,
+ ctx: &mut ExecutionCtx,
) -> VortexResult> {
let Some(target_sdtype) = dtype.as_struct_fields_opt() else {
return Ok(None);
@@ -73,7 +73,7 @@ impl CastKernel for Struct {
let validity = array
.validity()?
- .cast_nullability(dtype.nullability(), array.len())?;
+ .cast_nullability(dtype.nullability(), array.len(), ctx)?;
StructArray::try_new(
target_sdtype.names().clone(),
diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs
index 8a802969d39..6e95e3eaff8 100644
--- a/vortex-array/src/arrays/struct_/compute/rules.rs
+++ b/vortex-array/src/arrays/struct_/compute/rules.rs
@@ -18,43 +18,46 @@ use crate::arrays::scalar_fn::ScalarFnFactoryExt;
use crate::arrays::slice::SliceReduceAdaptor;
use crate::arrays::struct_::StructArrayExt;
use crate::builtins::ArrayBuiltins;
+use crate::dtype::DType;
use crate::optimizer::rules::ArrayParentReduceRule;
use crate::optimizer::rules::ParentRuleSet;
use crate::scalar_fn::EmptyOptions;
-use crate::scalar_fn::fns::cast::Cast;
+use crate::scalar_fn::fns::cast::CastReduce;
+use crate::scalar_fn::fns::cast::CastReduceAdaptor;
use crate::scalar_fn::fns::get_item::GetItem;
use crate::scalar_fn::fns::mask::Mask;
use crate::scalar_fn::fns::mask::MaskReduceAdaptor;
use crate::validity::Validity;
pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[
- ParentRuleSet::lift(&StructCastPushDownRule),
+ ParentRuleSet::lift(&CastReduceAdaptor(Struct)),
ParentRuleSet::lift(&StructGetItemRule),
ParentRuleSet::lift(&MaskReduceAdaptor(Struct)),
ParentRuleSet::lift(&SliceReduceAdaptor(Struct)),
ParentRuleSet::lift(&TakeReduceAdaptor(Struct)),
]);
-/// Rule to push down cast into struct fields.
+/// Push the cast into struct fields without execution.
///
-/// TODO(joe/rob): should be have this in casts.
+/// Supports schema evolution by allowing new nullable fields to be added during the cast,
+/// filled with null values. For nullability changes, only handles the cheap path
+/// (`try_cast_nullability`); when statistics computation is required to determine whether
+/// the array contains invalid values, returns `Ok(None)` so [`CastKernel`] can run instead.
///
-/// This rule supports schema evolution by allowing new nullable fields to be added
-/// at the end of the struct, filled with null values.
-#[derive(Debug)]
-struct StructCastPushDownRule;
-impl ArrayParentReduceRule for StructCastPushDownRule {
- type Parent = ExactScalarFn;
+/// [`CastKernel`]: crate::scalar_fn::fns::cast::CastKernel
+impl CastReduce for Struct {
+ fn cast(array: ArrayView<'_, Struct>, dtype: &DType) -> VortexResult> {
+ let Some(target_fields) = dtype.as_struct_fields_opt() else {
+ return Ok(None);
+ };
- fn reduce_parent(
- &self,
- array: ArrayView<'_, Struct>,
- parent: ScalarFnArrayView,
- _child_idx: usize,
- ) -> VortexResult> {
- let Some(target_fields) = parent.options.as_struct_fields_opt() else {
+ let Some(validity) = array
+ .validity()?
+ .trivial_cast_nullability(dtype.nullability(), array.len())?
+ else {
return Ok(None);
};
+
let mut new_fields = Vec::with_capacity(target_fields.nfields());
for (target_name, target_dtype) in target_fields.names().iter().zip(target_fields.fields())
@@ -78,20 +81,12 @@ impl ArrayParentReduceRule for StructCastPushDownRule {
}
}
- let validity = if parent.options.is_nullable() {
- array.validity()?.into_nullable()
- } else {
- array
- .validity()?
- .into_non_nullable(array.len())
- .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))?
- };
-
- let new_struct = unsafe {
- StructArray::new_unchecked(new_fields, target_fields.clone(), array.len(), validity)
- };
-
- Ok(Some(new_struct.into_array()))
+ Ok(Some(
+ unsafe {
+ StructArray::new_unchecked(new_fields, target_fields.clone(), array.len(), validity)
+ }
+ .into_array(),
+ ))
}
}
@@ -144,25 +139,31 @@ impl ArrayParentReduceRule for StructGetItemRule {
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use vortex_buffer::buffer;
+ use vortex_session::VortexSession;
use crate::IntoArray;
+ use crate::VortexSessionExecute;
use crate::arrays::StructArray;
use crate::arrays::VarBinViewArray;
use crate::arrays::struct_::StructArrayExt;
use crate::arrays::struct_::compute::rules::ConstantArray;
use crate::assert_arrays_eq;
use crate::builtins::ArrayBuiltins;
- #[expect(deprecated)]
- use crate::canonical::ToCanonical as _;
use crate::dtype::DType;
use crate::dtype::FieldNames;
use crate::dtype::Nullability;
use crate::dtype::PType;
use crate::dtype::StructFields;
use crate::scalar::Scalar;
+ use crate::session::ArraySession;
use crate::validity::Validity;
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
+
#[test]
fn test_struct_cast_field_reorder() {
// Source: {a, b}, Target: {c, b, a} - reordered + new null field
@@ -188,8 +189,12 @@ mod tests {
// Use `ArrayBuiltins::cast` which goes through the optimizer and applies
// `StructCastPushDownRule`.
- #[expect(deprecated)]
- let result = source.into_array().cast(target).unwrap().to_struct();
+ let result = source
+ .into_array()
+ .cast(target)
+ .unwrap()
+ .execute::(&mut SESSION.create_execution_ctx())
+ .unwrap();
assert_arrays_eq!(
result.unmasked_field_by_name("a").unwrap(),
VarBinViewArray::from_iter_nullable_str([Some("A")])
@@ -257,8 +262,12 @@ mod tests {
Nullability::NonNullable,
);
- #[expect(deprecated)]
- let result = source.into_array().cast(target).unwrap().to_struct();
+ let result = source
+ .into_array()
+ .cast(target)
+ .unwrap()
+ .execute::(&mut SESSION.create_execution_ctx())
+ .unwrap();
assert_eq!(result.unmasked_fields().len(), 2);
assert_arrays_eq!(
result.unmasked_field_by_name("a").unwrap(),
@@ -289,8 +298,12 @@ mod tests {
Nullability::NonNullable,
);
- #[expect(deprecated)]
- let result = source.into_array().cast(target).unwrap().to_struct();
+ let result = source
+ .into_array()
+ .cast(target)
+ .unwrap()
+ .execute::(&mut SESSION.create_execution_ctx())
+ .unwrap();
assert_eq!(
result.unmasked_field_by_name("val").unwrap().dtype(),
&DType::Primitive(PType::I64, Nullability::NonNullable)
diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs
index 6b02283aec8..e6403535e2d 100644
--- a/vortex-array/src/arrays/varbin/compute/cast.rs
+++ b/vortex-array/src/arrays/varbin/compute/cast.rs
@@ -4,13 +4,30 @@
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::VarBin;
use crate::arrays::VarBinArray;
use crate::arrays::varbin::VarBinArrayExt;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
+use crate::validity::Validity;
+
+fn build_with_validity(
+ array: ArrayView<'_, VarBin>,
+ new_dtype: DType,
+ new_validity: Validity,
+) -> VortexResult {
+ Ok(VarBinArray::try_new(
+ array.offsets().clone(),
+ array.bytes().clone(),
+ new_dtype,
+ new_validity,
+ )?
+ .into_array())
+}
impl CastReduce for VarBin {
fn cast(array: ArrayView<'_, VarBin>, dtype: &DType) -> VortexResult> {
@@ -18,33 +35,56 @@ impl CastReduce for VarBin {
return Ok(None);
}
+ let new_nullability = dtype.nullability();
+ let Some(new_validity) = array
+ .validity()?
+ .trivial_cast_nullability(new_nullability, array.len())?
+ else {
+ return Ok(None);
+ };
+ let new_dtype = array.dtype().with_nullability(new_nullability);
+ Ok(Some(build_with_validity(array, new_dtype, new_validity)?))
+ }
+}
+
+impl CastKernel for VarBin {
+ fn cast(
+ array: ArrayView<'_, VarBin>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
+ }
+
let new_nullability = dtype.nullability();
let new_validity = array
.validity()?
- .cast_nullability(new_nullability, array.len())?;
+ .cast_nullability(new_nullability, array.len(), ctx)?;
let new_dtype = array.dtype().with_nullability(new_nullability);
- Ok(Some(
- VarBinArray::try_new(
- array.offsets().clone(),
- array.bytes().clone(),
- new_dtype,
- new_validity,
- )?
- .into_array(),
- ))
+ Ok(Some(build_with_validity(array, new_dtype, new_validity)?))
}
}
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
+ use vortex_session::VortexSession;
+ use crate::Canonical;
use crate::IntoArray;
+ use crate::VortexSessionExecute;
use crate::arrays::VarBinArray;
use crate::builtins::ArrayBuiltins;
use crate::compute::conformance::cast::test_cast_conformance;
use crate::dtype::DType;
use crate::dtype::Nullability;
+ use crate::session::ArraySession;
+
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
#[rstest]
#[case(
@@ -71,14 +111,18 @@ mod tests {
}
#[rstest]
- #[should_panic]
#[case(DType::Utf8(Nullability::Nullable))]
- #[should_panic]
#[case(DType::Binary(Nullability::Nullable))]
fn try_cast_varbin_fail(#[case] source: DType) {
+ // Failure surfaces during execution via the kernel.
let non_nullable_source = source.as_nonnullable();
let varbin = VarBinArray::from_iter(vec![Some("a"), Some("b"), None], source);
- varbin.into_array().cast(non_nullable_source).unwrap();
+ let mut ctx = SESSION.create_execution_ctx();
+ let result = varbin
+ .into_array()
+ .cast(non_nullable_source)
+ .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array()));
+ assert!(result.is_err(), "Expected error, got: {result:?}");
}
#[rstest]
diff --git a/vortex-array/src/arrays/varbin/vtable/kernel.rs b/vortex-array/src/arrays/varbin/vtable/kernel.rs
index 8cb6a2ca377..9258e677933 100644
--- a/vortex-array/src/arrays/varbin/vtable/kernel.rs
+++ b/vortex-array/src/arrays/varbin/vtable/kernel.rs
@@ -6,8 +6,10 @@ use crate::arrays::dict::TakeExecuteAdaptor;
use crate::arrays::filter::FilterExecuteAdaptor;
use crate::kernel::ParentKernelSet;
use crate::scalar_fn::fns::binary::CompareExecuteAdaptor;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(VarBin)),
ParentKernelSet::lift(&CompareExecuteAdaptor(VarBin)),
ParentKernelSet::lift(&FilterExecuteAdaptor(VarBin)),
ParentKernelSet::lift(&TakeExecuteAdaptor(VarBin)),
diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs
index a8e649f52e4..9b3d6b45dc7 100644
--- a/vortex-array/src/arrays/varbinview/compute/cast.rs
+++ b/vortex-array/src/arrays/varbinview/compute/cast.rs
@@ -6,12 +6,32 @@ use std::sync::Arc;
use vortex_error::VortexResult;
use crate::ArrayRef;
+use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::VarBinView;
use crate::arrays::VarBinViewArray;
use crate::dtype::DType;
+use crate::scalar_fn::fns::cast::CastKernel;
use crate::scalar_fn::fns::cast::CastReduce;
+use crate::validity::Validity;
+
+fn build_with_validity(
+ array: ArrayView<'_, VarBinView>,
+ new_dtype: DType,
+ new_validity: Validity,
+) -> ArrayRef {
+ // SAFETY: casting just changes the DType, does not affect invariants on views/buffers.
+ unsafe {
+ VarBinViewArray::new_handle_unchecked(
+ array.views_handle().clone(),
+ Arc::clone(array.data_buffers()),
+ new_dtype,
+ new_validity,
+ )
+ .into_array()
+ }
+}
impl CastReduce for VarBinView {
fn cast(array: ArrayView<'_, VarBinView>, dtype: &DType) -> VortexResult> {
@@ -20,36 +40,55 @@ impl CastReduce for VarBinView {
}
let new_nullability = dtype.nullability();
- let new_validity = array
+ let Some(new_validity) = array
.validity()?
- .cast_nullability(new_nullability, array.len())?;
+ .trivial_cast_nullability(new_nullability, array.len())?
+ else {
+ return Ok(None);
+ };
let new_dtype = array.dtype().with_nullability(new_nullability);
+ Ok(Some(build_with_validity(array, new_dtype, new_validity)))
+ }
+}
- // SAFETY: casting just changes the DType, does not affect invariants on views/buffers.
- unsafe {
- Ok(Some(
- VarBinViewArray::new_handle_unchecked(
- array.views_handle().clone(),
- Arc::clone(array.data_buffers()),
- new_dtype,
- new_validity,
- )
- .into_array(),
- ))
+impl CastKernel for VarBinView {
+ fn cast(
+ array: ArrayView<'_, VarBinView>,
+ dtype: &DType,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult > {
+ if !array.dtype().eq_ignore_nullability(dtype) {
+ return Ok(None);
}
+
+ let new_nullability = dtype.nullability();
+ let new_validity = array
+ .validity()?
+ .cast_nullability(new_nullability, array.len(), ctx)?;
+ let new_dtype = array.dtype().with_nullability(new_nullability);
+ Ok(Some(build_with_validity(array, new_dtype, new_validity)))
}
}
#[cfg(test)]
mod tests {
+ use std::sync::LazyLock;
+
use rstest::rstest;
+ use vortex_session::VortexSession;
+ use crate::Canonical;
use crate::IntoArray;
+ use crate::VortexSessionExecute;
use crate::arrays::VarBinViewArray;
use crate::builtins::ArrayBuiltins;
use crate::compute::conformance::cast::test_cast_conformance;
use crate::dtype::DType;
use crate::dtype::Nullability;
+ use crate::session::ArraySession;
+
+ static SESSION: LazyLock =
+ LazyLock::new(|| VortexSession::empty().with::());
#[rstest]
#[case(
@@ -76,14 +115,18 @@ mod tests {
}
#[rstest]
- #[should_panic]
#[case(DType::Utf8(Nullability::Nullable))]
- #[should_panic]
#[case(DType::Binary(Nullability::Nullable))]
fn try_cast_varbin_fail(#[case] source: DType) {
+ // Failure surfaces during execution via the kernel.
let non_nullable_source = source.as_nonnullable();
let varbin = VarBinViewArray::from_iter(vec![Some("a"), Some("b"), None], source);
- varbin.into_array().cast(non_nullable_source).unwrap();
+ let mut ctx = SESSION.create_execution_ctx();
+ let result = varbin
+ .into_array()
+ .cast(non_nullable_source)
+ .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array()));
+ assert!(result.is_err(), "Expected error, got: {result:?}");
}
#[rstest]
diff --git a/vortex-array/src/arrays/varbinview/vtable/kernel.rs b/vortex-array/src/arrays/varbinview/vtable/kernel.rs
index 8486f0959f7..cd9d68010af 100644
--- a/vortex-array/src/arrays/varbinview/vtable/kernel.rs
+++ b/vortex-array/src/arrays/varbinview/vtable/kernel.rs
@@ -4,9 +4,11 @@
use crate::arrays::VarBinView;
use crate::arrays::dict::TakeExecuteAdaptor;
use crate::kernel::ParentKernelSet;
+use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
use crate::scalar_fn::fns::zip::ZipExecuteAdaptor;
pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[
+ ParentKernelSet::lift(&CastExecuteAdaptor(VarBinView)),
ParentKernelSet::lift(&TakeExecuteAdaptor(VarBinView)),
ParentKernelSet::lift(&ZipExecuteAdaptor(VarBinView)),
]);
diff --git a/vortex-array/src/scalar_fn/fns/cast/mod.rs b/vortex-array/src/scalar_fn/fns/cast/mod.rs
index 4bb21961552..cbab9615e97 100644
--- a/vortex-array/src/scalar_fn/fns/cast/mod.rs
+++ b/vortex-array/src/scalar_fn/fns/cast/mod.rs
@@ -198,7 +198,13 @@ impl ScalarFnVTable for Cast {
}
/// Cast a canonical array to the target dtype by dispatching to the appropriate
-/// [`CastReduce`] or [`CastKernel`] for each canonical encoding.
+/// [`CastKernel`] for each canonical encoding.
+///
+/// Canonical encodings that can manipulate validity directly all implement [`CastKernel`] —
+/// the kernel is the execution-time complement of their [`CastReduce`] rule and can compute
+/// statistics (e.g. min of the validity array) when the reduce rule had to give up.
+/// Encodings that delegate to scalars or storage (e.g. [`Null`], [`Constant`], [`Extension`])
+/// only implement [`CastReduce`] because they never need execution-level information.
fn cast_canonical(
canonical: CanonicalView<'_>,
dtype: &DType,
@@ -206,12 +212,12 @@ fn cast_canonical(
) -> VortexResult> {
match canonical {
CanonicalView::Null(a) => ::cast(a, dtype),
- CanonicalView::Bool(a) => ::cast(a, dtype),
+ CanonicalView::Bool(a) => ::cast(a, dtype, ctx),
CanonicalView::Primitive(a) => ::cast(a, dtype, ctx),
CanonicalView::Decimal(a) => ::cast(a, dtype, ctx),
- CanonicalView::VarBinView(a) => ::cast(a, dtype),
- CanonicalView::List(a) => ::cast(a, dtype),
- CanonicalView::FixedSizeList(a) => ::cast(a, dtype),
+ CanonicalView::VarBinView(a) => ::cast(a, dtype, ctx),
+ CanonicalView::List(a) => ::cast(a, dtype, ctx),
+ CanonicalView::FixedSizeList(a) => ::cast(a, dtype, ctx),
CanonicalView::Struct(a) => ::cast(a, dtype, ctx),
CanonicalView::Extension(a) => ::cast(a, dtype),
CanonicalView::Variant(_) => {
diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs
index 0389566bbba..b081783b809 100644
--- a/vortex-array/src/validity.rs
+++ b/vortex-array/src/validity.rs
@@ -12,7 +12,6 @@ use vortex_error::VortexExpect as _;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_err;
-use vortex_error::vortex_panic;
use vortex_mask::Mask;
use vortex_mask::MaskValues;
@@ -295,12 +294,11 @@ impl Validity {
_ => {}
};
- let own_nullability = if matches!(self, Validity::NonNullable) {
- Nullability::NonNullable
- } else {
- Nullability::Nullable
- };
+ if matches!(self, Validity::NonNullable) {
+ return Ok(Self::NonNullable);
+ }
+ // From here on we know that the validity is nullable
let source = match self {
Validity::NonNullable => BoolArray::from(BitBuffer::new_set(len)),
Validity::AllValid => BoolArray::from(BitBuffer::new_set(len)),
@@ -324,13 +322,10 @@ impl Validity {
None,
)?;
- Ok(Self::from_array(
- source.patch(&patches, ctx)?.into_array(),
- own_nullability,
- ))
+ Ok(Self::Array(source.patch(&patches, ctx)?.into_array()))
}
- /// Convert into a nullable variant
+ /// Convert into a nullable variant.
#[inline]
pub fn into_nullable(self) -> Validity {
match self {
@@ -339,9 +334,13 @@ impl Validity {
}
}
- /// Convert into a non-nullable variant
+ /// Convert into a non-nullable variant, computing statistics if necessary.
+ ///
+ /// Returns `None` when the array contains invalid values (so the cast cannot be performed),
+ /// either because it is [`Validity::AllInvalid`] or because the validity array's minimum is
+ /// `false`.
#[inline]
- pub fn into_non_nullable(self, len: usize) -> Option {
+ pub fn into_non_nullable(self, len: usize, ctx: &mut ExecutionCtx) -> Option {
match self {
_ if len == 0 => Some(Validity::NonNullable),
Self::NonNullable => Some(Self::NonNullable),
@@ -350,7 +349,7 @@ impl Validity {
Self::Array(is_valid) => {
is_valid
.statistics()
- .compute_min::(&mut LEGACY_SESSION.create_execution_ctx())
+ .compute_min::(ctx)
.vortex_expect("validity array must support min")
.then(|| {
// min true => all true
@@ -360,28 +359,92 @@ impl Validity {
}
}
- /// Convert into a variant compatible with the given nullability, if possible.
+ /// Convert into a non-nullable variant without running execution.
+ ///
+ /// This is the cheap counterpart to [`Self::into_non_nullable`]: it inspects already-computed
+ /// statistics rather than triggering execution.
+ ///
+ /// Return values:
+ /// - `Ok(Some(NonNullable))` — the cast is provably safe.
+ /// - `Ok(None)` — We need to perform compute to determine whether cast is valid. Callers should fall back to [`Self::into_non_nullable`], typically by
+ /// returning `Ok(None)` from a `CastReduce` rule so the corresponding `CastKernel` runs.
+ /// - `Err(_)` — we know the cast must fail (e.g. [`Validity::AllInvalid`]).
+ #[inline]
+ pub fn trivial_into_non_nullable(self, len: usize) -> VortexResult> {
+ match self {
+ _ if len == 0 => Ok(Some(Validity::NonNullable)),
+ Self::NonNullable => Ok(Some(Self::NonNullable)),
+ Self::AllValid => Ok(Some(Self::NonNullable)),
+ Self::AllInvalid => {
+ Err(vortex_err!(InvalidArgument: "Cannot cast AllInvalid to NonNullable"))
+ }
+ Self::Array(_) => Ok(None),
+ }
+ }
+
+ /// Convert into a variant compatible with the given nullability.
+ ///
+ /// This is the execution-time half of the nullability-cast pair. It is paired with
+ /// [`Self::trivial_cast_nullability`], which is used by `CastReduce` rules. The pattern is:
+ ///
+ /// - **`CastReduce` rules** (metadata-only rewrites in the optimizer) call
+ /// [`Self::trivial_cast_nullability`]. If it returns `Ok(None)`, the rule returns `Ok(None)`
+ /// and the cast is deferred to execution.
+ /// - **`CastKernel` impls** (executed via [`ExecuteParentKernel`]) call this method, which
+ /// may run the underlying validity array to compute statistics.
+ ///
+ /// Returns `Err` when nullability cannot be cast (for example, casting to non-nullable while
+ /// invalid values are present).
+ ///
+ /// [`ExecuteParentKernel`]: crate::kernel::ExecuteParentKernel
#[inline]
- pub fn cast_nullability(self, nullability: Nullability, len: usize) -> VortexResult {
+ pub fn cast_nullability(
+ self,
+ nullability: Nullability,
+ len: usize,
+ ctx: &mut ExecutionCtx,
+ ) -> VortexResult {
match nullability {
- Nullability::NonNullable => self.into_non_nullable(len).ok_or_else(|| {
+ Nullability::NonNullable => self.into_non_nullable(len, ctx).ok_or_else(|| {
vortex_err!(InvalidArgument: "Cannot cast array with invalid values to non-nullable type.")
}),
Nullability::Nullable => Ok(self.into_nullable()),
}
}
- /// Create Validity from boolean array with given nullability of the array.
+ /// Best-effort, non-executing variant of [`Self::cast_nullability`].
///
- /// Note: You want to pass the nullability of parent array and not the nullability of the validity array itself
- /// as that is always nonnullable
- fn from_array(value: ArrayRef, nullability: Nullability) -> Self {
- if !matches!(value.dtype(), DType::Bool(Nullability::NonNullable)) {
- vortex_panic!("Expected a non-nullable boolean array")
- }
+ /// Use this from `CastReduce` rules — they run inside the optimizer where execution is not
+ /// available. The pairing with [`Self::cast_nullability`] is symmetric: every encoding that
+ /// implements `CastReduce` and inspects validity should also implement `CastKernel` so that
+ /// the harder cases (where statistics are not yet cached) can still be handled at execution
+ /// time.
+ ///
+ /// Return values:
+ /// - `Ok(Some(_))` — the cast is provably safe and the new [`Validity`] is returned.
+ /// - `Ok(None)` — the cast cannot be reduced cheaply (the `CastKernel` should be tried via
+ /// [`Self::cast_nullability`]).
+ /// - `Err(_)` — the cast is provably impossible.
+ ///
+ /// Typical usage inside a `CastReduce`:
+ ///
+ /// ```ignore
+ /// let Some(new_validity) = array
+ /// .validity()?
+ /// .trivial_cast_nullability(dtype.nullability(), array.len())?
+ /// else {
+ /// return Ok(None);
+ /// };
+ /// ```
+ #[inline]
+ pub fn trivial_cast_nullability(
+ self,
+ nullability: Nullability,
+ len: usize,
+ ) -> VortexResult> {
match nullability {
- Nullability::NonNullable => Self::NonNullable,
- Nullability::Nullable => Self::Array(value),
+ Nullability::NonNullable => self.trivial_into_non_nullable(len),
+ Nullability::Nullable => Ok(Some(self.into_nullable())),
}
}
@@ -393,15 +456,6 @@ impl Validity {
Self::Array(a) => Some(a.len()),
}
}
-
- #[inline]
- pub fn uncompressed_size(&self) -> usize {
- if let Validity::Array(a) = self {
- a.len().div_ceil(8)
- } else {
- 0
- }
- }
}
impl From for Validity {
diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs
index 7dc1fccad80..dfd61deb80b 100644
--- a/vortex-btrblocks/src/schemes/integer.rs
+++ b/vortex-btrblocks/src/schemes/integer.rs
@@ -670,14 +670,8 @@ impl Scheme for RunEndScheme {
// SAFETY: compression doesn't affect invariants.
Ok(unsafe {
- RunEnd::new_unchecked(
- compressed_ends,
- compressed_values,
- 0,
- data.array_len(),
- exec_ctx,
- )
- .into_array()
+ RunEnd::new_unchecked(compressed_ends, compressed_values, 0, data.array_len())
+ .into_array()
})
}
}
diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs
index 485fff7ee30..16360ff287d 100644
--- a/vortex-layout/src/layouts/zoned/zone_map.rs
+++ b/vortex-layout/src/layouts/zoned/zone_map.rs
@@ -6,7 +6,6 @@
use std::sync::Arc;
use vortex_array::ArrayRef;
-use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::ConstantArray;
@@ -103,7 +102,7 @@ impl ZoneMap {
return applied.execute::(&mut ctx);
}
- let row_count_array = row_count_array(self.zone_len, self.row_count, num_zones, &mut ctx)?;
+ let row_count_array = row_count_array(self.zone_len, self.row_count, num_zones)?;
let substituted = substitute_row_count(applied, &row_count_array)?;
substituted.execute::(&mut ctx)
}
@@ -114,12 +113,7 @@ impl ZoneMap {
/// `zone_len` is the nominal zone size; only the final zone may be shorter. The
/// result is a [`ConstantArray`] for uniform zone sizes, otherwise a two-run
/// run-end encoded array whose trailing run carries the final zone length.
-fn row_count_array(
- zone_len: u64,
- row_count: u64,
- num_zones: usize,
- ctx: &mut ExecutionCtx,
-) -> VortexResult {
+fn row_count_array(zone_len: u64, row_count: u64, num_zones: usize) -> VortexResult {
let last_zone_len = row_count - zone_len.saturating_mul((num_zones as u64) - 1);
if num_zones == 1 || last_zone_len == zone_len {
return Ok(ConstantArray::new(last_zone_len, num_zones).into_array());
@@ -139,7 +133,7 @@ fn row_count_array(
// SAFETY: `ends` are strictly increasing, terminate at `num_zones`, and align one-to-one
// with the non-null run values.
- Ok(unsafe { RunEnd::new_unchecked(ends, values, 0, num_zones, ctx) }.into_array())
+ Ok(unsafe { RunEnd::new_unchecked(ends, values, 0, num_zones) }.into_array())
}
#[cfg(test)]