feat(parser): add hierarchy scans and selective cell reads
Reuse library builders and cell traversal for full reads, lightweight hierarchy scans, and exact cell byte ranges. Preserve file offsets across readers and reject malformed ranges and FFI pointer/count inputs without aborting. Document scan schemas and ID assignment. Add regression coverage for Arrow round trips, transforms, subset ordering, empty selections, and invalid input.
This commit is contained in:
parent
24438c4840
commit
9643089f2d
1 changed files with 848 additions and 56 deletions
349
tests/selective_reads.rs
Normal file
349
tests/selective_reads.rs
Normal file
|
|
@ -0,0 +1,349 @@
|
|||
mod common;
|
||||
|
||||
use arrow::array::{
|
||||
Array, BooleanArray, Float64Array, ListArray, StringArray, StructArray, UInt32Array,
|
||||
UInt64Array,
|
||||
};
|
||||
use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi};
|
||||
use common::{library_header, record};
|
||||
use klamath_rs_ext::{
|
||||
basic::encode_real8,
|
||||
last_error_message,
|
||||
library::{read_library, read_selected_library, scan_library},
|
||||
read_cells_bytes, records, scan_bytes,
|
||||
};
|
||||
use std::{ffi::CStr, ptr};
|
||||
|
||||
fn xy(values: &[i32]) -> Vec<u8> {
|
||||
values.iter().flat_map(|v| v.to_be_bytes()).collect()
|
||||
}
|
||||
|
||||
fn fixture() -> Vec<u8> {
|
||||
let mut bytes = library_header();
|
||||
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
|
||||
record(&mut bytes, records::RTAG_STRNAME, b"top\0");
|
||||
record(&mut bytes, records::RTAG_SREF, &[]);
|
||||
record(&mut bytes, records::RTAG_SNAME, b"leaf");
|
||||
record(&mut bytes, records::RTAG_XY, &xy(&[3, -4]));
|
||||
record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes());
|
||||
record(&mut bytes, records::RTAG_PROPVALUE, b"value\0");
|
||||
record(&mut bytes, records::RTAG_ENDEL, &[]);
|
||||
record(&mut bytes, records::RTAG_AREF, &[]);
|
||||
record(&mut bytes, records::RTAG_SNAME, b"leaf");
|
||||
record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]);
|
||||
record(
|
||||
&mut bytes,
|
||||
records::RTAG_MAG,
|
||||
&encode_real8(2.0).unwrap().to_be_bytes(),
|
||||
);
|
||||
record(
|
||||
&mut bytes,
|
||||
records::RTAG_ANGLE,
|
||||
&encode_real8(90.0).unwrap().to_be_bytes(),
|
||||
);
|
||||
record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]);
|
||||
record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23]));
|
||||
record(&mut bytes, records::RTAG_ENDEL, &[]);
|
||||
record(&mut bytes, records::RTAG_ENDSTR, &[]);
|
||||
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
|
||||
record(&mut bytes, records::RTAG_STRNAME, b"leaf");
|
||||
record(&mut bytes, records::RTAG_BOUNDARY, &[]);
|
||||
record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes());
|
||||
record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes());
|
||||
record(
|
||||
&mut bytes,
|
||||
records::RTAG_XY,
|
||||
&xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]),
|
||||
);
|
||||
record(&mut bytes, records::RTAG_ENDEL, &[]);
|
||||
record(&mut bytes, records::RTAG_ENDSTR, &[]);
|
||||
record(&mut bytes, records::RTAG_ENDLIB, &[]);
|
||||
bytes
|
||||
}
|
||||
|
||||
fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray {
|
||||
let list = array
|
||||
.column_by_name(name)
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<ListArray>()
|
||||
.unwrap();
|
||||
list.value(row)
|
||||
.as_any()
|
||||
.downcast_ref::<StructArray>()
|
||||
.unwrap()
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 {
|
||||
array
|
||||
.column_by_name(name)
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt64Array>()
|
||||
.unwrap()
|
||||
.value(row)
|
||||
}
|
||||
|
||||
fn ranges(scan: &StructArray) -> Vec<(u64, u64)> {
|
||||
let cells = list_struct(scan, "cells", 0);
|
||||
(0..cells.len())
|
||||
.map(|i| {
|
||||
(
|
||||
u64_value(&cells, "struct_start_offset", i),
|
||||
u64_value(&cells, "struct_end_offset", i),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanned_ranges_reproduce_the_full_library_including_offsets() {
|
||||
let bytes = fixture();
|
||||
let (_, full) = read_library(&bytes).unwrap();
|
||||
let (scan_remaining, scan) = scan_library(&bytes).unwrap();
|
||||
assert!(scan_remaining.is_empty());
|
||||
let ranges = ranges(&scan);
|
||||
assert_eq!(ranges.len(), 2);
|
||||
for &(start, end) in &ranges {
|
||||
assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]);
|
||||
assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]);
|
||||
}
|
||||
let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap();
|
||||
assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged.
|
||||
assert_eq!(full, selected);
|
||||
let cells = list_struct(&full, "cells", 0);
|
||||
assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_preserves_reference_transforms_and_skips_geometry() {
|
||||
let bytes = fixture();
|
||||
let (_, scan) = scan_library(&bytes).unwrap();
|
||||
let cells = list_struct(&scan, "cells", 0);
|
||||
let refs = list_struct(&cells, "refs", 0);
|
||||
assert_eq!(refs.len(), 2);
|
||||
assert_eq!(list_struct(&cells, "refs", 1).len(), 0);
|
||||
let target = refs
|
||||
.column_by_name("target")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt32Array>()
|
||||
.unwrap();
|
||||
assert_eq!(target.values().as_ref(), &[1, 1]);
|
||||
assert_eq!(
|
||||
u64_value(&refs, "xy", 0),
|
||||
(3_u64 << 32) | (-4_i32 as u32 as u64)
|
||||
);
|
||||
assert_eq!(u64_value(&refs, "xy0", 0), 0);
|
||||
assert_eq!(u64_value(&refs, "xy1", 0), 0);
|
||||
let counts = refs
|
||||
.column_by_name("counts")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt32Array>()
|
||||
.unwrap();
|
||||
assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]);
|
||||
assert!(
|
||||
refs.column_by_name("invert_y")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<BooleanArray>()
|
||||
.unwrap()
|
||||
.value(1)
|
||||
);
|
||||
assert_eq!(
|
||||
refs.column_by_name("scale")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap()
|
||||
.value(1),
|
||||
2.0
|
||||
);
|
||||
assert_eq!(
|
||||
refs.column_by_name("angle_rad")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap()
|
||||
.value(1),
|
||||
std::f64::consts::FRAC_PI_2
|
||||
);
|
||||
assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32);
|
||||
assert_eq!(u64_value(&refs, "xy1", 1), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selections_support_subsets_reordering_and_empty_results() {
|
||||
let bytes = fixture();
|
||||
let (_, scan) = scan_library(&bytes).unwrap();
|
||||
let ranges = ranges(&scan);
|
||||
let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap();
|
||||
let leaf_cells = list_struct(&leaf, "cells", 0);
|
||||
assert_eq!(leaf_cells.len(), 1);
|
||||
let leaf_names = leaf
|
||||
.column_by_name("cell_names")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<ListArray>()
|
||||
.unwrap()
|
||||
.value(0);
|
||||
assert_eq!(
|
||||
leaf_names
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap()
|
||||
.value(0),
|
||||
"leaf"
|
||||
);
|
||||
let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap();
|
||||
let reordered_cells = list_struct(&reordered, "cells", 0);
|
||||
assert_eq!(
|
||||
u64_value(&reordered_cells, "file_offset", 0),
|
||||
u64_value(&leaf_cells, "file_offset", 0)
|
||||
);
|
||||
let refs = list_struct(&reordered_cells, "sref_props", 1);
|
||||
assert_eq!(
|
||||
refs.column_by_name("target")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt32Array>()
|
||||
.unwrap()
|
||||
.value(0),
|
||||
0
|
||||
);
|
||||
let (_, empty) = read_selected_library(&bytes, &[]).unwrap();
|
||||
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_ranges_must_cover_exactly_one_complete_cell() {
|
||||
let bytes = fixture();
|
||||
let (_, scan) = scan_library(&bytes).unwrap();
|
||||
let ranges = ranges(&scan);
|
||||
let (start, end) = ranges[0];
|
||||
for range in [
|
||||
(end, start),
|
||||
(start, u64::MAX),
|
||||
(start, end - 1),
|
||||
(start + 4, end),
|
||||
(start, end + 4),
|
||||
(start, ranges[1].1),
|
||||
] {
|
||||
assert!(
|
||||
read_selected_library(&bytes, &[range]).is_err(),
|
||||
"accepted {range:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() {
|
||||
let bytes = fixture();
|
||||
let mut array = FFI_ArrowArray::empty();
|
||||
let mut schema = FFI_ArrowSchema::empty();
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
read_cells_bytes(
|
||||
bytes.as_ptr(),
|
||||
bytes.len(),
|
||||
ptr::null(),
|
||||
0,
|
||||
&mut array,
|
||||
&mut schema,
|
||||
)
|
||||
},
|
||||
0
|
||||
);
|
||||
let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
|
||||
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
|
||||
|
||||
let mut array = FFI_ArrowArray::empty();
|
||||
assert_eq!(
|
||||
unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) },
|
||||
0
|
||||
);
|
||||
let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
|
||||
let ranges: Vec<u64> = ranges(&scan)
|
||||
.into_iter()
|
||||
.flat_map(|(a, b)| [a, b])
|
||||
.collect();
|
||||
let mut array = FFI_ArrowArray::empty();
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
read_cells_bytes(
|
||||
bytes.as_ptr(),
|
||||
bytes.len(),
|
||||
ranges.as_ptr(),
|
||||
ranges.len() / 2,
|
||||
&mut array,
|
||||
&mut schema,
|
||||
)
|
||||
},
|
||||
0
|
||||
);
|
||||
let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
|
||||
assert_eq!(selected, read_library(&bytes).unwrap().1);
|
||||
assert!(last_error_message().is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() {
|
||||
let bytes = fixture();
|
||||
let mut array = FFI_ArrowArray::empty();
|
||||
let mut schema = FFI_ArrowSchema::empty();
|
||||
assert_eq!(
|
||||
unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) },
|
||||
-1
|
||||
);
|
||||
assert!(
|
||||
unsafe { CStr::from_ptr(last_error_message()) }
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains("scan failed")
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) },
|
||||
-1
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
read_cells_bytes(
|
||||
bytes.as_ptr(),
|
||||
bytes.len(),
|
||||
ptr::null(),
|
||||
1,
|
||||
&mut array,
|
||||
&mut schema,
|
||||
)
|
||||
},
|
||||
-1
|
||||
);
|
||||
assert!(
|
||||
unsafe { CStr::from_ptr(last_error_message()) }
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains("ranges pointer was null")
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
read_cells_bytes(
|
||||
bytes.as_ptr(),
|
||||
bytes.len(),
|
||||
ptr::null(),
|
||||
usize::MAX,
|
||||
&mut array,
|
||||
&mut schema,
|
||||
)
|
||||
},
|
||||
-1
|
||||
);
|
||||
assert!(
|
||||
unsafe { CStr::from_ptr(last_error_message()) }
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains("range count is too large")
|
||||
);
|
||||
assert!(array.is_released());
|
||||
}
|
||||
Loading…
Reference in a new issue