pheon/tests/selective_reads.rs
Jan Petykiewicz ae33c4948b fix(parser): calculate array reference pitches without overflow
Use i64 coordinate differences and checked i32 pitches in both scans and full reads. Reject nonpositive counts, malformed coordinate records, and pitches that cannot fit the packed schema.
2026-11-14 14:39:01 -08:00

403 lines
12 KiB
Rust

mod common;
use arrow::array::{
Array, BooleanArray, Float64Array, ListArray, StringArray, StructArray, UInt32Array,
UInt64Array,
};
use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi};
use common::{library_header, record};
use klamath_rs_ext::{
basic::encode_real8,
last_error_message,
library::{read_library, read_selected_library, scan_library},
read_cells_bytes, records, scan_bytes,
};
use std::{ffi::CStr, ptr};
fn xy(values: &[i32]) -> Vec<u8> {
values.iter().flat_map(|v| v.to_be_bytes()).collect()
}
fn fixture() -> Vec<u8> {
let mut bytes = library_header();
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"top\0");
record(&mut bytes, records::RTAG_SREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
record(&mut bytes, records::RTAG_XY, &xy(&[3, -4]));
record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes());
record(&mut bytes, records::RTAG_PROPVALUE, b"value\0");
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_AREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]);
record(
&mut bytes,
records::RTAG_MAG,
&encode_real8(2.0).unwrap().to_be_bytes(),
);
record(
&mut bytes,
records::RTAG_ANGLE,
&encode_real8(90.0).unwrap().to_be_bytes(),
);
record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]);
record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23]));
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"leaf");
record(&mut bytes, records::RTAG_BOUNDARY, &[]);
record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes());
record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes());
record(
&mut bytes,
records::RTAG_XY,
&xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]),
);
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_ENDLIB, &[]);
bytes
}
fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray {
let list = array
.column_by_name(name)
.unwrap()
.as_any()
.downcast_ref::<ListArray>()
.unwrap();
list.value(row)
.as_any()
.downcast_ref::<StructArray>()
.unwrap()
.clone()
}
fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec<u8> {
let mut bytes = library_header();
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"top\0");
record(&mut bytes, records::RTAG_AREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
let counts: Vec<u8> = counts.into_iter().flat_map(i16::to_be_bytes).collect();
record(&mut bytes, records::RTAG_COLROW, &counts);
record(
&mut bytes,
records::RTAG_XY,
&xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]),
);
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_ENDLIB, &[]);
bytes
}
#[test]
fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() {
let bytes = aref_fixture(
[i32::MIN, i32::MAX],
[i32::MAX - 1, i32::MIN + 1],
[i32::MIN, i32::MIN + 1],
[2, 2],
);
let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0);
let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0);
for refs in [
list_struct(&full_cells, "arefs", 0),
list_struct(&scan_cells, "refs", 0),
] {
let column = u64_value(&refs, "xy0", 0);
let row = u64_value(&refs, "xy1", 0);
assert_eq!((column >> 32) as i32, i32::MAX);
assert_eq!(column as i32, -i32::MAX);
assert_eq!((row >> 32) as i32, 0);
assert_eq!(row as i32, -i32::MAX);
}
}
#[test]
fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() {
for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] {
let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts);
assert!(read_library(&bytes).is_err());
assert!(scan_library(&bytes).is_err());
}
let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]);
assert!(read_library(&bytes).is_err());
assert!(scan_library(&bytes).is_err());
}
fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 {
array
.column_by_name(name)
.unwrap()
.as_any()
.downcast_ref::<UInt64Array>()
.unwrap()
.value(row)
}
fn ranges(scan: &StructArray) -> Vec<(u64, u64)> {
let cells = list_struct(scan, "cells", 0);
(0..cells.len())
.map(|i| {
(
u64_value(&cells, "struct_start_offset", i),
u64_value(&cells, "struct_end_offset", i),
)
})
.collect()
}
#[test]
fn scanned_ranges_reproduce_the_full_library_including_offsets() {
let bytes = fixture();
let (_, full) = read_library(&bytes).unwrap();
let (scan_remaining, scan) = scan_library(&bytes).unwrap();
assert!(scan_remaining.is_empty());
let ranges = ranges(&scan);
assert_eq!(ranges.len(), 2);
for &(start, end) in &ranges {
assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]);
assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]);
}
let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap();
assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged.
assert_eq!(full, selected);
let cells = list_struct(&full, "cells", 0);
assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8);
}
#[test]
fn scan_preserves_reference_transforms_and_skips_geometry() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let cells = list_struct(&scan, "cells", 0);
let refs = list_struct(&cells, "refs", 0);
assert_eq!(refs.len(), 2);
assert_eq!(list_struct(&cells, "refs", 1).len(), 0);
let target = refs
.column_by_name("target")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap();
assert_eq!(target.values().as_ref(), &[1, 1]);
assert_eq!(
u64_value(&refs, "xy", 0),
(3_u64 << 32) | (-4_i32 as u32 as u64)
);
assert_eq!(u64_value(&refs, "xy0", 0), 0);
assert_eq!(u64_value(&refs, "xy1", 0), 0);
let counts = refs
.column_by_name("counts")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap();
assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]);
assert!(
refs.column_by_name("invert_y")
.unwrap()
.as_any()
.downcast_ref::<BooleanArray>()
.unwrap()
.value(1)
);
assert_eq!(
refs.column_by_name("scale")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap()
.value(1),
2.0
);
assert_eq!(
refs.column_by_name("angle_rad")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap()
.value(1),
std::f64::consts::FRAC_PI_2
);
assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32);
assert_eq!(u64_value(&refs, "xy1", 1), 10);
}
#[test]
fn selections_support_subsets_reordering_and_empty_results() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let ranges = ranges(&scan);
let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap();
let leaf_cells = list_struct(&leaf, "cells", 0);
assert_eq!(leaf_cells.len(), 1);
let leaf_names = leaf
.column_by_name("cell_names")
.unwrap()
.as_any()
.downcast_ref::<ListArray>()
.unwrap()
.value(0);
assert_eq!(
leaf_names
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.value(0),
"leaf"
);
let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap();
let reordered_cells = list_struct(&reordered, "cells", 0);
assert_eq!(
u64_value(&reordered_cells, "file_offset", 0),
u64_value(&leaf_cells, "file_offset", 0)
);
let refs = list_struct(&reordered_cells, "sref_props", 1);
assert_eq!(
refs.column_by_name("target")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap()
.value(0),
0
);
let (_, empty) = read_selected_library(&bytes, &[]).unwrap();
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
}
#[test]
fn selected_ranges_must_cover_exactly_one_complete_cell() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let ranges = ranges(&scan);
let (start, end) = ranges[0];
for range in [
(end, start),
(start, u64::MAX),
(start, end - 1),
(start + 4, end),
(start, end + 4),
(start, ranges[1].1),
] {
assert!(
read_selected_library(&bytes, &[range]).is_err(),
"accepted {range:?}"
);
}
}
#[test]
fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() {
let bytes = fixture();
let mut array = FFI_ArrowArray::empty();
let mut schema = FFI_ArrowSchema::empty();
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
0,
&mut array,
&mut schema,
)
},
0
);
let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
let mut array = FFI_ArrowArray::empty();
assert_eq!(
unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) },
0
);
let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
let ranges: Vec<u64> = ranges(&scan)
.into_iter()
.flat_map(|(a, b)| [a, b])
.collect();
let mut array = FFI_ArrowArray::empty();
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ranges.as_ptr(),
ranges.len() / 2,
&mut array,
&mut schema,
)
},
0
);
let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
assert_eq!(selected, read_library(&bytes).unwrap().1);
assert!(last_error_message().is_null());
}
#[test]
fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() {
let bytes = fixture();
let mut array = FFI_ArrowArray::empty();
let mut schema = FFI_ArrowSchema::empty();
assert_eq!(
unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) },
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("scan failed")
);
assert_eq!(
unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) },
-1
);
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
1,
&mut array,
&mut schema,
)
},
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("ranges pointer was null")
);
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
usize::MAX,
&mut array,
&mut schema,
)
},
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("range count is too large")
);
assert!(array.is_released());
}