pheon/tests/selective_reads.rs

403 lines
12 KiB
Rust
Raw Normal View History

mod common;
use arrow::array::{
Array, BooleanArray, Float64Array, ListArray, StringArray, StructArray, UInt32Array,
UInt64Array,
};
use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi};
use common::{library_header, record};
use klamath_rs_ext::{
basic::encode_real8,
last_error_message,
library::{read_library, read_selected_library, scan_library},
read_cells_bytes, records, scan_bytes,
};
use std::{ffi::CStr, ptr};
fn xy(values: &[i32]) -> Vec<u8> {
values.iter().flat_map(|v| v.to_be_bytes()).collect()
}
fn fixture() -> Vec<u8> {
let mut bytes = library_header();
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"top\0");
record(&mut bytes, records::RTAG_SREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
record(&mut bytes, records::RTAG_XY, &xy(&[3, -4]));
record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes());
record(&mut bytes, records::RTAG_PROPVALUE, b"value\0");
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_AREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]);
record(
&mut bytes,
records::RTAG_MAG,
&encode_real8(2.0).unwrap().to_be_bytes(),
);
record(
&mut bytes,
records::RTAG_ANGLE,
&encode_real8(90.0).unwrap().to_be_bytes(),
);
record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]);
record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23]));
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"leaf");
record(&mut bytes, records::RTAG_BOUNDARY, &[]);
record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes());
record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes());
record(
&mut bytes,
records::RTAG_XY,
&xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]),
);
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_ENDLIB, &[]);
bytes
}
fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray {
let list = array
.column_by_name(name)
.unwrap()
.as_any()
.downcast_ref::<ListArray>()
.unwrap();
list.value(row)
.as_any()
.downcast_ref::<StructArray>()
.unwrap()
.clone()
}
fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec<u8> {
let mut bytes = library_header();
record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
record(&mut bytes, records::RTAG_STRNAME, b"top\0");
record(&mut bytes, records::RTAG_AREF, &[]);
record(&mut bytes, records::RTAG_SNAME, b"leaf");
let counts: Vec<u8> = counts.into_iter().flat_map(i16::to_be_bytes).collect();
record(&mut bytes, records::RTAG_COLROW, &counts);
record(
&mut bytes,
records::RTAG_XY,
&xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]),
);
record(&mut bytes, records::RTAG_ENDEL, &[]);
record(&mut bytes, records::RTAG_ENDSTR, &[]);
record(&mut bytes, records::RTAG_ENDLIB, &[]);
bytes
}
#[test]
fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() {
let bytes = aref_fixture(
[i32::MIN, i32::MAX],
[i32::MAX - 1, i32::MIN + 1],
[i32::MIN, i32::MIN + 1],
[2, 2],
);
let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0);
let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0);
for refs in [
list_struct(&full_cells, "arefs", 0),
list_struct(&scan_cells, "refs", 0),
] {
let column = u64_value(&refs, "xy0", 0);
let row = u64_value(&refs, "xy1", 0);
assert_eq!((column >> 32) as i32, i32::MAX);
assert_eq!(column as i32, -i32::MAX);
assert_eq!((row >> 32) as i32, 0);
assert_eq!(row as i32, -i32::MAX);
}
}
#[test]
fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() {
for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] {
let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts);
assert!(read_library(&bytes).is_err());
assert!(scan_library(&bytes).is_err());
}
let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]);
assert!(read_library(&bytes).is_err());
assert!(scan_library(&bytes).is_err());
}
fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 {
array
.column_by_name(name)
.unwrap()
.as_any()
.downcast_ref::<UInt64Array>()
.unwrap()
.value(row)
}
fn ranges(scan: &StructArray) -> Vec<(u64, u64)> {
let cells = list_struct(scan, "cells", 0);
(0..cells.len())
.map(|i| {
(
u64_value(&cells, "struct_start_offset", i),
u64_value(&cells, "struct_end_offset", i),
)
})
.collect()
}
#[test]
fn scanned_ranges_reproduce_the_full_library_including_offsets() {
let bytes = fixture();
let (_, full) = read_library(&bytes).unwrap();
let (scan_remaining, scan) = scan_library(&bytes).unwrap();
assert!(scan_remaining.is_empty());
let ranges = ranges(&scan);
assert_eq!(ranges.len(), 2);
for &(start, end) in &ranges {
assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]);
assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]);
}
let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap();
assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged.
assert_eq!(full, selected);
let cells = list_struct(&full, "cells", 0);
assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8);
}
#[test]
fn scan_preserves_reference_transforms_and_skips_geometry() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let cells = list_struct(&scan, "cells", 0);
let refs = list_struct(&cells, "refs", 0);
assert_eq!(refs.len(), 2);
assert_eq!(list_struct(&cells, "refs", 1).len(), 0);
let target = refs
.column_by_name("target")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap();
assert_eq!(target.values().as_ref(), &[1, 1]);
assert_eq!(
u64_value(&refs, "xy", 0),
(3_u64 << 32) | (-4_i32 as u32 as u64)
);
assert_eq!(u64_value(&refs, "xy0", 0), 0);
assert_eq!(u64_value(&refs, "xy1", 0), 0);
let counts = refs
.column_by_name("counts")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap();
assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]);
assert!(
refs.column_by_name("invert_y")
.unwrap()
.as_any()
.downcast_ref::<BooleanArray>()
.unwrap()
.value(1)
);
assert_eq!(
refs.column_by_name("scale")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap()
.value(1),
2.0
);
assert_eq!(
refs.column_by_name("angle_rad")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap()
.value(1),
std::f64::consts::FRAC_PI_2
);
assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32);
assert_eq!(u64_value(&refs, "xy1", 1), 10);
}
#[test]
fn selections_support_subsets_reordering_and_empty_results() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let ranges = ranges(&scan);
let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap();
let leaf_cells = list_struct(&leaf, "cells", 0);
assert_eq!(leaf_cells.len(), 1);
let leaf_names = leaf
.column_by_name("cell_names")
.unwrap()
.as_any()
.downcast_ref::<ListArray>()
.unwrap()
.value(0);
assert_eq!(
leaf_names
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.value(0),
"leaf"
);
let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap();
let reordered_cells = list_struct(&reordered, "cells", 0);
assert_eq!(
u64_value(&reordered_cells, "file_offset", 0),
u64_value(&leaf_cells, "file_offset", 0)
);
let refs = list_struct(&reordered_cells, "sref_props", 1);
assert_eq!(
refs.column_by_name("target")
.unwrap()
.as_any()
.downcast_ref::<UInt32Array>()
.unwrap()
.value(0),
0
);
let (_, empty) = read_selected_library(&bytes, &[]).unwrap();
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
}
#[test]
fn selected_ranges_must_cover_exactly_one_complete_cell() {
let bytes = fixture();
let (_, scan) = scan_library(&bytes).unwrap();
let ranges = ranges(&scan);
let (start, end) = ranges[0];
for range in [
(end, start),
(start, u64::MAX),
(start, end - 1),
(start + 4, end),
(start, end + 4),
(start, ranges[1].1),
] {
assert!(
read_selected_library(&bytes, &[range]).is_err(),
"accepted {range:?}"
);
}
}
#[test]
fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() {
let bytes = fixture();
let mut array = FFI_ArrowArray::empty();
let mut schema = FFI_ArrowSchema::empty();
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
0,
&mut array,
&mut schema,
)
},
0
);
let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
assert_eq!(list_struct(&empty, "cells", 0).len(), 0);
let mut array = FFI_ArrowArray::empty();
assert_eq!(
unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) },
0
);
let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
let ranges: Vec<u64> = ranges(&scan)
.into_iter()
.flat_map(|(a, b)| [a, b])
.collect();
let mut array = FFI_ArrowArray::empty();
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ranges.as_ptr(),
ranges.len() / 2,
&mut array,
&mut schema,
)
},
0
);
let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap());
assert_eq!(selected, read_library(&bytes).unwrap().1);
assert!(last_error_message().is_null());
}
#[test]
fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() {
let bytes = fixture();
let mut array = FFI_ArrowArray::empty();
let mut schema = FFI_ArrowSchema::empty();
assert_eq!(
unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) },
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("scan failed")
);
assert_eq!(
unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) },
-1
);
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
1,
&mut array,
&mut schema,
)
},
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("ranges pointer was null")
);
assert_eq!(
unsafe {
read_cells_bytes(
bytes.as_ptr(),
bytes.len(),
ptr::null(),
usize::MAX,
&mut array,
&mut schema,
)
},
-1
);
assert!(
unsafe { CStr::from_ptr(last_error_message()) }
.to_str()
.unwrap()
.contains("range count is too large")
);
assert!(array.is_released());
}