mod common; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; use arrow_array::{ Array, BooleanArray, Float64Array, Int16Array, Int32Array, ListArray, StringArray, StructArray, UInt32Array, UInt64Array, }; use common::{library_header, record}; use pheon::{ basic::encode_real8, last_error_message, library::{read_library, read_selected_library, scan_library}, read_cells_bytes, read_path, records, scan_bytes, }; use std::{ ffi::{CStr, CString}, ptr, }; fn xy(values: &[i32]) -> Vec { values.iter().flat_map(|v| v.to_be_bytes()).collect() } fn fixture() -> Vec { let mut bytes = library_header(); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"top\0"); record(&mut bytes, records::RTAG_SREF, &[]); record(&mut bytes, records::RTAG_SNAME, b"leaf"); record(&mut bytes, records::RTAG_XY, &xy(&[3, -4])); record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); record(&mut bytes, records::RTAG_ENDEL, &[]); record(&mut bytes, records::RTAG_AREF, &[]); record(&mut bytes, records::RTAG_SNAME, b"leaf"); record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]); record( &mut bytes, records::RTAG_MAG, &encode_real8(2.0).unwrap().to_be_bytes(), ); record( &mut bytes, records::RTAG_ANGLE, &encode_real8(90.0).unwrap().to_be_bytes(), ); record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]); record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23])); record(&mut bytes, records::RTAG_ENDEL, &[]); record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"leaf"); record(&mut bytes, records::RTAG_BOUNDARY, &[]); record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes()); record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes()); record( &mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), ); record(&mut bytes, records::RTAG_ENDEL, &[]); record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); bytes } fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray { let list = array .column_by_name(name) .unwrap() .as_any() .downcast_ref::() .unwrap(); list.value(row) .as_any() .downcast_ref::() .unwrap() .clone() } fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec { let mut bytes = library_header(); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"top\0"); record(&mut bytes, records::RTAG_AREF, &[]); record(&mut bytes, records::RTAG_SNAME, b"leaf"); let counts: Vec = counts.into_iter().flat_map(i16::to_be_bytes).collect(); record(&mut bytes, records::RTAG_COLROW, &counts); record( &mut bytes, records::RTAG_XY, &xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]), ); record(&mut bytes, records::RTAG_ENDEL, &[]); record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); bytes } #[test] fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { let bytes = aref_fixture( [i32::MIN, i32::MAX], [i32::MAX - 1, i32::MIN + 1], [i32::MIN, i32::MIN + 1], [2, 2], ); let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0); let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0); for refs in [ list_struct(&full_cells, "arefs", 0), list_struct(&scan_cells, "refs", 0), ] { let column = u64_value(&refs, "xy0", 0); let row = u64_value(&refs, "xy1", 0); assert_eq!((column >> 32) as i32, i32::MAX); assert_eq!(column as i32, -i32::MAX); assert_eq!((row >> 32) as i32, 0); assert_eq!(row as i32, -i32::MAX); } } #[test] fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); assert!(read_library(&bytes).is_err()); assert!(scan_library(&bytes).is_err()); } let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); assert!(read_library(&bytes).is_err()); assert!(scan_library(&bytes).is_err()); } #[test] fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { for (column, row, counts) in [ ([10, 0], [0, 20], [3, 2]), ([-10, 0], [0, 20], [3, 2]), ([0, 10], [20, 0], [3, 2]), ([20, 0], [0, 10], [2, 3]), ([20, 0], [-10, 0], [2, 3]), ] { let bytes = aref_fixture([0, 0], column, row, counts); let range = [(library_header().len() as u64, bytes.len() as u64 - 4)]; for error in [ read_library(&bytes).unwrap_err().1, scan_library(&bytes).unwrap_err().1, read_selected_library(&bytes, &range).unwrap_err().1, ] { assert!(format!("{error:?}").contains("not an integer database-unit increment")); } } let bytes = aref_fixture([0, 0], [10, 0], [0, 20], [3, 2]); let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!( unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, -1 ); let range = [library_header().len() as u64, bytes.len() as u64 - 4]; assert_eq!( unsafe { read_cells_bytes( bytes.as_ptr(), bytes.len(), range.as_ptr(), 1, &mut array, &mut schema, ) }, -1 ); let path = std::env::temp_dir().join(format!("pheon-fractional-aref-{}.gds", std::process::id())); struct Cleanup(std::path::PathBuf); impl Drop for Cleanup { fn drop(&mut self) { let _ = std::fs::remove_file(&self.0); } } let _cleanup = Cleanup(path.clone()); std::fs::write(&path, &bytes).unwrap(); let path = CString::new(path.to_str().unwrap()).unwrap(); assert_eq!( unsafe { read_path(path.as_ptr(), &mut array, &mut schema) }, -1 ); let message = unsafe { CStr::from_ptr(last_error_message()) } .to_str() .unwrap(); assert!(message.contains("not an integer database-unit increment")); assert!(array.is_released()); // The object reviewer keeps the exact endpoints, including fractional pitches. let mut bytes = bytes; bytes.truncate(bytes.len() - 4); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"leaf"); record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); let doc = pheon::review::Document::parse(&bytes).unwrap(); assert_eq!( pheon::review::Comparison::between(&doc, &doc).exit_code(), 0 ); } #[test] fn rectangle_detection_handles_extreme_coordinates_and_preserves_other_polygons() { let mut bytes = library_header(); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"top\0"); let wide = [ i32::MIN, i32::MIN, i32::MAX, i32::MIN, i32::MAX, i32::MAX, i32::MIN, i32::MAX, i32::MIN, i32::MIN, ]; let reversed = [ i32::MIN, i32::MIN, i32::MIN, i32::MAX, i32::MAX, i32::MAX, i32::MAX, i32::MIN, i32::MIN, i32::MIN, ]; let diagonal = [0, 0, 10, 10, 0, 10, 10, 0, 0, 0]; let degenerate = [0, 0, 0, 0, 10, 0, 10, 0, 0, 0]; for points in [&wide, &reversed, &diagonal, °enerate] { record(&mut bytes, records::RTAG_BOUNDARY, &[]); record(&mut bytes, records::RTAG_LAYER, &[0, 1]); record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); record(&mut bytes, records::RTAG_XY, &xy(points)); record(&mut bytes, records::RTAG_ENDEL, &[]); } record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); let full = read_library(&bytes).unwrap().1; let scan = scan_library(&bytes).unwrap().1; assert_eq!( full, read_selected_library(&bytes, &ranges(&scan)).unwrap().1 ); let cells = list_struct(&full, "cells", 0); let rects = list_struct(&cells, "rect_batches", 0); let values = rects .column_by_name("rects") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(0); let values = values.as_any().downcast_ref::().unwrap(); assert_eq!( values.values().as_ref(), &[ i32::MIN, i32::MIN, i32::MAX, i32::MAX, i32::MIN, i32::MIN, i32::MAX, i32::MAX ] ); let boundaries = list_struct(&cells, "boundary_batches", 0); let values = boundaries .column_by_name("vertices") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(0); let values = values.as_any().downcast_ref::().unwrap(); assert_eq!( values.values().as_ref(), [&diagonal[..8], °enerate[..8]].concat() ); } #[test] fn paths_preserve_missing_explicit_zero_and_negative_widths() { let mut bytes = library_header(); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"top\0"); for width in [None, Some(0_i32), Some(-100)] { record(&mut bytes, records::RTAG_PATH, &[]); record(&mut bytes, records::RTAG_LAYER, &[0, 1]); record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); if let Some(width) = width { record(&mut bytes, records::RTAG_WIDTH, &width.to_be_bytes()); } record(&mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 10])); record(&mut bytes, records::RTAG_ENDEL, &[]); } record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); let (_, full) = read_library(&bytes).unwrap(); let (_, scan) = scan_library(&bytes).unwrap(); let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); assert_eq!(full, selected); let cells = list_struct(&full, "cells", 0); let paths = list_struct(&cells, "paths", 0); let widths = paths .column_by_name("width") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert_eq!( widths.iter().collect::>(), [None, Some(0), Some(-100)] ); } fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 { array .column_by_name(name) .unwrap() .as_any() .downcast_ref::() .unwrap() .value(row) } fn ranges(scan: &StructArray) -> Vec<(u64, u64)> { let cells = list_struct(scan, "cells", 0); (0..cells.len()) .map(|i| { ( u64_value(&cells, "struct_start_offset", i), u64_value(&cells, "struct_end_offset", i), ) }) .collect() } #[test] fn scanned_ranges_reproduce_the_full_library_including_offsets() { let bytes = fixture(); let (_, full) = read_library(&bytes).unwrap(); let (scan_remaining, scan) = scan_library(&bytes).unwrap(); assert!(scan_remaining.is_empty()); let ranges = ranges(&scan); assert_eq!(ranges.len(), 2); for &(start, end) in &ranges { assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]); assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]); } let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap(); assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged. assert_eq!(full, selected); let cells = list_struct(&full, "cells", 0); assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8); } #[test] fn scan_preserves_reference_transforms_and_skips_geometry() { let bytes = fixture(); let (_, scan) = scan_library(&bytes).unwrap(); let cells = list_struct(&scan, "cells", 0); let refs = list_struct(&cells, "refs", 0); assert_eq!(refs.len(), 2); assert_eq!(list_struct(&cells, "refs", 1).len(), 0); let target = refs .column_by_name("target") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert_eq!(target.values().as_ref(), &[1, 1]); assert_eq!( u64_value(&refs, "xy", 0), (3_u64 << 32) | (-4_i32 as u32 as u64) ); assert_eq!(u64_value(&refs, "xy0", 0), 0); assert_eq!(u64_value(&refs, "xy1", 0), 0); let counts = refs .column_by_name("counts") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]); assert!( refs.column_by_name("invert_y") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(1) ); assert_eq!( refs.column_by_name("scale") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(1), 2.0 ); assert_eq!( refs.column_by_name("angle_rad") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(1), std::f64::consts::FRAC_PI_2 ); assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32); assert_eq!(u64_value(&refs, "xy1", 1), 10); } #[test] fn selections_support_subsets_reordering_and_empty_results() { let bytes = fixture(); let (_, scan) = scan_library(&bytes).unwrap(); let ranges = ranges(&scan); let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap(); let leaf_cells = list_struct(&leaf, "cells", 0); assert_eq!(leaf_cells.len(), 1); let leaf_names = leaf .column_by_name("cell_names") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(0); assert_eq!( leaf_names .as_any() .downcast_ref::() .unwrap() .value(0), "leaf" ); let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap(); let reordered_cells = list_struct(&reordered, "cells", 0); assert_eq!( u64_value(&reordered_cells, "file_offset", 0), u64_value(&leaf_cells, "file_offset", 0) ); let refs = list_struct(&reordered_cells, "sref_props", 1); assert_eq!( refs.column_by_name("target") .unwrap() .as_any() .downcast_ref::() .unwrap() .value(0), 0 ); let (_, empty) = read_selected_library(&bytes, &[]).unwrap(); assert_eq!(list_struct(&empty, "cells", 0).len(), 0); } #[test] fn id_tables_preserve_encounter_order_and_selection_local_ids() { let mut bytes = library_header(); for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha", 9, 4)] { record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); if name == "zeta" { record(&mut bytes, records::RTAG_SREF, &[]); record(&mut bytes, records::RTAG_SNAME, b"alpha\0"); record(&mut bytes, records::RTAG_XY, &xy(&[0, 0])); record(&mut bytes, records::RTAG_ENDEL, &[]); } record(&mut bytes, records::RTAG_BOUNDARY, &[]); record(&mut bytes, records::RTAG_LAYER, &layer.to_be_bytes()); record(&mut bytes, records::RTAG_DATATYPE, &datatype.to_be_bytes()); record( &mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), ); record(&mut bytes, records::RTAG_ENDEL, &[]); record(&mut bytes, records::RTAG_ENDSTR, &[]); } record(&mut bytes, records::RTAG_ENDLIB, &[]); let (_, scan) = scan_library(&bytes).unwrap(); let (_, full) = read_library(&bytes).unwrap(); let cell_ranges = ranges(&scan); let check = |library: &StructArray, names: &[&str], layers: &[u32], ids: &[u32]| { let values = |name| { library .column_by_name(name) .unwrap() .as_any() .downcast_ref::() .unwrap() .value(0) }; let name_values = values("cell_names"); assert_eq!( name_values .as_any() .downcast_ref::() .unwrap() .iter() .map(Option::unwrap) .collect::>(), names ); let cells = list_struct(library, "cells", 0); assert_eq!( cells .column_by_name("id") .unwrap() .as_any() .downcast_ref::() .unwrap() .values() .as_ref(), ids ); if library.column_by_name("layers").is_some() { let layer_values = values("layers"); assert_eq!( layer_values .as_any() .downcast_ref::() .unwrap() .values() .as_ref(), layers ); } }; let layer_a = (9 << 16) | 4; let layer_b = (2 << 16) | 8; check( &full, &["zeta", "alpha", "middle"], &[layer_a, layer_b], &[0, 2, 1], ); check(&scan, &["zeta", "alpha", "middle"], &[], &[0, 2, 1]); let (_, all_selected) = read_selected_library(&bytes, &cell_ranges).unwrap(); assert_eq!(all_selected, full); let (_, subset) = read_selected_library(&bytes, &cell_ranges[1..]).unwrap(); check(&subset, &["middle", "alpha"], &[layer_b, layer_a], &[0, 1]); let (_, reversed) = read_selected_library(&bytes, &[cell_ranges[2], cell_ranges[1], cell_ranges[0]]).unwrap(); check( &reversed, &["alpha", "middle", "zeta"], &[layer_a, layer_b], &[0, 1, 2], ); } #[test] fn streamed_and_buffered_properties_preserve_duplicates_order_and_empty_lists() { let mut bytes = library_header(); record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, b"top\0"); for (kind, properties) in [ (records::RTAG_BOUNDARY, true), (records::RTAG_PATH, true), (records::RTAG_PATH, false), ] { record(&mut bytes, kind, &[]); record(&mut bytes, records::RTAG_LAYER, &[0, 1]); record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); let vertices = if kind == records::RTAG_BOUNDARY { xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]) } else { xy(&[0, 0, 10, 10]) }; record(&mut bytes, records::RTAG_XY, &vertices); if properties { for (key, value) in [(7_i16, b"first\0".as_slice()), (7, b"second"), (9, b"")] { record(&mut bytes, records::RTAG_PROPATTR, &key.to_be_bytes()); record(&mut bytes, records::RTAG_PROPVALUE, value); } } record(&mut bytes, records::RTAG_ENDEL, &[]); } record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); let (remaining, full) = read_library(&bytes).unwrap(); assert!(remaining.is_empty()); let (_, scan) = scan_library(&bytes).unwrap(); let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); assert_eq!(selected, full); let cells = list_struct(&full, "cells", 0); let boundaries = list_struct(&cells, "boundary_props", 0); let paths = list_struct(&cells, "paths", 0); assert_eq!(boundaries.len(), 1); assert_eq!(paths.len(), 2); for elements in [&boundaries, &paths] { let properties = list_struct(elements, "properties", 0); let keys = properties .column_by_name("key") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert_eq!(keys.values().as_ref(), &[7, 7, 9]); let values = properties .column_by_name("value") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert_eq!( values.iter().collect::>(), [Some("first"), Some("second"), Some("")] ); } let empty_properties = paths .column_by_name("properties") .unwrap() .as_any() .downcast_ref::() .unwrap(); assert!(!empty_properties.is_null(1)); assert_eq!(empty_properties.value_length(1), 0); } #[test] fn selected_ranges_must_cover_exactly_one_complete_cell() { let bytes = fixture(); let (_, scan) = scan_library(&bytes).unwrap(); let ranges = ranges(&scan); let (start, end) = ranges[0]; for range in [ (end, start), (start, u64::MAX), (start, end - 1), (start + 4, end), (start, end + 4), (start, ranges[1].1), ] { assert!( read_selected_library(&bytes, &[range]).is_err(), "accepted {range:?}" ); } } #[test] fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() { let bytes = fixture(); let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!( unsafe { read_cells_bytes( bytes.as_ptr(), bytes.len(), ptr::null(), 0, &mut array, &mut schema, ) }, 0 ); let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); assert_eq!(list_struct(&empty, "cells", 0).len(), 0); let mut array = FFI_ArrowArray::empty(); assert_eq!( unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, 0 ); let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); let ranges: Vec = ranges(&scan) .into_iter() .flat_map(|(a, b)| [a, b]) .collect(); let mut array = FFI_ArrowArray::empty(); assert_eq!( unsafe { read_cells_bytes( bytes.as_ptr(), bytes.len(), ranges.as_ptr(), ranges.len() / 2, &mut array, &mut schema, ) }, 0 ); let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); assert_eq!(selected, read_library(&bytes).unwrap().1); assert!(last_error_message().is_null()); } #[test] fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() { let bytes = fixture(); let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!( unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) }, -1 ); assert!( unsafe { CStr::from_ptr(last_error_message()) } .to_str() .unwrap() .contains("scan failed") ); assert_eq!( unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) }, -1 ); assert_eq!( unsafe { read_cells_bytes( bytes.as_ptr(), bytes.len(), ptr::null(), 1, &mut array, &mut schema, ) }, -1 ); assert!( unsafe { CStr::from_ptr(last_error_message()) } .to_str() .unwrap() .contains("ranges pointer was null") ); assert_eq!( unsafe { read_cells_bytes( bytes.as_ptr(), bytes.len(), ptr::null(), usize::MAX, &mut array, &mut schema, ) }, -1 ); assert!( unsafe { CStr::from_ptr(last_error_message()) } .to_str() .unwrap() .contains("range count is too large") ); assert!(array.is_released()); }