diff --git a/ARROW_SCHEMA.md b/ARROW_SCHEMA.md index f80112b..39047a9 100644 --- a/ARROW_SCHEMA.md +++ b/ARROW_SCHEMA.md @@ -24,7 +24,7 @@ Each cell in the `cells` list is a `StructArray`. | Field | Type | Description | |---|---|---| | `id` | `UInt32` | Unique identifier for the cell. | -| `file_offset` | `UInt64` | Byte offset of the cell in the original GDSII file. | +| `file_offset` | `UInt64` | Byte offset immediately after the cell's `STRNAME` record (the first element, or `ENDSTR` for an empty cell). Identical for full and selected reads. | | `srefs` | `List(SRef)` | Simple structure references (no properties). | | `arefs` | `List(ARef)` | Array references (no properties). | | `sref_props` | `List(SRefProp)` | Structure references with properties. | @@ -37,6 +37,31 @@ Each cell in the `cells` list is a `StructArray`. | `boxes` | `List(BoxNode)` | Box elements. | | `texts` | `List(Text)` | Text labels. | +## Hierarchy Scan Schema + +`scan_library` and `scan_bytes` return the same library metadata and `cell_names` +as a full read, with the following `cells` schema and no `layers` field: + +| Field | Type | Description | +|---|---|---| +| `id` | `UInt32` | Cell ID indexing this result's `cell_names`. | +| `struct_start_offset` | `UInt64` | Inclusive byte offset of the `BGNSTR` record. | +| `struct_end_offset` | `UInt64` | Exclusive byte offset immediately after `ENDSTR`. | +| `refs` | `List(ScanRef)` | SREFs and AREFs, including references with properties, sorted by target ID. | + +Each scan reference has `target`, `invert_y`, `scale`, `angle_rad`, `xy`, `xy0`, +`xy1`, and `counts`, using the ARef types and packed encodings described below. +SREFs use zero basis vectors and counts `(1, 1)`. Properties are omitted from the +scan output; geometry and text are skipped without decoding. + +Pass the half-open scan ranges to `read_selected_library` or `read_cells_bytes` +to obtain the full library schema for just those cells. Each range must contain +exactly one complete cell, from `BGNSTR` through `ENDSTR`. Cells appear in the +requested order. Cell and layer IDs are assigned within each result, so use +`cell_names` and `layers` from that result when resolving them; IDs can differ +from a full read or a scan. An empty selection returns metadata and empty lists. +The Rust random-access reader returns the original input slice as its remainder. + ## Element Types ### Common Packed Encodings diff --git a/README.md b/README.md index 9f65aaa..e5a43a7 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,25 @@ after a successful parser call. Errors belong to the calling thread. Copy the message before making another parser call on that thread, and before the thread exits; calls on other threads do not invalidate it. Parser panics that unwind are converted into errors. + +For partial reads, call `scan_bytes` with a borrowed GDSII byte buffer to obtain +cell ranges and references without decoding geometry. Pass the desired ranges +to `read_cells_bytes` to load their full contents. Both calls use the same Arrow +outputs and status/error handling as `read_path`: + +```c +int scan_bytes(const uint8_t *data, size_t size, + struct ArrowArray *array, struct ArrowSchema *schema); +int read_cells_bytes(const uint8_t *data, size_t size, + const uint64_t *ranges, size_t range_count, + struct ArrowArray *array, struct ArrowSchema *schema); +``` + +`ranges` contains alternating inclusive start and exclusive end offsets, with +two native-endian `uint64_t` values per cell. Each range must cover one complete +cell. A null `ranges` pointer is accepted when `range_count` is zero. Likewise, +null `data` is accepted for zero bytes and produces an ordinary parse error for +the empty input. Nonempty buffers must be readable for the duration of the call; +range values must be aligned. Exported Arrow data owns its storage independently +of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan +schema, offset definitions, and ID assignment. diff --git a/src/elements.rs b/src/elements.rs index e42e3f8..76988ba 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -71,6 +71,18 @@ struct ARefRecord { properties: Vec, } +#[derive(Clone)] +pub struct ScanRefRecord { + pub target_id: u32, + pub invert_y: bool, + pub scale: f64, + pub angle_rad: f64, + pub xy: u64, + pub xy0: u64, + pub xy1: u64, + pub counts: u32, +} + enum RefRecord { SRef(SRefRecord), ARef(ARefRecord), @@ -78,6 +90,7 @@ enum RefRecord { const UNIT_COUNTS: u32 = (1_u32 << 16) | 1_u32; +const ZERO_XY: u64 = 0; fn layer_id_for( @@ -280,6 +293,38 @@ pub fn read_elements<'a>( } +pub fn scan_elements<'a>( + input: &'a [u8], + names: &mut HashMap, + ) -> IResult<'a, Vec> { + let mut input = input; + let mut refs = Vec::::new(); + let (_, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDSTR { + (input, _) = match header.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, header.tag, names)?; + refs.push(scan_ref_from_record(ref_record)); + (next_input, ()) + }, + records::RTAG_BOUNDARY + | records::RTAG_PATH + | records::RTAG_NODE + | records::RTAG_BOX + | records::RTAG_TEXT => skip_element(input)?, + _ => { + let result = take_bytes(input, header.data_size + 4)?; + (result.0, ()) + } + }; + (_, header) = RecordHeader::read(input)?; + } + refs.sort_by_key(|scan_ref| scan_ref.target_id); + (input, _) = take_bytes(input, 4_usize)?; + Ok((input, refs)) +} + + fn append_sref( sref_builder: &mut DListBuilder, sref: SRefRecord, @@ -301,6 +346,45 @@ fn append_sref( } +fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { + match ref_record { + RefRecord::SRef(sref) => ScanRefRecord { + target_id: sref.target_id, + invert_y: sref.invert_y, + scale: sref.scale, + angle_rad: sref.angle_rad, + xy: sref.xy, + xy0: ZERO_XY, + xy1: ZERO_XY, + counts: UNIT_COUNTS, + }, + RefRecord::ARef(aref) => ScanRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + xy0: aref.xy0, + xy1: aref.xy1, + counts: aref.counts, + }, + } +} + + +fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { + let mut input = input; + loop { + let (next_input, header) = RecordHeader::read(input)?; + let (next_input, _) = take_bytes(next_input, header.data_size)?; + input = next_input; + if header.tag == ENDEL::tag() { + return Ok((input, ())); + } + } +} + + fn append_aref( aref_builder: &mut DListBuilder, aref: ARefRecord, diff --git a/src/lib.rs b/src/lib.rs index 59cada7..d6c5011 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,7 @@ pub mod records; pub mod elements; pub mod library; -use crate::library::read_library; +use crate::library::{read_library, read_selected_library, scan_library}; use byteorder::{ByteOrder, BigEndian}; use std::any::Any; @@ -20,6 +20,7 @@ use std::ptr; use std::os::unix::ffi::OsStringExt; use std::str; use std::fs; +use std::slice; use arrow::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; use arrow::array::{Array, StructArray}; @@ -133,6 +134,76 @@ pub unsafe extern "C" fn read_path( } +unsafe fn input_slice<'a, T>(data: *const T, size: usize, name: &str) -> Result<&'a [T], String> { + if size == 0 { + return Ok(&[]); + } + if data.is_null() { + return Err(format!("{name} pointer was null")); + } + if !data.is_aligned() { + return Err(format!("{name} pointer was misaligned")); + } + if size.checked_mul(size_of::()).is_none_or(|bytes| bytes > isize::MAX as usize) { + return Err(format!("{name} length is too large")); + } + Ok(unsafe { slice::from_raw_parts(data, size) }) +} + + +/// Scan cell ranges and hierarchy references. Return 0 on success or -1 on error. +/// +/// # Safety +/// Non-null `data` must point to `size` readable bytes. Null is allowed when +/// `size` is zero. Output pointers have the same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scan_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("scan_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = scan_library(input) + .map_err(|err| format!("scan failed: {err:?}"))?; + Ok(struct_arr) + }) +} + + +/// Read the cells in half-open byte ranges. Return 0 on success or -1 on error. +/// +/// # Safety +/// `data` has the same requirements as `scan_bytes`. Non-null `ranges` must +/// point to `range_count * 2` readable, aligned u64 values, alternating start +/// and end offsets. Null is allowed for zero ranges. Output pointers have the +/// same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_cells_bytes( + data: *const u8, + size: usize, + ranges: *const u64, + range_count: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_cells_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let range_size = range_count.checked_mul(2) + .ok_or_else(|| "range count is too large".to_string())?; + let flat_ranges = unsafe { input_slice(ranges, range_size, "ranges") }?; + let cell_ranges: Vec<(u64, u64)> = flat_ranges + .chunks_exact(2) + .map(|chunk| (chunk[0], chunk[1])) + .collect(); + let (_input, struct_arr) = read_selected_library(input, &cell_ranges) + .map_err(|err| format!("selected-cell read failed: {err:?}"))?; + Ok(struct_arr) + }) +} + + macro_rules! impl_i16be { ( $tt:ty, $arr:ident, $size:ident ) => { { diff --git a/src/library.rs b/src/library.rs index bf4c890..c128671 100644 --- a/src/library.rs +++ b/src/library.rs @@ -9,7 +9,7 @@ pub use crate::record; pub use crate::record::{RecordHeader, Record}; pub use crate::records; pub use crate::elements; -pub use crate::elements::{read_elements, DListBuilder, FListBuilder}; +pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; pub use crate::basic::{IResult, OResult, take_bytes, fail}; use std::string::String; @@ -18,7 +18,7 @@ use std::sync::Arc; use arrow::datatypes::{DataType, Field, Fields}; use arrow::array::{ - StructBuilder, StringBuilder, UInt64Builder, UInt32Builder, Int16Builder, Float64Builder, + BooleanBuilder, StructBuilder, StringBuilder, UInt64Builder, UInt32Builder, Int16Builder, Float64Builder, StructArray, }; @@ -98,9 +98,38 @@ impl FileHeader { } -pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { - let input_size = input.len(); +fn make_name_list_type() -> DataType { + DataType::List(Arc::new( + Field::new_list_field(DataType::Utf8, false) + )) +} + +fn make_time_type() -> DataType { + DataType::FixedSizeList(Arc::new( + Field::new_list_field(DataType::Int16, false), + ), + 6, + ) +} + + +fn make_top_level_builder(cells_field: Field, extra_fields: Vec) -> StructBuilder { + let mut fields = vec![ + Field::new("meters_per_db_unit", DataType::Float64, false), + Field::new("user_units_per_db_unit", DataType::Float64, false), + Field::new("lib_name", DataType::Utf8, false), + Field::new("mod_time", make_time_type(), false), + Field::new("acc_time", make_time_type(), false), + cells_field, + Field::new("cell_names", make_name_list_type(), false), + ]; + fields.extend(extra_fields); + StructBuilder::from_fields(fields, 0) +} + + +fn make_library_builder() -> StructBuilder { let property_t = DataType::Struct(Fields::from(vec![ Field::new("key", DataType::Int16, false), Field::new("value", DataType::Utf8, false), @@ -248,20 +277,10 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { Field::new_list_field(boxnode_struct_t, false) )); - let name_list_t = DataType::List(Arc::new( - Field::new_list_field(DataType::Utf8, false) - )); - let layer_list_t = DataType::List(Arc::new( Field::new_list_field(DataType::UInt32, false) )); - let time_t = DataType::FixedSizeList(Arc::new( - Field::new_list_field(DataType::Int16, false), - ), - 6, - ); - let cell_struct_t = DataType::Struct(Fields::from(vec![ Field::new("id", DataType::UInt32, false), Field::new("file_offset", DataType::UInt64, false), @@ -282,26 +301,59 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { Field::new_list_field(cell_struct_t, false) )); - let mut lib_builder = StructBuilder::from_fields(vec![ - Field::new("meters_per_db_unit", DataType::Float64, false), - Field::new("user_units_per_db_unit", DataType::Float64, false), - Field::new("lib_name", DataType::Utf8, false), - Field::new("mod_time", time_t.clone(), false), - Field::new("acc_time", time_t.clone(), false), - Field::new("cells", cells_list_t, false), - Field::new("cell_names", name_list_t, false), + make_top_level_builder( + Field::new("cells", cells_list_t, false), + vec![ Field::new("layers", layer_list_t, false), ], - 0, - ); + ) +} - let (input, header) = FileHeader::read(input)?; + +fn make_scan_builder() -> StructBuilder { + let scan_ref_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("target", DataType::UInt32, false), + Field::new("invert_y", DataType::Boolean, false), + Field::new("scale", DataType::Float64, false), + Field::new("angle_rad", DataType::Float64, false), + Field::new("xy", DataType::UInt64, false), + Field::new("xy0", DataType::UInt64, false), + Field::new("xy1", DataType::UInt64, false), + Field::new("counts", DataType::UInt32, false), + ])); + + let scan_ref_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_ref_struct_t, false) + )); + + let scan_cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("struct_start_offset", DataType::UInt64, false), + Field::new("struct_end_offset", DataType::UInt64, false), + Field::new("refs", scan_ref_list_t, false), + ])); + + let scan_cells_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", scan_cells_list_t, false), + Vec::new(), + ) +} + + +fn append_file_header( + lib_builder: &mut StructBuilder, + header: &FileHeader, + ) { let dbu_builder = lib_builder.field_builder::(0).unwrap(); dbu_builder.append_value(header.meters_per_db_unit); let uu_builder = lib_builder.field_builder::(1).unwrap(); uu_builder.append_value(header.user_units_per_db_unit); let libname_builder = lib_builder.field_builder::(2).unwrap(); - libname_builder.append_value(String::from_utf8(header.name).unwrap()); + libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); let mtl_builder = lib_builder.field_builder::(3).unwrap(); let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); mt_builder.append_values(&header.mod_time, &[true; 6]); @@ -310,47 +362,28 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); at_builder.append_values(&header.acc_time, &[true; 6]); atl_builder.append(true); +} - let mut names = HashMap::::new(); - let mut layers = HashMap::::new(); - let cells_builder = lib_builder.field_builder::(5).unwrap(); - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDLIB { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_BGNSTR { - let name_bytes; - (input, name_bytes) = records::STRNAME::read(input)?; - let name = String::from_utf8(name_bytes).unwrap(); - //println!("{name}"); - - let next_id = names.len(); - let id = names.entry(name).or_insert(next_id.try_into().unwrap()); - let position = input_size - input.len(); - - let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); - let id_builder = cell_builder.field_builder::(0).unwrap(); - id_builder.append_value(*id); - - let offset_builder = cell_builder.field_builder::(1).unwrap(); - offset_builder.append_value(position.try_into().unwrap()); - - (input, _) = read_elements(input, cell_builder, &mut names, &mut layers)?; - - cell_builder.append(true); - } - (input, header) = RecordHeader::read(input)?; - } - cells_builder.append(true); +fn append_names( + lib_builder: &mut StructBuilder, + field_index: usize, + names: HashMap, + ) { let mut ids: HashMap = names.into_iter().map(|(kk, vv)| (vv, kk)).collect(); - let names_builder = lib_builder.field_builder::(6).unwrap(); + let names_builder = lib_builder.field_builder::(field_index).unwrap(); let name_builder = names_builder.values().as_any_mut().downcast_mut::().unwrap(); for id in 0..ids.len() { name_builder.append_value(ids.remove(&id.try_into().unwrap()).unwrap()); } names_builder.append(true); +} + +fn append_layers( + lib_builder: &mut StructBuilder, + layers: HashMap, + ) { let mut layer_ids: HashMap = layers.into_iter().map(|(kk, vv)| (vv, kk)).collect(); let layers_builder = lib_builder.field_builder::(7).unwrap(); let layer_builder = layers_builder.values().as_any_mut().downcast_mut::().unwrap(); @@ -358,6 +391,211 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { layer_builder.append_value(layer_ids.remove(&layer_id.try_into().unwrap()).unwrap()); } layers_builder.append(true); +} + + +fn append_scan_ref( + refs_builder: &mut DListBuilder, + scan_ref: ScanRefRecord, + ) { + let ref_builder = refs_builder.values().as_any_mut().downcast_mut::().unwrap(); + let target_builder = ref_builder.field_builder::(0).unwrap(); + target_builder.append_value(scan_ref.target_id); + let invert_builder = ref_builder.field_builder::(1).unwrap(); + invert_builder.append_value(scan_ref.invert_y); + let scale_builder = ref_builder.field_builder::(2).unwrap(); + scale_builder.append_value(scan_ref.scale); + let angle_builder = ref_builder.field_builder::(3).unwrap(); + angle_builder.append_value(scan_ref.angle_rad); + let xy_builder = ref_builder.field_builder::(4).unwrap(); + xy_builder.append_value(scan_ref.xy); + let xy0_builder = ref_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(scan_ref.xy0); + let xy1_builder = ref_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(scan_ref.xy1); + let counts_builder = ref_builder.field_builder::(7).unwrap(); + counts_builder.append_value(scan_ref.counts); + ref_builder.append(true); +} + + +fn intern_name(names: &mut HashMap, name: String) -> u32 { + let next_id = names.len(); + *names.entry(name).or_insert(next_id.try_into().unwrap()) +} + + +fn append_full_cell<'a>( + cells_builder: &mut DListBuilder, + names: &mut HashMap, + layers: &mut HashMap, + struct_input: &'a [u8], + id: u32, + file_offset: u64, + ) -> IResult<'a, ()> { + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let offset_builder = cell_builder.field_builder::(1).unwrap(); + offset_builder.append_value(file_offset); + + let (next_input, _) = read_elements(struct_input, cell_builder, names, layers)?; + cell_builder.append(true); + Ok((next_input, ())) +} + + +fn walk_structs<'a, F>( + input: &'a [u8], + base_offset: usize, + names: &mut HashMap, + mut on_struct: F, + ) -> IResult<'a, ()> +where + F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, +{ + let input_size = input.len(); + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDLIB { + let record_start = base_offset + input_size - input.len() - 4; + (input, _) = take_bytes(input, header.data_size)?; + if header.tag == records::RTAG_BGNSTR { + let name_bytes; + (input, name_bytes) = records::STRNAME::read(input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let id = intern_name(names, name); + let (next_input, ()) = on_struct(input, record_start, id, names)?; + input = next_input; + } + (input, header) = RecordHeader::read(input)?; + } + Ok((input, ())) +} + + +pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_library_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, _record_start, id, names| { + let position = base_offset + walk_input_size - struct_input.len(); + append_full_cell( + cells_builder, + names, + &mut layers, + struct_input, + id, + position.try_into().unwrap(), + ) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Scan cell byte ranges and references without decoding geometry. +pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_scan_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, record_start, id, names| { + let (after_cell, refs) = scan_elements(struct_input, names)?; + let struct_end = base_offset + walk_input_size - after_cell.len(); + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let start_builder = cell_builder.field_builder::(1).unwrap(); + start_builder.append_value(record_start.try_into().unwrap()); + + let end_builder = cell_builder.field_builder::(2).unwrap(); + end_builder.append_value(struct_end.try_into().unwrap()); + + let refs_builder = cell_builder.field_builder::(3).unwrap(); + for scan_ref in refs { + append_scan_ref(refs_builder, scan_ref); + } + refs_builder.append(true); + cell_builder.append(true); + Ok((after_cell, ())) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Read exact half-open cell ranges returned by `scan_library`, in caller order. +/// Each range includes BGNSTR through ENDSTR. Cell and layer IDs are local to +/// this result. This random-access operation returns the original input slice +/// as its remainder; it does not advance a sequential parsing cursor. +pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { + let mut lib_builder = make_library_builder(); + let (_, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + for (start_u64, end_u64) in ranges { + if start_u64 >= end_u64 || *end_u64 > input.len() as u64 { + return fail(input, format!("Cell byte range out of bounds: [{start_u64}, {end_u64})")); + } + let start = usize::try_from(*start_u64).unwrap(); + let end = usize::try_from(*end_u64).unwrap(); + + let struct_input = &input[start..end]; + let (struct_input, _) = records::BGNSTR::read(struct_input)?; + let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let file_offset = end - struct_input.len(); + + let id = intern_name(&mut names, name); + let (remaining_struct, _) = append_full_cell( + cells_builder, + &mut names, + &mut layers, + struct_input, + id, + file_offset.try_into().unwrap(), + )?; + if !remaining_struct.is_empty() { + return fail(remaining_struct, "Cell byte range extends past ENDSTR".to_string()); + } + } + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); lib_builder.append(true); let lib = lib_builder.finish(); diff --git a/src/record.rs b/src/record.rs index 8bc482c..44a9812 100644 --- a/src/record.rs +++ b/src/record.rs @@ -27,7 +27,10 @@ impl RecordHeader { pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { let (input, size) = parse_u16(input)?; let (input, tag) = parse_u16(input)?; - Ok((input, RecordHeader{tag, data_size:size - 4})) + let Some(data_size) = size.checked_sub(4) else { + return fail(input, format!("Record size {size} is smaller than its header")); + }; + Ok((input, RecordHeader{tag, data_size})) } pub fn pack_into(&self) -> [u8; 4] { diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs new file mode 100644 index 0000000..de9e34c --- /dev/null +++ b/tests/selective_reads.rs @@ -0,0 +1,349 @@ +mod common; + +use arrow::array::{ + Array, BooleanArray, Float64Array, ListArray, StringArray, StructArray, UInt32Array, + UInt64Array, +}; +use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; +use common::{library_header, record}; +use klamath_rs_ext::{ + basic::encode_real8, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_cells_bytes, records, scan_bytes, +}; +use std::{ffi::CStr, ptr}; + +fn xy(values: &[i32]) -> Vec { + values.iter().flat_map(|v| v.to_be_bytes()).collect() +} + +fn fixture() -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_XY, &xy(&[3, -4])); + record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]); + record( + &mut bytes, + records::RTAG_MAG, + &encode_real8(2.0).unwrap().to_be_bytes(), + ); + record( + &mut bytes, + records::RTAG_ANGLE, + &encode_real8(90.0).unwrap().to_be_bytes(), + ); + record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]); + record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray { + let list = array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + list.value(row) + .as_any() + .downcast_ref::() + .unwrap() + .clone() +} + +fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 { + array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row) +} + +fn ranges(scan: &StructArray) -> Vec<(u64, u64)> { + let cells = list_struct(scan, "cells", 0); + (0..cells.len()) + .map(|i| { + ( + u64_value(&cells, "struct_start_offset", i), + u64_value(&cells, "struct_end_offset", i), + ) + }) + .collect() +} + +#[test] +fn scanned_ranges_reproduce_the_full_library_including_offsets() { + let bytes = fixture(); + let (_, full) = read_library(&bytes).unwrap(); + let (scan_remaining, scan) = scan_library(&bytes).unwrap(); + assert!(scan_remaining.is_empty()); + let ranges = ranges(&scan); + assert_eq!(ranges.len(), 2); + for &(start, end) in &ranges { + assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]); + assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]); + } + let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap(); + assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged. + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8); +} + +#[test] +fn scan_preserves_reference_transforms_and_skips_geometry() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let cells = list_struct(&scan, "cells", 0); + let refs = list_struct(&cells, "refs", 0); + assert_eq!(refs.len(), 2); + assert_eq!(list_struct(&cells, "refs", 1).len(), 0); + let target = refs + .column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(target.values().as_ref(), &[1, 1]); + assert_eq!( + u64_value(&refs, "xy", 0), + (3_u64 << 32) | (-4_i32 as u32 as u64) + ); + assert_eq!(u64_value(&refs, "xy0", 0), 0); + assert_eq!(u64_value(&refs, "xy1", 0), 0); + let counts = refs + .column_by_name("counts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]); + assert!( + refs.column_by_name("invert_y") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1) + ); + assert_eq!( + refs.column_by_name("scale") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + 2.0 + ); + assert_eq!( + refs.column_by_name("angle_rad") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + std::f64::consts::FRAC_PI_2 + ); + assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32); + assert_eq!(u64_value(&refs, "xy1", 1), 10); +} + +#[test] +fn selections_support_subsets_reordering_and_empty_results() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap(); + let leaf_cells = list_struct(&leaf, "cells", 0); + assert_eq!(leaf_cells.len(), 1); + let leaf_names = leaf + .column_by_name("cell_names") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!( + leaf_names + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "leaf" + ); + let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap(); + let reordered_cells = list_struct(&reordered, "cells", 0); + assert_eq!( + u64_value(&reordered_cells, "file_offset", 0), + u64_value(&leaf_cells, "file_offset", 0) + ); + let refs = list_struct(&reordered_cells, "sref_props", 1); + assert_eq!( + refs.column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + let (_, empty) = read_selected_library(&bytes, &[]).unwrap(); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); +} + +#[test] +fn selected_ranges_must_cover_exactly_one_complete_cell() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (start, end) = ranges[0]; + for range in [ + (end, start), + (start, u64::MAX), + (start, end - 1), + (start + 4, end), + (start, end + 4), + (start, ranges[1].1), + ] { + assert!( + read_selected_library(&bytes, &[range]).is_err(), + "accepted {range:?}" + ); + } +} + +#[test] +fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + 0 + ); + let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); + + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + 0 + ); + let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + let ranges: Vec = ranges(&scan) + .into_iter() + .flat_map(|(a, b)| [a, b]) + .collect(); + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + ranges.len() / 2, + &mut array, + &mut schema, + ) + }, + 0 + ); + let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(selected, read_library(&bytes).unwrap().1); + assert!(last_error_message().is_null()); +} + +#[test] +fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("scan failed") + ); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) }, + -1 + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("ranges pointer was null") + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + usize::MAX, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("range count is too large") + ); + assert!(array.is_released()); +}