From 54aa7576781572fb1184738b3d116b7181215cf6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 20:44:48 -0800 Subject: [PATCH] Validate GDSII record sizes through one fallible path Route every record read through size validation and report malformed payloads as parse errors instead of panicking. Apply metadata size constraints through the Record trait, hide Arrow reader internals, and remove unused wrappers and scratch code. Add coverage for direct, skipped, and payload reads, truncated buffers, and FFI error offsets. --- src/basic.rs | 1 - src/elements.rs | 38 +++++++-------- src/lib.rs | 2 +- src/library.rs | 11 ++--- src/misc.py | 39 --------------- src/record.rs | 77 ++++++++++++++--------------- src/records.rs | 41 +++------------- tests/ffi_errors.rs | 6 +-- tests/record_validation.rs | 99 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 148 deletions(-) delete mode 100644 src/misc.py create mode 100644 tests/record_validation.rs diff --git a/src/basic.rs b/src/basic.rs index f089ff3..1189b8e 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -206,7 +206,6 @@ mod tests { fn test_parse_bitarray() { use crate::basic::parse_bitarray; - //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); diff --git a/src/elements.rs b/src/elements.rs index f043ccc..02a8112 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,7 +4,6 @@ /// use crate::records::{ - //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, @@ -13,11 +12,10 @@ use crate::records::{ use crate::records; use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; //OResult +use crate::basic::{IResult, fail, take_bytes}; use std::string::String; use std::collections::HashMap; -//use std::io::Write; use arrow_array::builder::{ StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, @@ -25,8 +23,8 @@ use arrow_array::builder::{ }; -pub type DListBuilder = ListBuilder>; -pub type FListBuilder = FixedSizeListBuilder>; +pub(crate) type DListBuilder = ListBuilder>; +pub(crate) type FListBuilder = FixedSizeListBuilder>; struct PropertyRecord { key: i16, @@ -72,15 +70,15 @@ struct ARefRecord { } #[derive(Clone)] -pub struct ScanRefRecord { - pub target_id: u32, - pub invert_y: bool, - pub scale: f64, - pub angle_rad: f64, - pub xy: u64, - pub xy0: u64, - pub xy1: u64, - pub counts: u32, +pub(crate) struct ScanRefRecord { + pub(crate) target_id: u32, + pub(crate) invert_y: bool, + pub(crate) scale: f64, + pub(crate) angle_rad: f64, + pub(crate) xy: u64, + pub(crate) xy0: u64, + pub(crate) xy1: u64, + pub(crate) counts: u32, } enum RefRecord { @@ -117,7 +115,7 @@ fn insert_layer( } -pub fn read_elements<'a>( +pub(crate) fn read_elements<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, names: &mut HashMap, @@ -293,7 +291,7 @@ pub fn read_elements<'a>( } -pub fn scan_elements<'a>( +pub(crate) fn scan_elements<'a>( input: &'a [u8], names: &mut HashMap, ) -> IResult<'a, Vec> { @@ -583,7 +581,7 @@ fn append_boundary_prop( } -pub fn read_path<'a>( +fn read_path<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -654,7 +652,7 @@ pub fn read_path<'a>( Ok((input, ())) } -pub fn read_boxnode<'a>( +fn read_boxnode<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16, @@ -698,7 +696,7 @@ pub fn read_boxnode<'a>( Ok((input, ())) } -pub fn read_text<'a>( +fn read_text<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -967,7 +965,7 @@ fn read_ref<'a>( /// Preserves record order and duplicate property attributes. /// Stops reading after consuming ENDEL record. /// -pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { +fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; props_builder.append(true); Ok((input, ())) diff --git a/src/lib.rs b/src/lib.rs index 0abcca5..c31b907 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ pub mod basic; pub mod record; pub mod records; -pub mod elements; +mod elements; pub mod library; pub mod review; diff --git a/src/library.rs b/src/library.rs index 1ea4c6b..e455f8b 100644 --- a/src/library.rs +++ b/src/library.rs @@ -3,14 +3,11 @@ /// use std::io::Write; -//use std::collections::HashMap; -pub use crate::record; -pub use crate::record::{RecordHeader, Record}; -pub use crate::records; -pub use crate::elements; -pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -pub use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::record::{RecordHeader, Record}; +use crate::records; +use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +use crate::basic::{IResult, OResult, take_bytes, fail}; use std::string::String; use std::collections::HashMap; diff --git a/src/misc.py b/src/misc.py deleted file mode 100644 index 1e507c7..0000000 --- a/src/misc.py +++ /dev/null @@ -1,39 +0,0 @@ -''' -https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py -''' -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import ctypes -import pyarrow as pa -from pyarrow.cffi import ffi - - c_schema = ffi.new('struct ArrowSchema*') - ptr_schema = int(ffi.cast('uintptr_t', c_schema)) - c_array = ffi.new('struct ArrowArray*') - ptr_array = int(ffi.cast('uintptr_t', c_array)) - - # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) - - # import gc - # gc.collect() # Make sure no Arrow data dangles in a ref cycle - # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) - - arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) - - diff --git a/src/record.rs b/src/record.rs index cb64ea2..25d358a 100644 --- a/src/record.rs +++ b/src/record.rs @@ -4,16 +4,12 @@ use std::io::{self, Write}; -use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; use crate::records; -//#[no_mangle] -//pub extern "C" fn write_record_header( - - #[repr(C)] #[derive(Debug, Clone)] pub struct RecordHeader { @@ -53,7 +49,6 @@ pub trait RecordData { fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; fn pack_into(buf: &mut [u8], data: &Self::InData); - //fn size(data: &Self::BareData<'_>) -> u16; fn pack(data: &Self::InData) -> Self::ByteData; } @@ -73,22 +68,17 @@ pub trait Record { } } - fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { - RecordHeader::read(input) - } - fn write_header(ww: &mut W, data_size: u16) -> OResult { RecordHeader{tag: Self::tag(), data_size}.write(ww) } fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + if let Err(message) = Self::check_size(size) { + return fail(input, message); + } RData::read(input, size) } - fn pack_data(buf: &mut [u8], data: &RData::InData) { - RData::pack_into(buf, data) - } - /// /// Skip to the end of the next occurence of this record. /// @@ -116,8 +106,7 @@ pub trait Record { (input, _) = take_bytes(input, header.data_size)?; (input, header) = RecordHeader::read(input)?; } - let (input, data) = Self::read_data(input, header.data_size)?; - Ok((input, data)) + Self::read_data(input, header.data_size) } fn expect_header(input: &[u8]) -> IResult<'_, u16> { @@ -131,9 +120,7 @@ pub trait Record { fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { let (input, size) = Self::expect_header(input)?; - Self::check_size(size).unwrap(); - let (input, data) = Self::read_data(input, size)?; - Ok((input, data)) + Self::read_data(input, size) } fn write(ww: &mut W, data: &RData::InData) -> OResult { @@ -155,7 +142,9 @@ impl RecordData for BitArray { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_bitarray(input) } @@ -178,7 +167,9 @@ impl RecordData for Int2 { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_int2(input) } @@ -200,7 +191,9 @@ impl RecordData for Int4 { type ByteData = [u8; 4]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 4); + if size != 4 { + return fail(input, format!("Expected record size {}, got {size}", 4)); + } parse_int4(input) } @@ -223,8 +216,9 @@ impl RecordData for Int2Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(2) { + return fail(input, format!("Record size must be divisible by 2, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int2ArrayReader{bytes})) } @@ -267,8 +261,9 @@ impl RecordData for Int4Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(4) { + return fail(input, format!("Record size must be divisible by 4, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int4ArrayReader{bytes})) } @@ -311,7 +306,9 @@ impl RecordData for Real8 { type ByteData = [u8; 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 8); + if size != 8 { + return fail(input, format!("Expected record size {}, got {size}", 8)); + } parse_real8(input) } @@ -333,7 +330,9 @@ impl RecordData for Real8Pair { type ByteData = [u8; 2 * 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 8); + if size != 2 * 8 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 8)); + } let (input, data0) = parse_real8(input)?; let (input, data1) = parse_real8(input)?; Ok((input, (data0, data1))) @@ -350,10 +349,6 @@ impl RecordData for Real8Pair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct ASCII; @@ -385,7 +380,9 @@ impl RecordData for DateTimePair { type ByteData = [u8; 2 * 6 * 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 6 * 2); + if size != 2 * 6 * 2 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2)); + } let (input, data0) = parse_datetime(input)?; let (input, data1) = parse_datetime(input)?; Ok((input, [data0, data1])) @@ -402,10 +399,6 @@ impl RecordData for DateTimePair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct Empty; @@ -415,7 +408,9 @@ impl RecordData for Empty { type ByteData = [u8; 0]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 0); + if size != 0 { + return fail(input, format!("Expected record size {}, got {size}", 0)); + } Ok((input, ())) } @@ -425,6 +420,4 @@ impl RecordData for Empty { fn pack(_data: &Self::InData) -> Self::ByteData { [] } - //fn write(ww: &mut W, data: &Self::BareData) { - //} } diff --git a/src/records.rs b/src/records.rs index 4e86404..5481e2b 100644 --- a/src/records.rs +++ b/src/records.rs @@ -4,8 +4,6 @@ use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; -//use std::io::Write; - // record tags pub const RTAG_HEADER: u16 = 0x0002; pub const RTAG_BGNLIB: u16 = 0x0102; @@ -77,37 +75,13 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300; pub const RTAG_SPACERERROR: u16 = 0x4400; pub const RTAG_CONTACT: u16 = 0x4500; -/* -// data types -pub const DATA_TYPE_NONE: u16 = 0x00; -pub const DATA_TYPE_BIT: u16 = 0x01; -pub const DATA_TYPE_INT16: u16 = 0x02; -pub const DATA_TYPE_INT32: u16 = 0x03; -pub const DATA_TYPE_REAL32: u16 = 0x04; -pub const DATA_TYPE_REAL64: u16 = 0x05; -pub const DATA_TYPE_STR: u16 = 0x06; -pub const MAX_DATA_SIZE: usize = 8; - -/// Returns the size of the given data type in bytes. -pub fn data_size(t: u16) -> Option { - match t { - x if x == DATA_TYPE_NONE => 0, - x if x == DATA_TYPE_BIT => 2, - x if x == DATA_TYPE_INT16 => 2, - x if x == DATA_TYPE_INT32 => 4, - x if x == DATA_TYPE_REAL32 => 4, - x if x == DATA_TYPE_REAL64 => 8, - _ => 0 - } -*/ pub struct HEADER; impl Record for HEADER { fn tag() -> u16 { RTAG_HEADER } fn expected_size() -> Option { Some(2) } } -//impl Record for HEADER; pub struct BGNLIB; impl Record for BGNLIB { @@ -288,9 +262,8 @@ pub struct REFLIBS; impl Record for REFLIBS { fn tag() -> u16 { RTAG_REFLIBS } fn expected_size() -> Option { None } -} -impl REFLIBS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -303,9 +276,8 @@ pub struct FONTS; impl Record for FONTS { fn tag() -> u16 { RTAG_FONTS } fn expected_size() -> Option { None } -} -impl FONTS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -330,9 +302,8 @@ pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } fn expected_size() -> Option { None } -} -impl ATTRTABLE { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs index 0e74a5e..3b81821 100644 --- a/tests/ffi_errors.rs +++ b/tests/ffi_errors.rs @@ -31,20 +31,20 @@ fn error_message() -> String { } #[test] -fn path_errors_preserve_outputs_and_convert_parser_panics() { +fn path_errors_preserve_outputs_and_clear_after_success() { let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); assert!(error_message().contains("path pointer was null")); assert!(array.is_released()); - // A mismatched timestamp size currently makes Record::read panic. + // A mismatched timestamp size returns a parse error with a byte offset. let mut bytes = Vec::new(); common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); let file = TestFile::new("invalid-timestamps", &bytes); assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); - assert!(error_message().contains("Expected record size")); + assert!(error_message().contains("read failed at byte 10: Expected record size")); assert!(array.is_released()); let mut bytes = common::library_header(); diff --git a/tests/record_validation.rs b/tests/record_validation.rs new file mode 100644 index 0000000..d1e4fa7 --- /dev/null +++ b/tests/record_validation.rs @@ -0,0 +1,99 @@ +mod common; + +use pheon::{ + basic::ErrType, + record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, + }, + records, +}; + +#[test] +fn fixed_record_sizes_are_checked_by_every_read_entry_point() { + let payload = [0; 4]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &payload); + let mut prefixed = Vec::new(); + common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip"); + prefixed.extend_from_slice(&bytes); + + for result in [ + records::HEADER::read(&bytes), + records::HEADER::skip_and_read(&prefixed), + records::HEADER::read_data(&payload, 4), + ] { + let (remaining, reason) = result.unwrap_err(); + assert_eq!(remaining, payload); + assert!(matches!(reason, ErrType::Failed(message) + if message == "Expected record size 2, got 4")); + } + + // The underlying integer-array payload is aligned, but COLROW needs two values. + let payload = [0; 2]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_COLROW, &payload); + assert!(records::COLROW::read(&bytes).is_err()); + assert!(records::COLROW::skip_and_read(&bytes).is_err()); + assert!(records::COLROW::read_data(&payload, 2).is_err()); +} + +#[test] +fn metadata_size_constraints_apply_to_every_read_entry_point() { + macro_rules! check { + ($record:ident, $tag:ident) => {{ + let payload = [b'x'; 44]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + let mut prefixed = common::library_header(); + prefixed.extend_from_slice(&bytes); + assert_eq!(records::$record::read(&bytes).unwrap().1, payload); + assert_eq!( + records::$record::skip_and_read(&prefixed).unwrap().1, + payload + ); + assert_eq!( + records::$record::read_data(&payload, 44).unwrap().1, + payload + ); + + let payload = [b'x'; 46]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + assert!(records::$record::read(&bytes).is_err()); + assert!(records::$record::skip_and_read(&bytes).is_err()); + assert!(records::$record::read_data(&payload, 46).is_err()); + }}; + } + + check!(REFLIBS, RTAG_REFLIBS); + check!(FONTS, RTAG_FONTS); + check!(ATTRTABLE, RTAG_ATTRTABLE); +} + +#[test] +fn malformed_payload_sizes_return_errors_without_panicking() { + let payload = [0; 32]; + assert!(BitArray::read(&payload, 4).is_err()); + assert!(Int2::read(&payload, 4).is_err()); + assert!(Int4::read(&payload, 2).is_err()); + assert!(Real8::read(&payload, 4).is_err()); + assert!(Real8Pair::read(&payload, 8).is_err()); + assert!(DateTimePair::read(&payload, 12).is_err()); + assert!(Empty::read(&payload, 2).is_err()); + assert!(Int2Array::read(&payload, 3).is_err()); + assert!(Int4Array::read(&payload, 6).is_err()); + + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_XY, &payload[..6]); + assert!(records::XY::read(&bytes).is_err()); + assert!(records::XY::skip_and_read(&bytes).is_err()); + assert!(records::XY::read_data(&payload, 6).is_err()); + + // Valid sizes still reject a truncated buffer; an empty string remains valid. + assert!(Int4::read(&payload[..2], 4).is_err()); + assert!(Real8Pair::read(&payload[..8], 16).is_err()); + assert!(DateTimePair::read(&payload[..12], 24).is_err()); + assert!(Int4Array::read(&payload[..4], 8).is_err()); + assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::::new())); +}