diff --git a/src/basic.rs b/src/basic.rs index f089ff3..1189b8e 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -206,7 +206,6 @@ mod tests { fn test_parse_bitarray() { use crate::basic::parse_bitarray; - //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); diff --git a/src/elements.rs b/src/elements.rs index f043ccc..02a8112 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,7 +4,6 @@ /// use crate::records::{ - //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, @@ -13,11 +12,10 @@ use crate::records::{ use crate::records; use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; //OResult +use crate::basic::{IResult, fail, take_bytes}; use std::string::String; use std::collections::HashMap; -//use std::io::Write; use arrow_array::builder::{ StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, @@ -25,8 +23,8 @@ use arrow_array::builder::{ }; -pub type DListBuilder = ListBuilder>; -pub type FListBuilder = FixedSizeListBuilder>; +pub(crate) type DListBuilder = ListBuilder>; +pub(crate) type FListBuilder = FixedSizeListBuilder>; struct PropertyRecord { key: i16, @@ -72,15 +70,15 @@ struct ARefRecord { } #[derive(Clone)] -pub struct ScanRefRecord { - pub target_id: u32, - pub invert_y: bool, - pub scale: f64, - pub angle_rad: f64, - pub xy: u64, - pub xy0: u64, - pub xy1: u64, - pub counts: u32, +pub(crate) struct ScanRefRecord { + pub(crate) target_id: u32, + pub(crate) invert_y: bool, + pub(crate) scale: f64, + pub(crate) angle_rad: f64, + pub(crate) xy: u64, + pub(crate) xy0: u64, + pub(crate) xy1: u64, + pub(crate) counts: u32, } enum RefRecord { @@ -117,7 +115,7 @@ fn insert_layer( } -pub fn read_elements<'a>( +pub(crate) fn read_elements<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, names: &mut HashMap, @@ -293,7 +291,7 @@ pub fn read_elements<'a>( } -pub fn scan_elements<'a>( +pub(crate) fn scan_elements<'a>( input: &'a [u8], names: &mut HashMap, ) -> IResult<'a, Vec> { @@ -583,7 +581,7 @@ fn append_boundary_prop( } -pub fn read_path<'a>( +fn read_path<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -654,7 +652,7 @@ pub fn read_path<'a>( Ok((input, ())) } -pub fn read_boxnode<'a>( +fn read_boxnode<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16, @@ -698,7 +696,7 @@ pub fn read_boxnode<'a>( Ok((input, ())) } -pub fn read_text<'a>( +fn read_text<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -967,7 +965,7 @@ fn read_ref<'a>( /// Preserves record order and duplicate property attributes. /// Stops reading after consuming ENDEL record. /// -pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { +fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; props_builder.append(true); Ok((input, ())) diff --git a/src/lib.rs b/src/lib.rs index 0abcca5..c31b907 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ pub mod basic; pub mod record; pub mod records; -pub mod elements; +mod elements; pub mod library; pub mod review; diff --git a/src/library.rs b/src/library.rs index 1ea4c6b..e455f8b 100644 --- a/src/library.rs +++ b/src/library.rs @@ -3,14 +3,11 @@ /// use std::io::Write; -//use std::collections::HashMap; -pub use crate::record; -pub use crate::record::{RecordHeader, Record}; -pub use crate::records; -pub use crate::elements; -pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -pub use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::record::{RecordHeader, Record}; +use crate::records; +use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +use crate::basic::{IResult, OResult, take_bytes, fail}; use std::string::String; use std::collections::HashMap; diff --git a/src/misc.py b/src/misc.py deleted file mode 100644 index 1e507c7..0000000 --- a/src/misc.py +++ /dev/null @@ -1,39 +0,0 @@ -''' -https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py -''' -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import ctypes -import pyarrow as pa -from pyarrow.cffi import ffi - - c_schema = ffi.new('struct ArrowSchema*') - ptr_schema = int(ffi.cast('uintptr_t', c_schema)) - c_array = ffi.new('struct ArrowArray*') - ptr_array = int(ffi.cast('uintptr_t', c_array)) - - # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) - - # import gc - # gc.collect() # Make sure no Arrow data dangles in a ref cycle - # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) - - arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) - - diff --git a/src/record.rs b/src/record.rs index cb64ea2..25d358a 100644 --- a/src/record.rs +++ b/src/record.rs @@ -4,16 +4,12 @@ use std::io::{self, Write}; -use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; use crate::records; -//#[no_mangle] -//pub extern "C" fn write_record_header( - - #[repr(C)] #[derive(Debug, Clone)] pub struct RecordHeader { @@ -53,7 +49,6 @@ pub trait RecordData { fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; fn pack_into(buf: &mut [u8], data: &Self::InData); - //fn size(data: &Self::BareData<'_>) -> u16; fn pack(data: &Self::InData) -> Self::ByteData; } @@ -73,22 +68,17 @@ pub trait Record { } } - fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { - RecordHeader::read(input) - } - fn write_header(ww: &mut W, data_size: u16) -> OResult { RecordHeader{tag: Self::tag(), data_size}.write(ww) } fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + if let Err(message) = Self::check_size(size) { + return fail(input, message); + } RData::read(input, size) } - fn pack_data(buf: &mut [u8], data: &RData::InData) { - RData::pack_into(buf, data) - } - /// /// Skip to the end of the next occurence of this record. /// @@ -116,8 +106,7 @@ pub trait Record { (input, _) = take_bytes(input, header.data_size)?; (input, header) = RecordHeader::read(input)?; } - let (input, data) = Self::read_data(input, header.data_size)?; - Ok((input, data)) + Self::read_data(input, header.data_size) } fn expect_header(input: &[u8]) -> IResult<'_, u16> { @@ -131,9 +120,7 @@ pub trait Record { fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { let (input, size) = Self::expect_header(input)?; - Self::check_size(size).unwrap(); - let (input, data) = Self::read_data(input, size)?; - Ok((input, data)) + Self::read_data(input, size) } fn write(ww: &mut W, data: &RData::InData) -> OResult { @@ -155,7 +142,9 @@ impl RecordData for BitArray { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_bitarray(input) } @@ -178,7 +167,9 @@ impl RecordData for Int2 { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_int2(input) } @@ -200,7 +191,9 @@ impl RecordData for Int4 { type ByteData = [u8; 4]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 4); + if size != 4 { + return fail(input, format!("Expected record size {}, got {size}", 4)); + } parse_int4(input) } @@ -223,8 +216,9 @@ impl RecordData for Int2Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(2) { + return fail(input, format!("Record size must be divisible by 2, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int2ArrayReader{bytes})) } @@ -267,8 +261,9 @@ impl RecordData for Int4Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(4) { + return fail(input, format!("Record size must be divisible by 4, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int4ArrayReader{bytes})) } @@ -311,7 +306,9 @@ impl RecordData for Real8 { type ByteData = [u8; 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 8); + if size != 8 { + return fail(input, format!("Expected record size {}, got {size}", 8)); + } parse_real8(input) } @@ -333,7 +330,9 @@ impl RecordData for Real8Pair { type ByteData = [u8; 2 * 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 8); + if size != 2 * 8 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 8)); + } let (input, data0) = parse_real8(input)?; let (input, data1) = parse_real8(input)?; Ok((input, (data0, data1))) @@ -350,10 +349,6 @@ impl RecordData for Real8Pair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct ASCII; @@ -385,7 +380,9 @@ impl RecordData for DateTimePair { type ByteData = [u8; 2 * 6 * 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 6 * 2); + if size != 2 * 6 * 2 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2)); + } let (input, data0) = parse_datetime(input)?; let (input, data1) = parse_datetime(input)?; Ok((input, [data0, data1])) @@ -402,10 +399,6 @@ impl RecordData for DateTimePair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct Empty; @@ -415,7 +408,9 @@ impl RecordData for Empty { type ByteData = [u8; 0]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 0); + if size != 0 { + return fail(input, format!("Expected record size {}, got {size}", 0)); + } Ok((input, ())) } @@ -425,6 +420,4 @@ impl RecordData for Empty { fn pack(_data: &Self::InData) -> Self::ByteData { [] } - //fn write(ww: &mut W, data: &Self::BareData) { - //} } diff --git a/src/records.rs b/src/records.rs index 4e86404..5481e2b 100644 --- a/src/records.rs +++ b/src/records.rs @@ -4,8 +4,6 @@ use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; -//use std::io::Write; - // record tags pub const RTAG_HEADER: u16 = 0x0002; pub const RTAG_BGNLIB: u16 = 0x0102; @@ -77,37 +75,13 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300; pub const RTAG_SPACERERROR: u16 = 0x4400; pub const RTAG_CONTACT: u16 = 0x4500; -/* -// data types -pub const DATA_TYPE_NONE: u16 = 0x00; -pub const DATA_TYPE_BIT: u16 = 0x01; -pub const DATA_TYPE_INT16: u16 = 0x02; -pub const DATA_TYPE_INT32: u16 = 0x03; -pub const DATA_TYPE_REAL32: u16 = 0x04; -pub const DATA_TYPE_REAL64: u16 = 0x05; -pub const DATA_TYPE_STR: u16 = 0x06; -pub const MAX_DATA_SIZE: usize = 8; - -/// Returns the size of the given data type in bytes. -pub fn data_size(t: u16) -> Option { - match t { - x if x == DATA_TYPE_NONE => 0, - x if x == DATA_TYPE_BIT => 2, - x if x == DATA_TYPE_INT16 => 2, - x if x == DATA_TYPE_INT32 => 4, - x if x == DATA_TYPE_REAL32 => 4, - x if x == DATA_TYPE_REAL64 => 8, - _ => 0 - } -*/ pub struct HEADER; impl Record for HEADER { fn tag() -> u16 { RTAG_HEADER } fn expected_size() -> Option { Some(2) } } -//impl Record for HEADER; pub struct BGNLIB; impl Record for BGNLIB { @@ -288,9 +262,8 @@ pub struct REFLIBS; impl Record for REFLIBS { fn tag() -> u16 { RTAG_REFLIBS } fn expected_size() -> Option { None } -} -impl REFLIBS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -303,9 +276,8 @@ pub struct FONTS; impl Record for FONTS { fn tag() -> u16 { RTAG_FONTS } fn expected_size() -> Option { None } -} -impl FONTS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -330,9 +302,8 @@ pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } fn expected_size() -> Option { None } -} -impl ATTRTABLE { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs index 0e74a5e..3b81821 100644 --- a/tests/ffi_errors.rs +++ b/tests/ffi_errors.rs @@ -31,20 +31,20 @@ fn error_message() -> String { } #[test] -fn path_errors_preserve_outputs_and_convert_parser_panics() { +fn path_errors_preserve_outputs_and_clear_after_success() { let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); assert!(error_message().contains("path pointer was null")); assert!(array.is_released()); - // A mismatched timestamp size currently makes Record::read panic. + // A mismatched timestamp size returns a parse error with a byte offset. let mut bytes = Vec::new(); common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); let file = TestFile::new("invalid-timestamps", &bytes); assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); - assert!(error_message().contains("Expected record size")); + assert!(error_message().contains("read failed at byte 10: Expected record size")); assert!(array.is_released()); let mut bytes = common::library_header(); diff --git a/tests/record_validation.rs b/tests/record_validation.rs new file mode 100644 index 0000000..d1e4fa7 --- /dev/null +++ b/tests/record_validation.rs @@ -0,0 +1,99 @@ +mod common; + +use pheon::{ + basic::ErrType, + record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, + }, + records, +}; + +#[test] +fn fixed_record_sizes_are_checked_by_every_read_entry_point() { + let payload = [0; 4]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &payload); + let mut prefixed = Vec::new(); + common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip"); + prefixed.extend_from_slice(&bytes); + + for result in [ + records::HEADER::read(&bytes), + records::HEADER::skip_and_read(&prefixed), + records::HEADER::read_data(&payload, 4), + ] { + let (remaining, reason) = result.unwrap_err(); + assert_eq!(remaining, payload); + assert!(matches!(reason, ErrType::Failed(message) + if message == "Expected record size 2, got 4")); + } + + // The underlying integer-array payload is aligned, but COLROW needs two values. + let payload = [0; 2]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_COLROW, &payload); + assert!(records::COLROW::read(&bytes).is_err()); + assert!(records::COLROW::skip_and_read(&bytes).is_err()); + assert!(records::COLROW::read_data(&payload, 2).is_err()); +} + +#[test] +fn metadata_size_constraints_apply_to_every_read_entry_point() { + macro_rules! check { + ($record:ident, $tag:ident) => {{ + let payload = [b'x'; 44]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + let mut prefixed = common::library_header(); + prefixed.extend_from_slice(&bytes); + assert_eq!(records::$record::read(&bytes).unwrap().1, payload); + assert_eq!( + records::$record::skip_and_read(&prefixed).unwrap().1, + payload + ); + assert_eq!( + records::$record::read_data(&payload, 44).unwrap().1, + payload + ); + + let payload = [b'x'; 46]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + assert!(records::$record::read(&bytes).is_err()); + assert!(records::$record::skip_and_read(&bytes).is_err()); + assert!(records::$record::read_data(&payload, 46).is_err()); + }}; + } + + check!(REFLIBS, RTAG_REFLIBS); + check!(FONTS, RTAG_FONTS); + check!(ATTRTABLE, RTAG_ATTRTABLE); +} + +#[test] +fn malformed_payload_sizes_return_errors_without_panicking() { + let payload = [0; 32]; + assert!(BitArray::read(&payload, 4).is_err()); + assert!(Int2::read(&payload, 4).is_err()); + assert!(Int4::read(&payload, 2).is_err()); + assert!(Real8::read(&payload, 4).is_err()); + assert!(Real8Pair::read(&payload, 8).is_err()); + assert!(DateTimePair::read(&payload, 12).is_err()); + assert!(Empty::read(&payload, 2).is_err()); + assert!(Int2Array::read(&payload, 3).is_err()); + assert!(Int4Array::read(&payload, 6).is_err()); + + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_XY, &payload[..6]); + assert!(records::XY::read(&bytes).is_err()); + assert!(records::XY::skip_and_read(&bytes).is_err()); + assert!(records::XY::read_data(&payload, 6).is_err()); + + // Valid sizes still reject a truncated buffer; an empty string remains valid. + assert!(Int4::read(&payload[..2], 4).is_err()); + assert!(Real8Pair::read(&payload[..8], 16).is_err()); + assert!(DateTimePair::read(&payload[..12], 24).is_err()); + assert!(Int4Array::read(&payload[..4], 8).is_err()); + assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::::new())); +}