diff --git a/docs/native-api.md b/docs/native-api.md index ad64a05..5e9f9ab 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -36,6 +36,11 @@ writing. There is no high-level writer for the Arrow layout. REAL8 writing is exact for representable in-range `f64` values: magnitudes below `2^-260` underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected. +Record writes check payload sizes before emitting bytes. Header packing returns +an `io::Result`, including an error when the total size exceeds the format limit. +Arrow readers return parse errors for invalid UTF-8 strings and timestamp years +that exceed the Int16 range after conversion from the 1900 epoch. + ## C API Build the shared library with `cargo build --release --locked`. The diff --git a/src/basic.rs b/src/basic.rs index 1189b8e..4bda079 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -79,12 +79,23 @@ pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { let mut buf = [0_i16; 6]; - let mut input = input; - for bb in &mut buf { - (input, *bb) = parse_int2(input)?; + let (mut remaining, year) = parse_int2(input)?; + let Some(year) = year.checked_add(1900) else { + return fail( + input, + "Timestamp year exceeds the Int16 range after adding 1900".to_string(), + ); + }; + buf[0] = year; + for bb in &mut buf[1..] { + (remaining, *bb) = parse_int2(remaining)?; } - buf[0] += 1900; // Year is from 1900 - Ok((input, buf)) + Ok((remaining, buf)) +} + +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + String::from_utf8(bytes) + .map_err(|error| ErrType::Failed(format!("Invalid UTF-8 string: {}", error.utf8_error()))) } diff --git a/src/elements.rs b/src/elements.rs index 02a8112..32fd9c8 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -12,7 +12,7 @@ use crate::records::{ use crate::records; use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; +use crate::basic::{IResult, decode_utf8, fail, take_bytes}; use std::string::String; use std::collections::HashMap; @@ -784,7 +784,7 @@ fn read_text<'a>( xy_builder.append_value(xy); let (input, string_bytes) = STRING::read(input)?; - let string = String::from_utf8(string_bytes).unwrap(); + let string = decode_utf8(string_bytes).map_err(|error| (input, error))?; let string_builder = text_builder.field_builder::(10).unwrap(); string_builder.append_value(string); @@ -856,7 +856,7 @@ fn read_ref<'a>( let mut colrow = None; let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; - let struct_name = String::from_utf8(struct_name_bytes).unwrap(); + let struct_name = decode_utf8(struct_name_bytes).map_err(|error| (input, error))?; let next_id = names.len(); let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); @@ -986,7 +986,7 @@ fn walk_properties<'a>( on_property(PropertyRecord { key, - value: String::from_utf8(value_bytes).unwrap(), + value: decode_utf8(value_bytes).map_err(|error| (input, error))?, }); } (input, header) = RecordHeader::read(input)?; diff --git a/src/library.rs b/src/library.rs index e455f8b..d4db8f2 100644 --- a/src/library.rs +++ b/src/library.rs @@ -7,7 +7,7 @@ use std::io::Write; use crate::record::{RecordHeader, Record}; use crate::records; use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::basic::{ErrType, IResult, OResult, decode_utf8, take_bytes, fail}; use std::string::String; use std::collections::HashMap; @@ -324,13 +324,14 @@ fn make_scan_builder() -> StructBuilder { fn append_file_header( lib_builder: &mut StructBuilder, header: &FileHeader, - ) { + ) -> Result<(), ErrType> { + let name = decode_utf8(header.name.clone())?; let dbu_builder = lib_builder.field_builder::(0).unwrap(); dbu_builder.append_value(header.meters_per_db_unit); let uu_builder = lib_builder.field_builder::(1).unwrap(); uu_builder.append_value(header.user_units_per_db_unit); let libname_builder = lib_builder.field_builder::(2).unwrap(); - libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); + libname_builder.append_value(name); let mtl_builder = lib_builder.field_builder::(3).unwrap(); let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); mt_builder.append_values(&header.mod_time, &[true; 6]); @@ -339,6 +340,7 @@ fn append_file_header( let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); at_builder.append_values(&header.acc_time, &[true; 6]); atl_builder.append(true); + Ok(()) } @@ -441,14 +443,16 @@ where let (mut input, mut header) = RecordHeader::read(input)?; while header.tag != records::RTAG_ENDLIB { let record_start = base_offset + input_size - input.len() - 4; - (input, _) = take_bytes(input, header.data_size)?; if header.tag == records::RTAG_BGNSTR { + (input, _) = records::BGNSTR::read_data(input, header.data_size)?; let name_bytes; (input, name_bytes) = records::STRNAME::read(input)?; - let name = String::from_utf8(name_bytes).unwrap(); + let name = decode_utf8(name_bytes).map_err(|error| (input, error))?; let id = intern_name(names, name); let (next_input, ()) = on_struct(input, record_start, id, names)?; input = next_input; + } else { + (input, _) = take_bytes(input, header.data_size)?; } (input, header) = RecordHeader::read(input)?; } @@ -461,7 +465,7 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_library_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -497,7 +501,7 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_scan_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -541,8 +545,8 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { /// as its remainder; it does not advance a sequential parsing cursor. pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { let mut lib_builder = make_library_builder(); - let (_, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + let (header_end, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (header_end, error))?; let mut names = HashMap::::new(); let mut layers = HashMap::::new(); @@ -558,7 +562,7 @@ pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IRes let struct_input = &input[start..end]; let (struct_input, _) = records::BGNSTR::read(struct_input)?; let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; - let name = String::from_utf8(name_bytes).unwrap(); + let name = decode_utf8(name_bytes).map_err(|error| (struct_input, error))?; let file_offset = end - struct_input.len(); let id = intern_name(&mut names, name); diff --git a/src/record.rs b/src/record.rs index 25d358a..6de51d1 100644 --- a/src/record.rs +++ b/src/record.rs @@ -27,15 +27,16 @@ impl RecordHeader { Ok((input, RecordHeader{tag, data_size})) } - pub fn pack_into(&self) -> [u8; 4] { - let size = self.data_size.checked_add(4).expect("Record too big!"); + pub fn pack_into(&self) -> io::Result<[u8; 4]> { + let size = self.data_size.checked_add(4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; let [size_hi, size_lo] = size.to_be_bytes(); let [tag_hi, tag_lo] = self.tag.to_be_bytes(); - [size_hi, size_lo, tag_hi, tag_lo] + Ok([size_hi, size_lo, tag_hi, tag_lo]) } pub fn write(&self, ww: &mut W) -> OResult { - let bytes = self.pack_into(); + let bytes = self.pack_into()?; ww.write_all(&bytes)?; Ok(bytes.len()) } @@ -69,6 +70,8 @@ pub trait Record { } fn write_header(ww: &mut W, data_size: u16) -> OResult { + Self::check_size(data_size) + .map_err(|message| io::Error::new(io::ErrorKind::InvalidInput, message))?; RecordHeader{tag: Self::tag(), data_size}.write(ww) } @@ -126,9 +129,8 @@ pub trait Record { fn write(ww: &mut W, data: &RData::InData) -> OResult { let packed_data = RData::pack(data); let data_bytes = packed_data.as_ref(); - let len = u16::try_from(data_bytes.len()).ok() - .filter(|len| *len <= u16::MAX - 4) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + let len = u16::try_from(data_bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; Self::write_header(ww, len)?; ww.write_all(data_bytes)?; Ok(usize::from(len) + 4) diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs new file mode 100644 index 0000000..eca1116 --- /dev/null +++ b/tests/parser_errors.rs @@ -0,0 +1,155 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{ + basic::ErrType, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_bytes, read_cells_bytes, + record::RecordHeader, + records, scan_bytes, +}; +use std::ffi::CStr; + +fn fixture() -> (Vec, (u64, u64)) { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_SREF, &[]); + common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_TEXT, &[]); + common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]); + common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_STRING, b"label\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let end = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + (bytes, (start, end)) +} + +fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range { + let mut offset = 0; + loop { + let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap(); + let start = offset + 4; + let end = start + usize::from(header.data_size); + if header.tag == tag { + return start..end; + } + offset = end; + } +} + +fn check_ffi_error( + expected: &str, + call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32, +) { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(call(&mut array, &mut schema), -1); + let pointer = last_error_message(); + assert!(!pointer.is_null()); + let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); + assert!(error.contains(expected), "{error}"); + assert!( + error.len() < 256, + "error should not dump the input: {error}" + ); + assert!(array.is_released()); +} + +fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) { + let ranges = [range]; + let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)]; + if scan { + results.push(scan_library(bytes)); + } else { + // Hierarchy scans deliberately skip text geometry and its string payloads. + assert!(scan_library(bytes).is_ok()); + } + for result in results { + let (remaining, error) = result.unwrap_err(); + assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr()); + assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message))); + } + let expected = format!("at byte {offset}: {message}"); + check_ffi_error(&expected, |a, s| unsafe { + read_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + let offsets = [range.0, range.1]; + check_ffi_error(&expected, |a, s| unsafe { + read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s) + }); + if scan { + check_ffi_error(&expected, |a, s| unsafe { + scan_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + } +} + +#[test] +fn invalid_utf8_returns_located_parse_errors_in_each_reader() { + let (bytes, range) = fixture(); + assert!(read_library(&bytes).is_ok()); + assert!(read_selected_library(&bytes, &[range]).is_ok()); + assert!(scan_library(&bytes).is_ok()); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + records::RTAG_STRING, + records::RTAG_PROPVALUE, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0xff; + // UTF-8 is checked at the current parser cursor, after the record/header. + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + check_read_errors( + &malformed, + range, + offset, + "Invalid UTF-8 string:", + tag != records::RTAG_STRING, + ); + } +} + +#[test] +fn large_invalid_utf8_strings_do_not_dump_their_contents() { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, records::RTAG_PROPVALUE); + let mut replacement = Vec::new(); + let mut value = vec![b'a'; 60_000]; + value[30_000] = 0xff; + common::record(&mut replacement, records::RTAG_PROPVALUE, &value); + let added = replacement.len() - (payload.end - payload.start + 4); + bytes.splice(payload.start - 4..payload.end, replacement); + let range = (range.0, range.1 + added as u64); + let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end; + check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true); +} + +#[test] +fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() { + for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] { + for timestamp in [0, 1] { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, tag); + let offset = payload.start + timestamp * 12; + bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes()); + check_read_errors(&bytes, range, offset, "Timestamp year", true); + } + } +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py index ed962e2..64db3f2 100644 --- a/tests/python/test_arrow.py +++ b/tests/python/test_arrow.py @@ -85,6 +85,31 @@ class ArrowTests(unittest.TestCase): with self.assertRaises(TypeError): arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + def test_malformed_strings_and_years_raise_located_parse_errors(self): + data = layout() + cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0] + start = cell["struct_start_offset"] + ranges = [[start, cell["struct_end_offset"]]] + cases = [ + (38, b"\xff", "Invalid UTF-8"), + (start + 32, b"\xff", "Invalid UTF-8"), + ] + cases += [ + (offset, b"\x7f\xff", "Timestamp year") + for offset in (10, 22, start + 4, start + 16) + ] + for offset, value, message in cases: + malformed = bytearray(data) + malformed[offset:offset + len(value)] = value + for read in ( + arrow.read_bytes, + arrow.scan_bytes, + lambda payload: arrow.read_cells_bytes(payload, ranges), + ): + with self.subTest(offset=offset, reader=read): + with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"): + read(malformed) + def test_import_failure_releases_both_native_outputs(self): releases = [] diff --git a/tests/record_validation.rs b/tests/record_validation.rs index d1e4fa7..5fd55d3 100644 --- a/tests/record_validation.rs +++ b/tests/record_validation.rs @@ -1,7 +1,7 @@ mod common; use pheon::{ - basic::ErrType, + basic::{ErrType, parse_datetime}, record::{ ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, Record, RecordData, @@ -38,6 +38,29 @@ fn fixed_record_sizes_are_checked_by_every_read_entry_point() { assert!(records::COLROW::read_data(&payload, 2).is_err()); } +#[test] +fn timestamp_year_conversion_checks_the_int16_boundaries() { + for stored_year in [i16::MIN, -1900, 0, i16::MAX - 1900] { + let mut payload = [0; 12]; + payload[..2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, date) = parse_datetime(&payload).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(i32::from(date[0]), i32::from(stored_year) + 1900); + } + for stored_year in [i16::MAX - 1899, i16::MAX] { + for timestamp in [0, 1] { + let mut payload = [0; 24]; + let offset = timestamp * 12; + payload[offset..offset + 2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, error) = DateTimePair::read(&payload, 24).unwrap_err(); + assert_eq!(remaining.as_ptr(), payload[offset..].as_ptr()); + assert!( + matches!(error, ErrType::Failed(message) if message.contains("Timestamp year")) + ); + } + } +} + #[test] fn metadata_size_constraints_apply_to_every_read_entry_point() { macro_rules! check { diff --git a/tests/writing.rs b/tests/writing.rs index caf8816..dc07994 100644 --- a/tests/writing.rs +++ b/tests/writing.rs @@ -19,7 +19,7 @@ fn record_headers_include_their_own_size() { data_size: u16::MAX - 4, tag: 0x1003, }; - assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]); + assert_eq!(maximum.pack_into().unwrap(), [0xff, 0xff, 0x10, 0x03]); } #[test] @@ -135,4 +135,58 @@ fn oversized_records_fail_before_writing() { let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); assert!(bytes.is_empty()); + + // Header packing and direct header writes reject every overflowing size too. + for data_size in u16::MAX - 3..=u16::MAX { + let header = RecordHeader { + tag: records::RTAG_XY, + data_size, + }; + assert_eq!( + header.pack_into().unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + header.write(&mut bytes).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(bytes.is_empty()); + } +} + +#[test] +fn invalid_record_sizes_preserve_the_output() { + let mut bytes = vec![0xa5]; + for count in [0, 1, 3] { + let error = records::COLROW::write(&mut bytes, &vec![2; count]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } + let error = records::COLROW::write_header(&mut bytes, 2).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + + macro_rules! check_metadata { + ($record:ident) => { + for count in [43, 44, 45] { + let payload = vec![b'x'; count]; + let mut bytes = vec![0xa5]; + let result = records::$record::write(&mut bytes, &payload); + if count == 45 { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } else { + // Validation uses the padded record size, not the input length. + assert_eq!(result.unwrap(), 48); + assert_eq!(records::$record::read(&bytes[1..]).unwrap().1, payload); + } + } + let error = records::$record::write_header(&mut bytes, 46).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + }; + } + check_metadata!(REFLIBS); + check_metadata!(FONTS); + check_metadata!(ATTRTABLE); }