Reject invalid record sizes and return malformed-input errors
Validate payload sizes before record/header writes and make header packing return io::Result when the total size overflows the format limit. Return located parse errors for invalid UTF-8 and timestamp year overflow. Validate cell timestamps consistently in full reads, selected reads, and hierarchy scans while retaining scan behavior for skipped text geometry. Add regression coverage for preserved writer outputs, padded metadata sizes, malformed strings, timestamp boundaries, bounded errors, and Python bindings. Document the fallible header packing API. Validation: all 97 Rust tests pass in debug and release; all 16 Python tests pass against the rebuilt native library.
This commit is contained in:
parent
df62677f5c
commit
31afef0529
4 changed files with 307 additions and 28 deletions
155
tests/parser_errors.rs
Normal file
155
tests/parser_errors.rs
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
mod common;
|
||||
|
||||
use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
|
||||
use pheon::{
|
||||
basic::ErrType,
|
||||
last_error_message,
|
||||
library::{read_library, read_selected_library, scan_library},
|
||||
read_bytes, read_cells_bytes,
|
||||
record::RecordHeader,
|
||||
records, scan_bytes,
|
||||
};
|
||||
use std::ffi::CStr;
|
||||
|
||||
fn fixture() -> (Vec<u8>, (u64, u64)) {
|
||||
let mut bytes = common::library_header();
|
||||
let start = bytes.len() as u64;
|
||||
common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]);
|
||||
common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes());
|
||||
common::record(&mut bytes, records::RTAG_SREF, &[]);
|
||||
common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes());
|
||||
common::record(&mut bytes, records::RTAG_XY, &[0; 8]);
|
||||
common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes());
|
||||
common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0");
|
||||
common::record(&mut bytes, records::RTAG_ENDEL, &[]);
|
||||
common::record(&mut bytes, records::RTAG_TEXT, &[]);
|
||||
common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]);
|
||||
common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]);
|
||||
common::record(&mut bytes, records::RTAG_XY, &[0; 8]);
|
||||
common::record(&mut bytes, records::RTAG_STRING, b"label\0");
|
||||
common::record(&mut bytes, records::RTAG_ENDEL, &[]);
|
||||
common::record(&mut bytes, records::RTAG_ENDSTR, &[]);
|
||||
let end = bytes.len() as u64;
|
||||
common::record(&mut bytes, records::RTAG_ENDLIB, &[]);
|
||||
(bytes, (start, end))
|
||||
}
|
||||
|
||||
fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range<usize> {
|
||||
let mut offset = 0;
|
||||
loop {
|
||||
let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap();
|
||||
let start = offset + 4;
|
||||
let end = start + usize::from(header.data_size);
|
||||
if header.tag == tag {
|
||||
return start..end;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
|
||||
fn check_ffi_error(
|
||||
expected: &str,
|
||||
call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32,
|
||||
) {
|
||||
let mut array = FFI_ArrowArray::empty();
|
||||
let mut schema = FFI_ArrowSchema::empty();
|
||||
assert_eq!(call(&mut array, &mut schema), -1);
|
||||
let pointer = last_error_message();
|
||||
assert!(!pointer.is_null());
|
||||
let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap();
|
||||
assert!(error.contains(expected), "{error}");
|
||||
assert!(
|
||||
error.len() < 256,
|
||||
"error should not dump the input: {error}"
|
||||
);
|
||||
assert!(array.is_released());
|
||||
}
|
||||
|
||||
fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) {
|
||||
let ranges = [range];
|
||||
let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)];
|
||||
if scan {
|
||||
results.push(scan_library(bytes));
|
||||
} else {
|
||||
// Hierarchy scans deliberately skip text geometry and its string payloads.
|
||||
assert!(scan_library(bytes).is_ok());
|
||||
}
|
||||
for result in results {
|
||||
let (remaining, error) = result.unwrap_err();
|
||||
assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr());
|
||||
assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message)));
|
||||
}
|
||||
let expected = format!("at byte {offset}: {message}");
|
||||
check_ffi_error(&expected, |a, s| unsafe {
|
||||
read_bytes(bytes.as_ptr(), bytes.len(), a, s)
|
||||
});
|
||||
let offsets = [range.0, range.1];
|
||||
check_ffi_error(&expected, |a, s| unsafe {
|
||||
read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s)
|
||||
});
|
||||
if scan {
|
||||
check_ffi_error(&expected, |a, s| unsafe {
|
||||
scan_bytes(bytes.as_ptr(), bytes.len(), a, s)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_utf8_returns_located_parse_errors_in_each_reader() {
|
||||
let (bytes, range) = fixture();
|
||||
assert!(read_library(&bytes).is_ok());
|
||||
assert!(read_selected_library(&bytes, &[range]).is_ok());
|
||||
assert!(scan_library(&bytes).is_ok());
|
||||
for tag in [
|
||||
records::RTAG_LIBNAME,
|
||||
records::RTAG_STRNAME,
|
||||
records::RTAG_SNAME,
|
||||
records::RTAG_STRING,
|
||||
records::RTAG_PROPVALUE,
|
||||
] {
|
||||
let mut malformed = bytes.clone();
|
||||
let payload = payload_range(&malformed, tag);
|
||||
malformed[payload.start] = 0xff;
|
||||
// UTF-8 is checked at the current parser cursor, after the record/header.
|
||||
let offset = if tag == records::RTAG_LIBNAME {
|
||||
payload_range(&malformed, records::RTAG_UNITS).end
|
||||
} else {
|
||||
payload.end
|
||||
};
|
||||
check_read_errors(
|
||||
&malformed,
|
||||
range,
|
||||
offset,
|
||||
"Invalid UTF-8 string:",
|
||||
tag != records::RTAG_STRING,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_invalid_utf8_strings_do_not_dump_their_contents() {
|
||||
let (mut bytes, range) = fixture();
|
||||
let payload = payload_range(&bytes, records::RTAG_PROPVALUE);
|
||||
let mut replacement = Vec::new();
|
||||
let mut value = vec![b'a'; 60_000];
|
||||
value[30_000] = 0xff;
|
||||
common::record(&mut replacement, records::RTAG_PROPVALUE, &value);
|
||||
let added = replacement.len() - (payload.end - payload.start + 4);
|
||||
bytes.splice(payload.start - 4..payload.end, replacement);
|
||||
let range = (range.0, range.1 + added as u64);
|
||||
let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end;
|
||||
check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() {
|
||||
for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] {
|
||||
for timestamp in [0, 1] {
|
||||
let (mut bytes, range) = fixture();
|
||||
let payload = payload_range(&bytes, tag);
|
||||
let offset = payload.start + timestamp * 12;
|
||||
bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes());
|
||||
check_read_errors(&bytes, range, offset, "Timestamp year", true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -85,6 +85,31 @@ class ArrowTests(unittest.TestCase):
|
|||
with self.assertRaises(TypeError):
|
||||
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
|
||||
|
||||
def test_malformed_strings_and_years_raise_located_parse_errors(self):
|
||||
data = layout()
|
||||
cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0]
|
||||
start = cell["struct_start_offset"]
|
||||
ranges = [[start, cell["struct_end_offset"]]]
|
||||
cases = [
|
||||
(38, b"\xff", "Invalid UTF-8"),
|
||||
(start + 32, b"\xff", "Invalid UTF-8"),
|
||||
]
|
||||
cases += [
|
||||
(offset, b"\x7f\xff", "Timestamp year")
|
||||
for offset in (10, 22, start + 4, start + 16)
|
||||
]
|
||||
for offset, value, message in cases:
|
||||
malformed = bytearray(data)
|
||||
malformed[offset:offset + len(value)] = value
|
||||
for read in (
|
||||
arrow.read_bytes,
|
||||
arrow.scan_bytes,
|
||||
lambda payload: arrow.read_cells_bytes(payload, ranges),
|
||||
):
|
||||
with self.subTest(offset=offset, reader=read):
|
||||
with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"):
|
||||
read(malformed)
|
||||
|
||||
def test_import_failure_releases_both_native_outputs(self):
|
||||
releases = []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
mod common;
|
||||
|
||||
use pheon::{
|
||||
basic::ErrType,
|
||||
basic::{ErrType, parse_datetime},
|
||||
record::{
|
||||
ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair,
|
||||
Record, RecordData,
|
||||
|
|
@ -38,6 +38,29 @@ fn fixed_record_sizes_are_checked_by_every_read_entry_point() {
|
|||
assert!(records::COLROW::read_data(&payload, 2).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_year_conversion_checks_the_int16_boundaries() {
|
||||
for stored_year in [i16::MIN, -1900, 0, i16::MAX - 1900] {
|
||||
let mut payload = [0; 12];
|
||||
payload[..2].copy_from_slice(&stored_year.to_be_bytes());
|
||||
let (remaining, date) = parse_datetime(&payload).unwrap();
|
||||
assert!(remaining.is_empty());
|
||||
assert_eq!(i32::from(date[0]), i32::from(stored_year) + 1900);
|
||||
}
|
||||
for stored_year in [i16::MAX - 1899, i16::MAX] {
|
||||
for timestamp in [0, 1] {
|
||||
let mut payload = [0; 24];
|
||||
let offset = timestamp * 12;
|
||||
payload[offset..offset + 2].copy_from_slice(&stored_year.to_be_bytes());
|
||||
let (remaining, error) = DateTimePair::read(&payload, 24).unwrap_err();
|
||||
assert_eq!(remaining.as_ptr(), payload[offset..].as_ptr());
|
||||
assert!(
|
||||
matches!(error, ErrType::Failed(message) if message.contains("Timestamp year"))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_size_constraints_apply_to_every_read_entry_point() {
|
||||
macro_rules! check {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ fn record_headers_include_their_own_size() {
|
|||
data_size: u16::MAX - 4,
|
||||
tag: 0x1003,
|
||||
};
|
||||
assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]);
|
||||
assert_eq!(maximum.pack_into().unwrap(), [0xff, 0xff, 0x10, 0x03]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -135,4 +135,58 @@ fn oversized_records_fail_before_writing() {
|
|||
let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||
assert!(bytes.is_empty());
|
||||
|
||||
// Header packing and direct header writes reject every overflowing size too.
|
||||
for data_size in u16::MAX - 3..=u16::MAX {
|
||||
let header = RecordHeader {
|
||||
tag: records::RTAG_XY,
|
||||
data_size,
|
||||
};
|
||||
assert_eq!(
|
||||
header.pack_into().unwrap_err().kind(),
|
||||
io::ErrorKind::InvalidInput
|
||||
);
|
||||
assert_eq!(
|
||||
header.write(&mut bytes).unwrap_err().kind(),
|
||||
io::ErrorKind::InvalidInput
|
||||
);
|
||||
assert!(bytes.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_record_sizes_preserve_the_output() {
|
||||
let mut bytes = vec![0xa5];
|
||||
for count in [0, 1, 3] {
|
||||
let error = records::COLROW::write(&mut bytes, &vec![2; count]).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||
assert_eq!(bytes, [0xa5]);
|
||||
}
|
||||
let error = records::COLROW::write_header(&mut bytes, 2).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||
assert_eq!(bytes, [0xa5]);
|
||||
|
||||
macro_rules! check_metadata {
|
||||
($record:ident) => {
|
||||
for count in [43, 44, 45] {
|
||||
let payload = vec![b'x'; count];
|
||||
let mut bytes = vec![0xa5];
|
||||
let result = records::$record::write(&mut bytes, &payload);
|
||||
if count == 45 {
|
||||
assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput);
|
||||
assert_eq!(bytes, [0xa5]);
|
||||
} else {
|
||||
// Validation uses the padded record size, not the input length.
|
||||
assert_eq!(result.unwrap(), 48);
|
||||
assert_eq!(records::$record::read(&bytes[1..]).unwrap().1, payload);
|
||||
}
|
||||
}
|
||||
let error = records::$record::write_header(&mut bytes, 46).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||
assert_eq!(bytes, [0xa5]);
|
||||
};
|
||||
}
|
||||
check_metadata!(REFLIBS);
|
||||
check_metadata!(FONTS);
|
||||
check_metadata!(ATTRTABLE);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue