mod common; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use pheon::{ basic::{ErrType, encode_real8}, last_error_message, library::{read_library, read_selected_library, scan_library}, read_bytes, read_cells_bytes, record::RecordHeader, records, review::{Comparison, Document, dump_records}, scan_bytes, }; use std::ffi::CStr; fn fixture() -> (Vec, (u64, u64)) { let mut bytes = common::library_header(); let start = bytes.len() as u64; common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes()); common::record(&mut bytes, records::RTAG_SREF, &[]); common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes()); common::record(&mut bytes, records::RTAG_XY, &[0; 8]); common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); common::record(&mut bytes, records::RTAG_ENDEL, &[]); common::record(&mut bytes, records::RTAG_TEXT, &[]); common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]); common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]); common::record(&mut bytes, records::RTAG_XY, &[0; 8]); common::record(&mut bytes, records::RTAG_STRING, b"label\0"); common::record(&mut bytes, records::RTAG_ENDEL, &[]); common::record(&mut bytes, records::RTAG_ENDSTR, &[]); let end = bytes.len() as u64; common::record(&mut bytes, records::RTAG_ENDLIB, &[]); (bytes, (start, end)) } fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range { let mut offset = 0; loop { let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap(); let start = offset + 4; let end = start + usize::from(header.data_size); if header.tag == tag { return start..end; } offset = end; } } fn check_ffi_error( expected: &str, call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32, ) { let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!(call(&mut array, &mut schema), -1); let pointer = last_error_message(); assert!(!pointer.is_null()); let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); assert!(error.contains(expected), "{error}"); assert!( error.len() < 256, "error should not dump the input: {error}" ); assert!(array.is_released()); } fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) { let ranges = [range]; let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)]; if scan { results.push(scan_library(bytes)); } else { // Hierarchy scans deliberately skip text geometry and its string payloads. assert!(scan_library(bytes).is_ok()); } for result in results { let (remaining, error) = result.unwrap_err(); assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr()); assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message))); } let expected = format!("at byte {offset}: {message}"); check_ffi_error(&expected, |a, s| unsafe { read_bytes(bytes.as_ptr(), bytes.len(), a, s) }); let offsets = [range.0, range.1]; check_ffi_error(&expected, |a, s| unsafe { read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s) }); if scan { check_ffi_error(&expected, |a, s| unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), a, s) }); } } #[test] fn invalid_utf8_returns_located_parse_errors_in_each_reader() { let (bytes, range) = fixture(); assert!(read_library(&bytes).is_ok()); assert!(read_selected_library(&bytes, &[range]).is_ok()); assert!(scan_library(&bytes).is_ok()); for tag in [ records::RTAG_LIBNAME, records::RTAG_STRNAME, records::RTAG_SNAME, records::RTAG_STRING, records::RTAG_PROPVALUE, ] { let mut malformed = bytes.clone(); let payload = payload_range(&malformed, tag); malformed[payload.start] = 0xff; // UTF-8 is checked at the current parser cursor, after the record/header. let offset = if tag == records::RTAG_LIBNAME { payload_range(&malformed, records::RTAG_UNITS).end } else { payload.end }; check_read_errors( &malformed, range, offset, "Invalid UTF-8 string:", tag != records::RTAG_STRING, ); } } #[test] fn extra_name_padding_is_ignored_by_comparison_and_arrow_readers() { let (mut bytes, (start, end)) = fixture(); let target = payload_range(&bytes, records::RTAG_SNAME); bytes[target].copy_from_slice(b"leaf"); bytes.truncate(end as usize); common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); common::record(&mut bytes, records::RTAG_STRNAME, b"leaf"); common::record(&mut bytes, records::RTAG_ENDSTR, &[]); common::record(&mut bytes, records::RTAG_ENDLIB, &[]); let mut padded = bytes.clone(); for tag in [ records::RTAG_LIBNAME, records::RTAG_STRNAME, records::RTAG_SNAME, ] { let payload = payload_range(&padded, tag); let mut data = padded[payload.clone()].to_vec(); data.extend_from_slice(&[0; 2]); let mut record = Vec::new(); common::record(&mut record, tag, &data); padded.splice(payload.start - 4..payload.end, record); } let baseline = Document::parse(&bytes).unwrap(); let document = Document::parse(&padded).unwrap(); assert_eq!(document.cells["töp"].objects.len(), 2); assert!( Comparison::between(&baseline, &document) .outcome() .is_equal() ); let full = read_library(&padded).unwrap().1; let scan = scan_library(&padded).unwrap().1; let ranges = [(start + 2, end + 6), (end + 6, padded.len() as u64 - 4)]; let selected = read_selected_library(&padded, &ranges).unwrap().1; let expected = read_library(&bytes).unwrap().1; for actual in [&full, &scan, &selected] { for field in ["lib_name", "cell_names"] { assert_eq!(actual.column_by_name(field), expected.column_by_name(field)); } } assert_eq!(&full, &selected); } #[test] fn interior_nuls_in_names_are_rejected_by_comparison_and_arrow_readers() { let (bytes, range) = fixture(); for tag in [ records::RTAG_LIBNAME, records::RTAG_STRNAME, records::RTAG_SNAME, ] { let mut malformed = bytes.clone(); let payload = payload_range(&malformed, tag); malformed[payload.start] = 0; let offset = if tag == records::RTAG_LIBNAME { payload_range(&malformed, records::RTAG_UNITS).end } else { payload.end }; let message = "name contains an embedded NUL"; check_read_errors(&malformed, range, offset, message, true); assert_eq!( Document::parse(&malformed).unwrap_err(), format!("at byte {}: {message}", payload.start - 4) ); } } #[test] fn malformed_record_frames_are_rejected_at_the_same_offset() { let (bytes, range) = fixture(); // STRING also exercises framing in the Arrow scan's geometry skip path. for tag in [ records::RTAG_STRNAME, records::RTAG_XY, records::RTAG_STRING, ] { let start = payload_range(&bytes, tag).start - 4; for (size, message) in [ (2_u16, "Record size 2 is smaller than its header"), (5, "odd record length"), (u16::MAX - 1, "truncated record"), ] { let mut malformed = bytes.clone(); malformed[start..start + 2].copy_from_slice(&size.to_be_bytes()); check_read_errors(&malformed, range, start, message, true); assert_eq!( dump_records(&malformed).unwrap_err(), format!("at byte {start}: {message}") ); } } } #[test] fn nonpositive_unit_scales_are_rejected_by_comparison_and_arrow_readers() { let (bytes, range) = fixture(); let units = payload_range(&bytes, records::RTAG_UNITS); let message = "UNITS must be positive finite values"; for field in [units.start, units.start + 8] { for scale in [0.0, -1.0] { let mut malformed = bytes.clone(); malformed[field..field + 8] .copy_from_slice(&encode_real8(scale).unwrap().to_be_bytes()); check_read_errors(&malformed, range, units.start, message, true); assert_eq!( Document::parse(&malformed).unwrap_err(), format!("at byte {}: {message}", units.start - 4) ); } } } #[test] fn coordinate_payloads_require_whole_pairs() { let (mut bytes, (start, end)) = fixture(); let after_text = payload_range(&bytes, records::RTAG_TEXT).end; let xy = payload_range(&bytes[after_text..], records::RTAG_XY); let payload_start = after_text + xy.start; let mut replacement = Vec::new(); common::record(&mut replacement, records::RTAG_XY, &[0; 12]); bytes.splice(payload_start - 4..after_text + xy.end, replacement); let message = "XY payload must contain whole coordinate pairs"; check_read_errors(&bytes, (start, end + 4), payload_start, message, false); assert!(Document::parse(&bytes).unwrap_err().contains(message)); } #[test] fn large_invalid_utf8_strings_do_not_dump_their_contents() { let (mut bytes, range) = fixture(); let payload = payload_range(&bytes, records::RTAG_PROPVALUE); let mut replacement = Vec::new(); let mut value = vec![b'a'; 60_000]; value[30_000] = 0xff; common::record(&mut replacement, records::RTAG_PROPVALUE, &value); let added = replacement.len() - (payload.end - payload.start + 4); bytes.splice(payload.start - 4..payload.end, replacement); let range = (range.0, range.1 + added as u64); let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end; check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true); } #[test] fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() { for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] { for timestamp in [0, 1] { let (mut bytes, range) = fixture(); let payload = payload_range(&bytes, tag); let offset = payload.start + timestamp * 12; bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes()); check_read_errors(&bytes, range, offset, "Timestamp year", true); } } }