fix(writer): serialize complete records with correctly sized buffers

Include record headers in wire lengths, allocate initialized array and padded string buffers, and handle empty strings. Write all bytes to short writers and reject oversized records before writing.
This commit is contained in:
Jan Petykiewicz 2026-11-14 14:37:35 -08:00
commit 3783320f82
3 changed files with 106 additions and 15 deletions

View file

@ -98,8 +98,7 @@ pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> {
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec<u8>> {
let length = length as usize;
let (input, data) = take_bytes(input, length)?;
let last = data[length - 1];
let true_length = if last == 0 { length - 1 } else { length };
let true_length = length - usize::from(data.last() == Some(&0));
let vec = data[..true_length].to_vec();
Ok((input, vec))
}

View file

@ -1,8 +1,7 @@
///
/// Generic record-level read/write functionality.
///
use std::io::Write;
use std::convert::TryInto;
use std::io::{self, Write};
use byteorder::{ByteOrder, BigEndian};
@ -34,8 +33,8 @@ impl RecordHeader {
}
pub fn pack_into(&self) -> [u8; 4] {
assert!(self.data_size < 0xffff - 4, "Record too big!");
let vals = [self.data_size, self.tag];
let size = self.data_size.checked_add(4).expect("Record too big!");
let vals = [size, self.tag];
let mut buf = [0x77; 4];
BigEndian::write_u16_into(&vals, &mut buf);
buf
@ -43,7 +42,8 @@ impl RecordHeader {
pub fn write<W: Write>(&self, ww: &mut W) -> OResult {
let bytes = self.pack_into();
ww.write(&bytes)
ww.write_all(&bytes)?;
Ok(bytes.len())
}
}
@ -141,11 +141,12 @@ pub trait Record<RData: RecordData> {
fn write<W: Write>(ww: &mut W, data: &RData::InData) -> OResult {
let packed_data = RData::pack(data);
let data_bytes = packed_data.as_ref();
let len: u16 = data_bytes.len().try_into().expect("Record longer than max size (u16)!");
let mut size = 0;
size += Self::write_header(ww, len)?;
size += ww.write(data_bytes)?;
Ok(size)
let len = u16::try_from(data_bytes.len()).ok()
.filter(|len| *len <= u16::MAX - 4)
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?;
Self::write_header(ww, len)?;
ww.write_all(data_bytes)?;
Ok(usize::from(len) + 4)
}
}
@ -235,7 +236,7 @@ impl RecordData for Int2Array {
}
fn pack(data: &Self::InData) -> Self::ByteData {
let mut buf = Vec::with_capacity(data.len() * 2);
let mut buf = vec![0; data.len() * 2];
Self::pack_into(&mut buf, data);
buf
}
@ -276,7 +277,7 @@ impl RecordData for Int4Array {
}
fn pack(data: &Self::InData) -> Self::ByteData {
let mut buf = Vec::with_capacity(data.len() * 4);
let mut buf = vec![0; data.len() * 4];
Self::pack_into(&mut buf, data);
buf
}
@ -366,7 +367,7 @@ impl RecordData for ASCII {
}
fn pack(data: &Self::InData) -> Self::ByteData {
let mut buf = Vec::with_capacity(data.len() * 4);
let mut buf = vec![0; data.len() + data.len() % 2];
Self::pack_into(&mut buf, data);
buf
}

91
tests/writing.rs Normal file
View file

@ -0,0 +1,91 @@
use arrow::array::Array;
use klamath_rs_ext::{
library::{FileHeader, read_library},
record::{Record, RecordHeader},
records,
};
use std::io::{self, Write};
#[test]
fn record_headers_include_their_own_size() {
let mut bytes = Vec::new();
assert_eq!(records::HEADER::write(&mut bytes, &600).unwrap(), 6);
assert_eq!(bytes, [0, 6, 0, 2, 2, 88]);
let (remaining, value) = records::HEADER::read(&bytes).unwrap();
assert!(remaining.is_empty());
assert_eq!(value, 600);
let maximum = RecordHeader {
data_size: u16::MAX - 4,
tag: 0x1003,
};
assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]);
}
#[test]
fn integer_array_records_round_trip() {
let mut bytes = Vec::new();
records::COLROW::write(&mut bytes, &[2, 3]).unwrap();
let (remaining, values) = records::COLROW::read(&bytes).unwrap();
assert!(remaining.is_empty());
assert_eq!(values.collect::<Vec<_>>(), [2, 3]);
let mut bytes = Vec::new();
let values = [i32::MIN, -1, 0, i32::MAX];
records::XY::write(&mut bytes, &values).unwrap();
let (remaining, parsed) = records::XY::read(&bytes).unwrap();
assert!(remaining.is_empty());
assert_eq!(parsed.collect::<Vec<_>>(), values);
}
#[test]
fn string_records_pad_only_when_needed_and_support_empty_strings() {
for value in [b"".as_slice(), b"odd", b"even"] {
let mut bytes = Vec::new();
let expected_size = 4 + value.len() + value.len() % 2;
assert_eq!(
records::LIBNAME::write(&mut bytes, value).unwrap(),
expected_size
);
assert_eq!(bytes.len(), expected_size);
let (remaining, parsed) = records::LIBNAME::read(&bytes).unwrap();
assert!(remaining.is_empty());
assert_eq!(parsed, value);
if value.len() % 2 == 1 {
assert_eq!(bytes.last(), Some(&0));
}
}
}
struct ShortWriter(Vec<u8>);
impl Write for ShortWriter {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let size = bytes.len().min(1);
self.0.extend_from_slice(&bytes[..size]);
Ok(size)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[test]
fn library_header_round_trips_through_a_short_writer() {
let header = FileHeader::new(b"odd", 1e-9, 0.001);
let mut writer = ShortWriter(Vec::new());
let mut written = header.write(&mut writer).unwrap();
written += records::ENDLIB::write(&mut writer, &()).unwrap();
assert_eq!(written, writer.0.len());
let (remaining, library) = read_library(&writer.0).unwrap();
assert!(remaining.is_empty());
assert_eq!(library.len(), 1);
}
#[test]
fn oversized_records_fail_before_writing() {
let mut bytes = Vec::new();
let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
assert!(bytes.is_empty());
}