fix(writer): serialize complete records with correctly sized buffers
Include record headers in wire lengths, allocate initialized array and padded string buffers, and handle empty strings. Write all bytes to short writers and reject oversized records before writing.
This commit is contained in:
parent
aa2132dc6e
commit
3783320f82
3 changed files with 106 additions and 15 deletions
|
|
@ -98,8 +98,7 @@ pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> {
|
||||||
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec<u8>> {
|
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec<u8>> {
|
||||||
let length = length as usize;
|
let length = length as usize;
|
||||||
let (input, data) = take_bytes(input, length)?;
|
let (input, data) = take_bytes(input, length)?;
|
||||||
let last = data[length - 1];
|
let true_length = length - usize::from(data.last() == Some(&0));
|
||||||
let true_length = if last == 0 { length - 1 } else { length };
|
|
||||||
let vec = data[..true_length].to_vec();
|
let vec = data[..true_length].to_vec();
|
||||||
Ok((input, vec))
|
Ok((input, vec))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
///
|
///
|
||||||
/// Generic record-level read/write functionality.
|
/// Generic record-level read/write functionality.
|
||||||
///
|
///
|
||||||
use std::io::Write;
|
use std::io::{self, Write};
|
||||||
use std::convert::TryInto;
|
|
||||||
use byteorder::{ByteOrder, BigEndian};
|
use byteorder::{ByteOrder, BigEndian};
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -34,8 +33,8 @@ impl RecordHeader {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn pack_into(&self) -> [u8; 4] {
|
pub fn pack_into(&self) -> [u8; 4] {
|
||||||
assert!(self.data_size < 0xffff - 4, "Record too big!");
|
let size = self.data_size.checked_add(4).expect("Record too big!");
|
||||||
let vals = [self.data_size, self.tag];
|
let vals = [size, self.tag];
|
||||||
let mut buf = [0x77; 4];
|
let mut buf = [0x77; 4];
|
||||||
BigEndian::write_u16_into(&vals, &mut buf);
|
BigEndian::write_u16_into(&vals, &mut buf);
|
||||||
buf
|
buf
|
||||||
|
|
@ -43,7 +42,8 @@ impl RecordHeader {
|
||||||
|
|
||||||
pub fn write<W: Write>(&self, ww: &mut W) -> OResult {
|
pub fn write<W: Write>(&self, ww: &mut W) -> OResult {
|
||||||
let bytes = self.pack_into();
|
let bytes = self.pack_into();
|
||||||
ww.write(&bytes)
|
ww.write_all(&bytes)?;
|
||||||
|
Ok(bytes.len())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -141,11 +141,12 @@ pub trait Record<RData: RecordData> {
|
||||||
fn write<W: Write>(ww: &mut W, data: &RData::InData) -> OResult {
|
fn write<W: Write>(ww: &mut W, data: &RData::InData) -> OResult {
|
||||||
let packed_data = RData::pack(data);
|
let packed_data = RData::pack(data);
|
||||||
let data_bytes = packed_data.as_ref();
|
let data_bytes = packed_data.as_ref();
|
||||||
let len: u16 = data_bytes.len().try_into().expect("Record longer than max size (u16)!");
|
let len = u16::try_from(data_bytes.len()).ok()
|
||||||
let mut size = 0;
|
.filter(|len| *len <= u16::MAX - 4)
|
||||||
size += Self::write_header(ww, len)?;
|
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?;
|
||||||
size += ww.write(data_bytes)?;
|
Self::write_header(ww, len)?;
|
||||||
Ok(size)
|
ww.write_all(data_bytes)?;
|
||||||
|
Ok(usize::from(len) + 4)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -235,7 +236,7 @@ impl RecordData for Int2Array {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pack(data: &Self::InData) -> Self::ByteData {
|
fn pack(data: &Self::InData) -> Self::ByteData {
|
||||||
let mut buf = Vec::with_capacity(data.len() * 2);
|
let mut buf = vec![0; data.len() * 2];
|
||||||
Self::pack_into(&mut buf, data);
|
Self::pack_into(&mut buf, data);
|
||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
@ -276,7 +277,7 @@ impl RecordData for Int4Array {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pack(data: &Self::InData) -> Self::ByteData {
|
fn pack(data: &Self::InData) -> Self::ByteData {
|
||||||
let mut buf = Vec::with_capacity(data.len() * 4);
|
let mut buf = vec![0; data.len() * 4];
|
||||||
Self::pack_into(&mut buf, data);
|
Self::pack_into(&mut buf, data);
|
||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
@ -366,7 +367,7 @@ impl RecordData for ASCII {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pack(data: &Self::InData) -> Self::ByteData {
|
fn pack(data: &Self::InData) -> Self::ByteData {
|
||||||
let mut buf = Vec::with_capacity(data.len() * 4);
|
let mut buf = vec![0; data.len() + data.len() % 2];
|
||||||
Self::pack_into(&mut buf, data);
|
Self::pack_into(&mut buf, data);
|
||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
|
||||||
91
tests/writing.rs
Normal file
91
tests/writing.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
use arrow::array::Array;
|
||||||
|
use klamath_rs_ext::{
|
||||||
|
library::{FileHeader, read_library},
|
||||||
|
record::{Record, RecordHeader},
|
||||||
|
records,
|
||||||
|
};
|
||||||
|
use std::io::{self, Write};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn record_headers_include_their_own_size() {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
assert_eq!(records::HEADER::write(&mut bytes, &600).unwrap(), 6);
|
||||||
|
assert_eq!(bytes, [0, 6, 0, 2, 2, 88]);
|
||||||
|
let (remaining, value) = records::HEADER::read(&bytes).unwrap();
|
||||||
|
assert!(remaining.is_empty());
|
||||||
|
assert_eq!(value, 600);
|
||||||
|
let maximum = RecordHeader {
|
||||||
|
data_size: u16::MAX - 4,
|
||||||
|
tag: 0x1003,
|
||||||
|
};
|
||||||
|
assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn integer_array_records_round_trip() {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
records::COLROW::write(&mut bytes, &[2, 3]).unwrap();
|
||||||
|
let (remaining, values) = records::COLROW::read(&bytes).unwrap();
|
||||||
|
assert!(remaining.is_empty());
|
||||||
|
assert_eq!(values.collect::<Vec<_>>(), [2, 3]);
|
||||||
|
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let values = [i32::MIN, -1, 0, i32::MAX];
|
||||||
|
records::XY::write(&mut bytes, &values).unwrap();
|
||||||
|
let (remaining, parsed) = records::XY::read(&bytes).unwrap();
|
||||||
|
assert!(remaining.is_empty());
|
||||||
|
assert_eq!(parsed.collect::<Vec<_>>(), values);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn string_records_pad_only_when_needed_and_support_empty_strings() {
|
||||||
|
for value in [b"".as_slice(), b"odd", b"even"] {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let expected_size = 4 + value.len() + value.len() % 2;
|
||||||
|
assert_eq!(
|
||||||
|
records::LIBNAME::write(&mut bytes, value).unwrap(),
|
||||||
|
expected_size
|
||||||
|
);
|
||||||
|
assert_eq!(bytes.len(), expected_size);
|
||||||
|
let (remaining, parsed) = records::LIBNAME::read(&bytes).unwrap();
|
||||||
|
assert!(remaining.is_empty());
|
||||||
|
assert_eq!(parsed, value);
|
||||||
|
if value.len() % 2 == 1 {
|
||||||
|
assert_eq!(bytes.last(), Some(&0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ShortWriter(Vec<u8>);
|
||||||
|
|
||||||
|
impl Write for ShortWriter {
|
||||||
|
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||||
|
let size = bytes.len().min(1);
|
||||||
|
self.0.extend_from_slice(&bytes[..size]);
|
||||||
|
Ok(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn library_header_round_trips_through_a_short_writer() {
|
||||||
|
let header = FileHeader::new(b"odd", 1e-9, 0.001);
|
||||||
|
let mut writer = ShortWriter(Vec::new());
|
||||||
|
let mut written = header.write(&mut writer).unwrap();
|
||||||
|
written += records::ENDLIB::write(&mut writer, &()).unwrap();
|
||||||
|
assert_eq!(written, writer.0.len());
|
||||||
|
let (remaining, library) = read_library(&writer.0).unwrap();
|
||||||
|
assert!(remaining.is_empty());
|
||||||
|
assert_eq!(library.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn oversized_records_fail_before_writing() {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err();
|
||||||
|
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
||||||
|
assert!(bytes.is_empty());
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue