pheon/src/basic.rs
Jan Petykiewicz 3783320f82 fix(writer): serialize complete records with correctly sized buffers
Include record headers in wire lengths, allocate initialized array and padded string buffers, and handle empty strings. Write all bytes to short writers and reject oversized records before writing.
2026-11-14 14:37:35 -08:00

392 lines
10 KiB
Rust

///
/// Functionality for parsing and writing basic data types
///
use byteorder::{ByteOrder, BigEndian};
use std::io;
use std::fmt;
pub type OResult = Result<usize, io::Error>;
pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>;
#[derive(Debug)]
pub enum ErrType {
Incomplete(Option<usize>),
Failed(String),
}
pub fn fail<O>(input: &[u8], msg: String) -> IResult<'_, O> {
Err((input, ErrType::Failed(msg)))
}
pub fn incomplete<O>(input: &[u8], size: Option<usize>) -> IResult<'_, O> {
Err((input, ErrType::Incomplete(size)))
}
pub fn take_bytes<CC: Into<usize>>(input: &[u8], count: CC) -> IResult<'_, &[u8]> {
let cc = count.into();
if input.len() < cc {
incomplete(input, Some(cc))
} else {
let (taken, input) = input.split_at(cc);
Ok((input, taken))
}
}
/*
* Parse functions
*/
pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> {
let (input, buf) = take_bytes(input, 2_usize)?;
let val = BigEndian::read_u16(buf);
Ok((input, val))
}
pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> {
let (input, buf) = take_bytes(input, 2_usize)?;
let val = BigEndian::read_i16(buf);
Ok((input, val))
}
pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> {
let (input, buf) = take_bytes(input, 4_usize)?;
let val = BigEndian::read_i32(buf);
Ok((input, val))
}
/// Convert GDS REAL8 to IEEE float64
pub fn decode_real8(int: u64) -> f64 {
let neg = int & 0x8000_0000_0000_0000;
let exp = (int >> 56) & 0x7f;
let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64;
if neg != 0 {
mant *= -1.0
}
let exp2 = 4 * (exp as i32 - 64) - 56;
mant * 2_f64.powi(exp2)
}
pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> {
let (input, buf) = take_bytes(input, 8_usize)?;
let data = BigEndian::read_u64(buf);
Ok((input, decode_real8(data)))
}
pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> {
let mut buf = [0_i16; 6];
let mut input = input;
for bb in &mut buf {
(input, *bb) = parse_int2(input)?;
}
buf[0] += 1900; // Year is from 1900
Ok((input, buf))
}
pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> {
let mut bits = [false; 16];
let (input, val) = parse_int2(input)?;
for (ii, bit) in bits.iter_mut().enumerate() {
*bit = ((val >> (16 - 1 - ii)) & 0x01) == 1;
}
Ok((input, bits))
}
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec<u8>> {
let length = length as usize;
let (input, data) = take_bytes(input, length)?;
let true_length = length - usize::from(data.last() == Some(&0));
let vec = data[..true_length].to_vec();
Ok((input, vec))
}
/*
* Pack functions
*/
pub fn bitarray2int(bits: &[bool; 16]) -> u16 {
let mut int: u16 = 0;
for (ii, bit) in bits.iter().enumerate() {
int |= (*bit as u16) << (16 - 1 - ii);
}
int
}
pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) {
BigEndian::write_u16(buf, bitarray2int(bits));
}
pub fn pack_int2(buf: &mut [u8], int: i16) {
BigEndian::write_i16(buf, int);
}
pub fn pack_int4(buf: &mut [u8], int: i32) {
BigEndian::write_i32(buf, int);
}
pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> {
BigEndian::write_u64(buf, encode_real8(fnum)?);
Ok(())
}
pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize {
let len = data.len();
buf[..len].copy_from_slice(data);
if len % 2 == 1 {
buf[len] = 0;
len + 1
} else {
len
}
}
pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) {
assert!(buf.len() >= 6 * 2);
let year = date[0] - 1900;
pack_int2(buf, year);
for ii in 1..6 {
pack_int2(&mut buf[(2 * ii)..], date[ii]);
}
}
#[derive(Debug, Clone)]
pub struct FloatTooBigError {
float_value: f64,
}
impl fmt::Display for FloatTooBigError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Float {0} is too large for Real8", self.float_value)
}
}
/// Convert from float64 to GDS REAL8 representation.
pub fn encode_real8(fnum: f64) -> Result<u64, FloatTooBigError> {
// Split the ieee float bitfields
let ieee = fnum.to_bits();
let sign = ieee & 0x8000_0000_0000_0000;
let ieee_exp = (ieee >> 52) as i32 & 0x7ff;
let ieee_mant = ieee & 0x000f_ffff_ffff_ffff;
let subnorm = (ieee_exp == 0) & (ieee_mant != 0);
if (ieee_exp == 0) & (ieee_mant == 0) {
return Ok(0)
}
// IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023)
// IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022)
// GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64)
// = (gds_mant / 2^56) * 2^(4 * gds_exp - 256)
// Convert exponent.
let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII)
let mut exp16 = exp2 / 4;
let rest = exp2 % 4;
// Compensate for exponent coarseness
let comp = rest != 0;
let mut shift;
if comp {
exp16 += 1;
shift = 4 - rest;
} else {
shift = rest;
}
shift -= 3; // account for gds bit position
// add leading one
let mut gds_mant_unshifted = ieee_mant;
if !subnorm {
gds_mant_unshifted += 0x10_0000_0000_0000;
}
let mut gds_mant = if shift > 0 {
gds_mant_unshifted >> shift
} else {
gds_mant_unshifted << -shift
};
// add gds exponent bias
let mut gds_exp = exp16 + 64;
if gds_exp < -14 {
// number is too small
return Ok(0)
}
let neg_biased = gds_exp < 0;
if neg_biased {
gds_mant >>= gds_exp * 4;
gds_exp = 0;
}
let too_big = (gds_exp > 0x7f) & !subnorm;
if too_big {
return Err(FloatTooBigError{float_value: fnum});
}
let gds_exp_bits = (gds_exp as u64) << 56;
let real8 = sign | gds_exp_bits | gds_mant;
Ok(real8)
}
#[cfg(test)]
mod tests {
#[test]
fn test_parse_bitarray() {
use crate::basic::parse_bitarray;
//assert!(parse_bitarray(b"59") == 13625);
assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]);
assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]);
let arr_0001 = parse_bitarray(b"\x00\x01").unwrap().1;
for (ii, &vv) in arr_0001.iter().enumerate() {
assert_eq!(ii == 15, vv);
}
let arr_8000 = parse_bitarray(b"\x80\x00").unwrap().1;
for (ii, &vv) in arr_8000.iter().enumerate() {
assert_eq!(ii == 0, vv);
}
}
#[test]
fn test_parse_int2() {
use crate::basic::parse_int2;
assert_eq!(parse_int2(b"59").unwrap().1, 13625);
assert_eq!(parse_int2(b"\0\0").unwrap().1, 0);
assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1);
}
#[test]
fn test_parse_int4() {
use crate::basic::parse_int4;
assert_eq!(parse_int4(b"4321").unwrap().1, 875770417);
}
#[test]
fn test_decode_real8() {
use crate::basic::decode_real8;
// zeroes
assert_eq!(decode_real8(0x0), 0.0);
assert_eq!(decode_real8(1<<63), 0.0); // negative
assert_eq!(decode_real8(0xff << 56), 0.0); // denormalized
assert_eq!(decode_real8(0x4110 << 48), 1.0);
assert_eq!(decode_real8(0xC120 << 48), -2.0);
}
#[test]
#[should_panic]
fn test_encode_real8_panic() {
use crate::basic::encode_real8;
encode_real8(1e80).unwrap();
}
#[test]
fn test_parse_real8() {
use crate::basic:: parse_real8;
assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1);
assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1);
assert_eq!(-2.0, parse_real8(&[0xC1, 0x20, 0, 0, 0, 0, 0, 0]).unwrap().1);
}
#[test]
fn test_parse_ascii() {
use crate::basic::parse_ascii;
assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345");
assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte
assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456");
}
#[test]
fn test_pack_bitarray() {
use crate::basic::pack_bitarray;
let mut buf = [10; 3];
let mut bools = [false; 16];
bools[1] = true;
bools[2] = true;
bools[11] = true;
pack_bitarray(&mut buf, &bools);
assert_eq!(buf[0], 0b0110_0000);
assert_eq!(buf[1], 0b0001_0000);
assert_eq!(buf[2], 10);
}
#[test]
fn test_pack_int2() {
use crate::basic::pack_int2;
let mut buf = [10; 3 * 2];
pack_int2(&mut buf, -3);
pack_int2(&mut buf[2..], 2);
pack_int2(&mut buf[4..], -1);
assert_eq!(buf[0..2], [0xFF, 0xFD]);
assert_eq!(buf[2..4], [0x00, 0x02]);
assert_eq!(buf[4..6], [0xFF, 0xFF]);
}
#[test]
fn test_pack_int4() {
use crate::basic::pack_int4;
let mut buf = [10; 3 * 4];
pack_int4(&mut buf, -3);
pack_int4(&mut buf[4..], 2);
pack_int4(&mut buf[8..], -1);
assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]);
assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]);
assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]);
}
#[test]
fn test_encode_real8() {
use crate::basic::{encode_real8, decode_real8};
const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12];
for vv in REALS {
print!("{vv}\n");
assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON);
}
}
#[test]
fn test_pack_real8() {
use crate::basic::{pack_real8, parse_real8};
const COUNT: usize = 7;
const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12];
let mut buf = [10; 8 * COUNT];
for (ii, &vv) in REALS.iter().enumerate() {
pack_real8(&mut buf[ii * 8..], vv).unwrap();
}
for (ii, &vv) in REALS.iter().enumerate() {
print!("{vv}\n");
let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1;
assert!((parsed_val - vv).abs() < f64::EPSILON);
}
}
#[test]
fn test_pack_ascii() {
use crate::basic::pack_ascii;
let mut buf = [10; 12];
pack_ascii(&mut buf[0..], "4321".as_bytes());
pack_ascii(&mut buf[6..], "321".as_bytes());
assert_eq!(&buf[0..4], "4321".as_bytes());
assert_eq!(&buf[4..6], [10, 10]);
assert_eq!(&buf[6..9], "321".as_bytes());
assert_eq!(&buf[9..], [0, 10, 10]);
}
}