WIP on (no branch): 48db472 snapshot 2021-12-18 21:05:00.635887

This commit is contained in:
jan 2022-03-20 16:28:34 -07:00
commit 07100dee8e

View file

@ -1,80 +1,106 @@
/*
* Functionality for parsing and writing basic data types
*/
use nom;
use nom::{IResult};
use byteorder::BigEndian;
//use std::io::Write;
///
/// Functionality for parsing and writing basic data types
///
use byteorder::{ByteOrder, BigEndian};
use std::io;
pub type OResult = Result<usize, io::Error>;
pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>;
pub enum ErrType {
Incomplete(Option<usize>),
Failed(String),
}
pub fn fail<O>(input: &[u8], msg: String) -> IResult<O> {
Err((input, ErrType::Failed(msg)))
}
pub fn incomplete<O>(input: &[u8], size: Option<usize>) -> IResult<O> {
Err((input, ErrType::Incomplete(size)))
}
pub fn take_bytes<CC: Into<usize>>(input: &[u8], count: CC) -> IResult<&[u8]> {
let cc = count.into();
if input.len() > cc {
incomplete(input, Some(cc))
} else {
let (taken, input) = input.split_at(cc);
Ok((input, taken))
}
}
pub type OWResult = Result<usize, io::Error>;
/*
* Parse functions
*/
//pub fn parse_byte_as_bits(input: &[u8]) -> IResult<&[u8], (u8, u8, u8, u8, u8, u8, u8, u8)> {
// nom::bits::bits(nom::sequence::tuple((
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize),
// )))(input)
//}
pub fn parse_int2(input: &[u8]) -> IResult<&[u8], i16> {
nom::number::streaming::be_i16(input)?
pub fn parse_u16(input: &[u8]) -> IResult<u16> {
let (input, buf) = take_bytes(input, 2_usize)?;
let val = BigEndian::read_u16(&buf);
Ok((input, val))
}
pub fn parse_int4(input: &[u8]) -> IResult<&[u8], i32> {
nom::number::streaming::be_i32(input)?
pub fn parse_int2(input: &[u8]) -> IResult<i16> {
let (input, buf) = take_bytes(input, 2_usize)?;
let val = BigEndian::read_i16(&buf);
Ok((input, val))
}
pub fn parse_int4(input: &[u8]) -> IResult<i32> {
let (input, buf) = take_bytes(input, 4_usize)?;
let val = BigEndian::read_i32(&buf);
Ok((input, val))
}
/// Convert GDS REAL8 to IEEE float64
pub fn decode_real8(int: u64) -> f64 {
// Convert GDS REAL8 to IEEE float64
let neg = int & 0x8000_0000_0000_0000;
let exp = (int >> 56) & 0x7f;
let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64;
if neg != 0 {
mant *= -1
mant *= -1.0
}
mant * 2_f64.powi(4 * (exp - 64) - 56)
let exp2 = 4 * (exp as i32 - 64) - 56;
mant * 2_f64.powi(exp2)
}
pub fn parse_real8(input: &[u8]) -> IResult<&[u8], f64> {
let data = nom::number::streaming::be_u64(input)?;
IResult::Ok(decode_real8(data))
pub fn parse_real8(input: &[u8]) -> IResult<f64> {
let (input, buf) = take_bytes(input, 8_usize)?;
let data = BigEndian::read_u64(&buf);
Ok((input, decode_real8(data)))
}
pub fn parse_datetime(input: &[u8]) -> IResult<&[u8], [u16; 6]> {
let mut buf = [0_u16; 6];
let mut parts = nom::multi::fill(parse_int2, &mut buf)(input);
parts[0] += 1900; // Year is from 1900
IResult::Ok(parts)
pub fn parse_datetime(input: &[u8]) -> IResult<[i16; 6]> {
let mut buf = [0_i16; 6];
let mut input = input;
for ii in 0..6 {
(input, buf[ii]) = parse_int2(input)?;
}
buf[0] += 1900; // Year is from 1900
Ok((input, buf))
}
pub fn parse_bitarray(input: &[u8]) -> IResult<&[u8], [bool; 16]> {
let bits = [false; 16];
pub fn parse_bitarray(input: &[u8]) -> IResult<[bool; 16]> {
let mut bits = [false; 16];
let (input, val) = parse_int2(input)?;
for ii in 0..16 {
bits[ii] = ((val >> (16 - 1 - ii)) & 0x01) == 1;
}
bits
Ok((input, bits))
}
pub fn parse_ascii(input: &[u8], length: usize) -> IResult<&[u8], Vec<u8>> {
let last = input[length - 1];
let true_length = if last == '\0' { length - 1 } else { length };
let vec = input[..true_length].to_vec();
IResult::Ok((input[length..], vec))
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<Vec<u8>> {
let length = length as usize;
let (input, data) = take_bytes(input, length)?;
let last = data[length - 1];
let true_length = if last == 0 { length - 1 } else { length };
let vec = data[..true_length].to_vec();
Ok((input, vec))
}
@ -82,7 +108,7 @@ pub fn parse_ascii(input: &[u8], length: usize) -> IResult<&[u8], Vec<u8>> {
* Pack functions
*/
pub fn pack_bitarray(bits: &[bool; 16]) -> u16 {
pub fn bitarray2int(bits: &[bool; 16]) -> u16 {
let mut int: u16 = 0;
for ii in 0..16 {
int |= (bits[ii] as u16) << (16 - 1 - ii);
@ -90,24 +116,28 @@ pub fn pack_bitarray(bits: &[bool; 16]) -> u16 {
int
}
pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) {
BigEndian::write_u16(buf, bitarray2int(bits))
}
pub fn pack_int2(buf: &mut [u8], int: i16) {
BigEndian::write_i16(&mut buf, int)
BigEndian::write_i16(buf, int)
}
pub fn pack_int4(buf: &mut [u8], int: i32) {
BigEndian::write_i32(&mut buf, int)
BigEndian::write_i32(buf, int)
}
pub fn pack_real8(buf: &mut [u8], fnum: f64) {
BigEndian::write_u64(&mut buf, encode_real8(fnum))
BigEndian::write_u64(buf, encode_real8(fnum))
}
pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize {
let len = data.len();
buf[..len].copy_from_slice(data);
if len % 2 == 1 {
buf[len] = '\0';
buf[len] = 0;
len + 1
} else {
len
@ -115,19 +145,18 @@ pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize {
}
pub fn pack_datetime(buf: &mut [u8], date: &[u16; 6]) {
pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) {
assert!(buf.len() >= 6 * 2);
let year = date[0] - 1900;
pack_int2(&mut buf, year);
pack_int2(buf, year);
for ii in 1..6 {
pack_int2(&mut buf[(2 * ii)..], date[ii]);
}
}
/// Convert from float64 to GDS REAL8 representation.
pub fn encode_real8(fnum: f64) -> u64 {
// Convert from float64 to GDS REAL8 representation.
// Split the ieee float bitfields
let ieee = fnum.to_bits();
let sign = ieee & 0x8000_0000_0000_0000;
@ -191,8 +220,119 @@ pub fn encode_real8(fnum: f64) -> u64 {
panic!("Number too big for real8 format"); //TODO error handling
}
let gds_exp_bits = gds_exp << 56;
let gds_exp_bits = (gds_exp as u64) << 56;
let real8 = sign | gds_exp_bits | gds_mant;
real8
}
#[cfg(test)]
mod tests {
#[test]
fn test_parse_bitarray() {
use basic::parse_bitarray;
//assert!(parse_bitarray(b"59") == 13625);
assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]);
assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]);
let arr_0001 = parse_bitarray(b"\x00\x01").unwrap().1;
for (ii, &vv) in arr_0001.iter().enumerate() {
assert_eq!(ii == 15, vv);
}
let arr_8000 = parse_bitarray(b"\x80\x00").unwrap().1;
for (ii, &vv) in arr_8000.iter().enumerate() {
assert_eq!(ii == 0, vv);
}
}
#[test]
fn test_parse_int2() {
use basic::parse_int2;
assert_eq!(parse_int2(b"59").unwrap().1, 13625);
assert_eq!(parse_int2(b"\0\0").unwrap().1, 0);
assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1);
}
#[test]
fn test_parse_int4() {
use basic::parse_int4;
assert_eq!(parse_int4(b"4321").unwrap().1, 875770417);
}
#[test]
fn test_decode_real8() {
use basic::decode_real8;
// zeroes
assert_eq!(decode_real8(0x0), 0.0);
assert_eq!(decode_real8(1<<63), 0.0); // negative
assert_eq!(decode_real8(0xff << 56), 0.0); // denormalized
assert_eq!(decode_real8(0x4110 << 48), 1.0);
assert_eq!(decode_real8(0xC120 << 48), -2.0);
//TODO panics on invalid?
}
#[test]
fn test_parse_real8() {
use basic:: parse_real8;
assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1);
assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1);
assert_eq!(-2.0, parse_real8(&[0xC1, 0x20, 0, 0, 0, 0, 0, 0]).unwrap().1);
}
#[test]
fn test_parse_ascii() {
use basic::parse_ascii;
assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345");
assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte
assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456");
}
/*
fn test_pack_bitarray() {
packed = pack_bitarray(321)
assert_eq!(len(packed), 2);
assert_eq!(packed, struct.pack('>H', 321));
}
fn test_pack_int2() {
packed = pack_int2((3, 2, 1))
assert(len(packed) == 3*2)
assert(packed == struct.pack('>3h', 3, 2, 1))
assert(pack_int2([-3, 2, -1]) == struct.pack('>3h', -3, 2, -1))
}
fn test_pack_int4() {
packed = pack_int4((3, 2, 1))
assert(len(packed) == 3*4)
assert(packed == struct.pack('>3l', 3, 2, 1))
assert(pack_int4([-3, 2, -1]) == struct.pack('>3l', -3, 2, -1))
}
fn test_encode_real8() {
assert(encode_real8(numpy.array([0.0])) == 0)
arr = numpy.array((1.0, -2.0, 1e-9, 1e-3, 1e-12))
assert_array_equal(decode_real8(encode_real8(arr)), arr)
}
fn test_pack_real8() {
reals = (0, 1, -1, 0.5, 1e-9, 1e-3, 1e-12)
packed = pack_real8(reals)
assert(len(packed) == len(reals) * 8)
assert_array_equal(parse_real8(packed), reals)
}
fn test_pack_ascii() {
assert(pack_ascii(b'4321') == b'4321')
assert(pack_ascii(b'321') == b'321\0')
}
*/
}