Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy scans and selective cell loading. Validate numeric packing and array pitches, and preserve normalized REAL8 encodings exactly within the supported range. Add standalone dump, structural diff, hierarchy, layer-name annotation and Git diff commands. Provide scoped text and JSON reports, deterministic comparison, and record-preserving metadata edits. Package the shared library and optional Python Arrow readers as pheon using CFFI ABI mode and Python-independent platform wheels. Include locked source builds, native API checks, cross-platform artifact workflows, documentation and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer. Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14, and 71 installed companion integration tests pass. Full, scanned and selected Arrow layouts match the previous schemas.
This commit is contained in:
parent
e0a634f476
commit
b3346b40a4
1 changed files with 12956 additions and 1 deletions
353
src/basic.rs
Normal file
353
src/basic.rs
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
///
|
||||
/// Functionality for parsing and writing basic data types
|
||||
///
|
||||
use std::io;
|
||||
use std::fmt;
|
||||
|
||||
pub type OResult = Result<usize, io::Error>;
|
||||
|
||||
pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ErrType {
|
||||
Incomplete(Option<usize>),
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
pub fn fail<O>(input: &[u8], msg: String) -> IResult<'_, O> {
|
||||
Err((input, ErrType::Failed(msg)))
|
||||
}
|
||||
|
||||
pub fn incomplete<O>(input: &[u8], size: Option<usize>) -> IResult<'_, O> {
|
||||
Err((input, ErrType::Incomplete(size)))
|
||||
}
|
||||
|
||||
pub fn take_bytes<CC: Into<usize>>(input: &[u8], count: CC) -> IResult<'_, &[u8]> {
|
||||
let cc = count.into();
|
||||
if input.len() < cc {
|
||||
incomplete(input, Some(cc))
|
||||
} else {
|
||||
let (taken, input) = input.split_at(cc);
|
||||
Ok((input, taken))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Parse functions
|
||||
*/
|
||||
fn take_array<const N: usize>(input: &[u8]) -> IResult<'_, [u8; N]> {
|
||||
match input.split_first_chunk::<N>() {
|
||||
Some((bytes, remaining)) => Ok((remaining, *bytes)),
|
||||
None => incomplete(input, Some(N)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> {
|
||||
let (input, bytes) = take_array(input)?;
|
||||
Ok((input, u16::from_be_bytes(bytes)))
|
||||
}
|
||||
|
||||
pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> {
|
||||
let (input, bytes) = take_array(input)?;
|
||||
Ok((input, i16::from_be_bytes(bytes)))
|
||||
}
|
||||
|
||||
pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> {
|
||||
let (input, bytes) = take_array(input)?;
|
||||
Ok((input, i32::from_be_bytes(bytes)))
|
||||
}
|
||||
|
||||
/// Convert GDS REAL8 to IEEE float64
|
||||
pub fn decode_real8(int: u64) -> f64 {
|
||||
let neg = int & 0x8000_0000_0000_0000;
|
||||
let exp = (int >> 56) & 0x7f;
|
||||
let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64;
|
||||
if neg != 0 {
|
||||
mant *= -1.0
|
||||
}
|
||||
let exp2 = 4 * (exp as i32 - 64) - 56;
|
||||
mant * 2_f64.powi(exp2)
|
||||
}
|
||||
|
||||
pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> {
|
||||
let (input, bytes) = take_array(input)?;
|
||||
let data = u64::from_be_bytes(bytes);
|
||||
Ok((input, decode_real8(data)))
|
||||
}
|
||||
|
||||
|
||||
pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> {
|
||||
let mut buf = [0_i16; 6];
|
||||
let mut input = input;
|
||||
for bb in &mut buf {
|
||||
(input, *bb) = parse_int2(input)?;
|
||||
}
|
||||
buf[0] += 1900; // Year is from 1900
|
||||
Ok((input, buf))
|
||||
}
|
||||
|
||||
|
||||
pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> {
|
||||
let mut bits = [false; 16];
|
||||
let (input, val) = parse_int2(input)?;
|
||||
for (ii, bit) in bits.iter_mut().enumerate() {
|
||||
*bit = ((val >> (16 - 1 - ii)) & 0x01) == 1;
|
||||
}
|
||||
Ok((input, bits))
|
||||
}
|
||||
|
||||
|
||||
pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec<u8>> {
|
||||
let length = length as usize;
|
||||
let (input, data) = take_bytes(input, length)?;
|
||||
let true_length = length - usize::from(data.last() == Some(&0));
|
||||
let vec = data[..true_length].to_vec();
|
||||
Ok((input, vec))
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Pack functions
|
||||
*/
|
||||
|
||||
pub fn bitarray2int(bits: &[bool; 16]) -> u16 {
|
||||
let mut int: u16 = 0;
|
||||
for (ii, bit) in bits.iter().enumerate() {
|
||||
int |= (*bit as u16) << (16 - 1 - ii);
|
||||
}
|
||||
int
|
||||
}
|
||||
|
||||
pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) {
|
||||
buf[..2].copy_from_slice(&bitarray2int(bits).to_be_bytes());
|
||||
}
|
||||
|
||||
|
||||
pub fn pack_int2(buf: &mut [u8], int: i16) {
|
||||
buf[..2].copy_from_slice(&int.to_be_bytes());
|
||||
}
|
||||
|
||||
pub fn pack_int4(buf: &mut [u8], int: i32) {
|
||||
buf[..4].copy_from_slice(&int.to_be_bytes());
|
||||
}
|
||||
|
||||
pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> {
|
||||
let bytes = encode_real8(fnum)?.to_be_bytes();
|
||||
buf[..8].copy_from_slice(&bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize {
|
||||
let len = data.len();
|
||||
buf[..len].copy_from_slice(data);
|
||||
if len % 2 == 1 {
|
||||
buf[len] = 0;
|
||||
len + 1
|
||||
} else {
|
||||
len
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) {
|
||||
assert!(buf.len() >= 6 * 2);
|
||||
let year = date[0] - 1900;
|
||||
pack_int2(buf, year);
|
||||
for ii in 1..6 {
|
||||
pack_int2(&mut buf[(2 * ii)..], date[ii]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FloatTooBigError {
|
||||
float_value: f64,
|
||||
}
|
||||
impl fmt::Display for FloatTooBigError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "Float {0} is too large for Real8", self.float_value)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert float64 to a normalized GDS REAL8 representation.
|
||||
/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0.
|
||||
/// Nonfinite values and magnitudes at or above 2^252 return an error.
|
||||
pub fn encode_real8(fnum: f64) -> Result<u64, FloatTooBigError> {
|
||||
let ieee = fnum.to_bits();
|
||||
let ieee_exp = ((ieee >> 52) & 0x7ff) as i32;
|
||||
if ieee_exp == 0x7ff {
|
||||
return Err(FloatTooBigError { float_value: fnum });
|
||||
}
|
||||
|
||||
let exp2 = ieee_exp - 1023;
|
||||
if exp2 < -260 {
|
||||
// This also handles both signed zeros and every IEEE subnormal.
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let gds_exp = exp2.div_euclid(4) + 65;
|
||||
if gds_exp > 0x7f {
|
||||
return Err(FloatTooBigError { float_value: fnum });
|
||||
}
|
||||
|
||||
// The 53-bit IEEE significand needs only 0..3 left shifts to become
|
||||
// a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8.
|
||||
let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52);
|
||||
let mantissa = significand << exp2.rem_euclid(4);
|
||||
let sign = ieee & 0x8000_0000_0000_0000;
|
||||
Ok(sign | ((gds_exp as u64) << 56) | mantissa)
|
||||
}
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn test_parse_bitarray() {
|
||||
use crate::basic::parse_bitarray;
|
||||
|
||||
//assert!(parse_bitarray(b"59") == 13625);
|
||||
assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]);
|
||||
assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]);
|
||||
|
||||
let arr_0001 = parse_bitarray(b"\x00\x01").unwrap().1;
|
||||
for (ii, &vv) in arr_0001.iter().enumerate() {
|
||||
assert_eq!(ii == 15, vv);
|
||||
}
|
||||
|
||||
let arr_8000 = parse_bitarray(b"\x80\x00").unwrap().1;
|
||||
for (ii, &vv) in arr_8000.iter().enumerate() {
|
||||
assert_eq!(ii == 0, vv);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_int2() {
|
||||
use crate::basic::parse_int2;
|
||||
assert_eq!(parse_int2(b"59").unwrap().1, 13625);
|
||||
assert_eq!(parse_int2(b"\0\0").unwrap().1, 0);
|
||||
assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_int4() {
|
||||
use crate::basic::parse_int4;
|
||||
assert_eq!(parse_int4(b"4321").unwrap().1, 875770417);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_real8() {
|
||||
use crate::basic::decode_real8;
|
||||
|
||||
// zeroes
|
||||
assert_eq!(decode_real8(0x0), 0.0);
|
||||
assert_eq!(decode_real8(1<<63), 0.0); // negative
|
||||
assert_eq!(decode_real8(0xff << 56), 0.0); // denormalized
|
||||
|
||||
assert_eq!(decode_real8(0x4110 << 48), 1.0);
|
||||
assert_eq!(decode_real8(0xC120 << 48), -2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn test_encode_real8_panic() {
|
||||
use crate::basic::encode_real8;
|
||||
encode_real8(1e80).unwrap();
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn test_parse_real8() {
|
||||
use crate::basic:: parse_real8;
|
||||
|
||||
assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1);
|
||||
assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1);
|
||||
assert_eq!(-2.0, parse_real8(&[0xC1, 0x20, 0, 0, 0, 0, 0, 0]).unwrap().1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ascii() {
|
||||
use crate::basic::parse_ascii;
|
||||
|
||||
assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345");
|
||||
assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte
|
||||
assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pack_bitarray() {
|
||||
use crate::basic::pack_bitarray;
|
||||
let mut buf = [10; 3];
|
||||
let mut bools = [false; 16];
|
||||
bools[1] = true;
|
||||
bools[2] = true;
|
||||
bools[11] = true;
|
||||
|
||||
pack_bitarray(&mut buf, &bools);
|
||||
assert_eq!(buf[0], 0b0110_0000);
|
||||
assert_eq!(buf[1], 0b0001_0000);
|
||||
assert_eq!(buf[2], 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pack_int2() {
|
||||
use crate::basic::pack_int2;
|
||||
let mut buf = [10; 3 * 2];
|
||||
pack_int2(&mut buf, -3);
|
||||
pack_int2(&mut buf[2..], 2);
|
||||
pack_int2(&mut buf[4..], -1);
|
||||
assert_eq!(buf[0..2], [0xFF, 0xFD]);
|
||||
assert_eq!(buf[2..4], [0x00, 0x02]);
|
||||
assert_eq!(buf[4..6], [0xFF, 0xFF]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pack_int4() {
|
||||
use crate::basic::pack_int4;
|
||||
let mut buf = [10; 3 * 4];
|
||||
pack_int4(&mut buf, -3);
|
||||
pack_int4(&mut buf[4..], 2);
|
||||
pack_int4(&mut buf[8..], -1);
|
||||
assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]);
|
||||
assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]);
|
||||
assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_real8() {
|
||||
use crate::basic::{encode_real8, decode_real8};
|
||||
const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12];
|
||||
for vv in REALS {
|
||||
print!("{vv}\n");
|
||||
assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pack_real8() {
|
||||
use crate::basic::{pack_real8, parse_real8};
|
||||
const COUNT: usize = 7;
|
||||
const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12];
|
||||
let mut buf = [10; 8 * COUNT];
|
||||
for (ii, &vv) in REALS.iter().enumerate() {
|
||||
pack_real8(&mut buf[ii * 8..], vv).unwrap();
|
||||
}
|
||||
for (ii, &vv) in REALS.iter().enumerate() {
|
||||
print!("{vv}\n");
|
||||
let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1;
|
||||
assert!((parsed_val - vv).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pack_ascii() {
|
||||
use crate::basic::pack_ascii;
|
||||
let mut buf = [10; 12];
|
||||
pack_ascii(&mut buf[0..], "4321".as_bytes());
|
||||
pack_ascii(&mut buf[6..], "321".as_bytes());
|
||||
assert_eq!(&buf[0..4], "4321".as_bytes());
|
||||
assert_eq!(&buf[4..6], [10, 10]);
|
||||
assert_eq!(&buf[6..9], "321".as_bytes());
|
||||
assert_eq!(&buf[9..], [0, 10, 10]);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue