From 48db47217afffd8f489c21dd9e01f57b1df65978 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 18 Dec 2021 21:05:00 -0800 Subject: [PATCH 01/18] snapshot 2021-12-18 21:05:00.635887 --- .gitignore | 6 + Cargo.toml | 8 + src/__init__.rs | 29 +++ src/basic.rs | 198 +++++++++++++++ src/elements.rs | 481 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 7 + src/library.rs | 282 ++++++++++++++++++++++ src/record.rs | 219 +++++++++++++++++ src/records.rs | 595 ++++++++++++++++++++++++++++++++++++++++++++++ src/test_basic.rs | 121 ++++++++++ 10 files changed, 1946 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/__init__.rs create mode 100644 src/basic.rs create mode 100644 src/elements.rs create mode 100644 src/lib.rs create mode 100644 src/library.rs create mode 100644 src/record.rs create mode 100644 src/records.rs create mode 100644 src/test_basic.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d80173 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target/ +**/*.rs.bk +Cargo.lock + +*.swp +*.swo diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..414196a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rs-klamath" +version = "0.1.0" +authors = ["jan "] + +[dependencies] +byteorder = "^1" +nom = "^7" diff --git a/src/__init__.rs b/src/__init__.rs new file mode 100644 index 0000000..44d6c6b --- /dev/null +++ b/src/__init__.rs @@ -0,0 +1,29 @@ +/* + * `klamath` is a Python module for reading and writing to the GDSII file format. + * + * The goal is to keep this library simple: + * - Map data types directly wherever possible. + * * Presents an accurate representation of what is saved to the file. + * * Avoids excess copies / allocations for speed. + * * No "automatic" error checking, except when casting datatypes. + * If data integrity checks are provided at all, they must be + * explicitly run by the caller. + * - Low-level functionality is first-class. + * * Meant for use-cases where the caller wants to read or write + * individual GDS records. + * * Offers complete control over the written file. + * - Opinionated and limited high-level functionality. + * * Discards or ignores rarely-encountered data types. + * * Keeps functions simple and reusable. + * * Only de/encodes the file format, doesn't provide tools to modify + * the data itself. + * * Still requires explicit values for most fields. + * - No compilation + * * Uses `numpy` for speed, since it's commonly available / pre-built. + * * Building this library should not require a compiler. + * + * `klamath` was built to provide a fast and versatile GDS interface for + * [masque](https://mpxd.net/code/jan/masque), which provides higher-level + * tools for working with hierarchical design data and supports multiple + * file formats. + */ diff --git a/src/basic.rs b/src/basic.rs new file mode 100644 index 0000000..0cadc0d --- /dev/null +++ b/src/basic.rs @@ -0,0 +1,198 @@ +/* + * Functionality for parsing and writing basic data types + */ +use nom; +use nom::{IResult}; +use byteorder::BigEndian; +//use std::io::Write; +use std::io; + + +pub type OWResult = Result; + +/* + * Parse functions + */ +//pub fn parse_byte_as_bits(input: &[u8]) -> IResult<&[u8], (u8, u8, u8, u8, u8, u8, u8, u8)> { +// nom::bits::bits(nom::sequence::tuple(( +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// )))(input) +//} + + +pub fn parse_int2(input: &[u8]) -> IResult<&[u8], i16> { + nom::number::streaming::be_i16(input)? +} + +pub fn parse_int4(input: &[u8]) -> IResult<&[u8], i32> { + nom::number::streaming::be_i32(input)? +} + +pub fn decode_real8(int: u64) -> f64 { + // Convert GDS REAL8 to IEEE float64 + let neg = int & 0x8000_0000_0000_0000; + let exp = (int >> 56) & 0x7f; + let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64; + if neg != 0 { + mant *= -1 + } + mant * 2_f64.powi(4 * (exp - 64) - 56) +} + +pub fn parse_real8(input: &[u8]) -> IResult<&[u8], f64> { + let data = nom::number::streaming::be_u64(input)?; + IResult::Ok(decode_real8(data)) +} + + +pub fn parse_datetime(input: &[u8]) -> IResult<&[u8], [u16; 6]> { + let mut buf = [0_u16; 6]; + let mut parts = nom::multi::fill(parse_int2, &mut buf)(input); + parts[0] += 1900; // Year is from 1900 + IResult::Ok(parts) +} + + +pub fn parse_bitarray(input: &[u8]) -> IResult<&[u8], [bool; 16]> { + let bits = [false; 16]; + let (input, val) = parse_int2(input)?; + for ii in 0..16 { + bits[ii] = ((val >> (16 - 1 - ii)) & 0x01) == 1; + } + bits +} + + +pub fn parse_ascii(input: &[u8], length: usize) -> IResult<&[u8], Vec> { + let last = input[length - 1]; + let true_length = if last == '\0' { length - 1 } else { length }; + let vec = input[..true_length].to_vec(); + IResult::Ok((input[length..], vec)) +} + + +/* + * Pack functions + */ + +pub fn pack_bitarray(bits: &[bool; 16]) -> u16 { + let mut int: u16 = 0; + for ii in 0..16 { + int |= (bits[ii] as u16) << (16 - 1 - ii); + } + int +} + + +pub fn pack_int2(buf: &mut [u8], int: i16) { + BigEndian::write_i16(&mut buf, int) +} + +pub fn pack_int4(buf: &mut [u8], int: i32) { + BigEndian::write_i32(&mut buf, int) +} + +pub fn pack_real8(buf: &mut [u8], fnum: f64) { + BigEndian::write_u64(&mut buf, encode_real8(fnum)) +} + +pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { + let len = data.len(); + buf[..len].copy_from_slice(data); + if len % 2 == 1 { + buf[len] = '\0'; + len + 1 + } else { + len + } +} + + +pub fn pack_datetime(buf: &mut [u8], date: &[u16; 6]) { + assert!(buf.len() >= 6 * 2); + let year = date[0] - 1900; + pack_int2(&mut buf, year); + for ii in 1..6 { + pack_int2(&mut buf[(2 * ii)..], date[ii]); + } +} + + +pub fn encode_real8(fnum: f64) -> u64 { + // Convert from float64 to GDS REAL8 representation. + + // Split the ieee float bitfields + let ieee = fnum.to_bits(); + let sign = ieee & 0x8000_0000_0000_0000; + let ieee_exp = (ieee >> 52) as i32 & 0x7ff; + let ieee_mant = ieee & 0xf_ffff_ffff_ffff; + + let subnorm = (ieee_exp == 0) & (ieee_mant != 0); + if (ieee_exp == 0) & (ieee_mant == 0) { + return 0 + } + + // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) + // IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022) + // GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64) + // = (gds_mant / 2^56) * 2^(4 * gds_exp - 256) + + // Convert exponent. + let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII) + let mut exp16 = exp2 / 4; + let rest = exp2 % 4; + + // Compensate for exponent coarseness + let comp = rest != 0; + let mut shift; + if comp { + exp16 += 1; + shift = 4 - rest; + } else { + shift = rest; + } + shift -= 3; // account for gds bit position + + // add leading one + let mut gds_mant_unshifted = ieee_mant; + if !subnorm { + gds_mant_unshifted += 0x10_0000_0000_0000; + } + + let mut gds_mant = if shift > 0 { + gds_mant_unshifted >> shift + } else { + gds_mant_unshifted << -shift + }; + + // add gds exponent bias + let mut gds_exp = exp16 + 64; + + if gds_exp < -14 { + // number is too small + return 0 + } + + let neg_biased = gds_exp < 0; + if neg_biased { + gds_mant >>= gds_exp * 4; + gds_exp = 0; + } + + let too_big = (gds_exp > 0x7f) & !subnorm; + if too_big { + panic!("Number too big for real8 format"); //TODO error handling + } + + let gds_exp_bits = gds_exp << 56; + + let real8 = sign | gds_exp_bits | gds_mant; + real8 +} diff --git a/src/elements.rs b/src/elements.rs new file mode 100644 index 0000000..e972b77 --- /dev/null +++ b/src/elements.rs @@ -0,0 +1,481 @@ +/* + * Functionality for reading/writing elements (geometry, text labels, + * structure references) and associated properties. + */ + +//from .record import Record +// +use records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; + +use records; +use record::RecordHeader; +use basic::{OWResult}; + +use std::collections::HashMap; +use std::io::Write; +use nom::IResult; + + +pub fn read_properties(input: &[u8]) -> IResult<&[u8], HashMap::>> { + /* + * Read element properties. + * + * Assumes PROPATTR records have unique values. + * Stops reading after consuming ENDEL record. + * + * Args: + * stream: Stream to read from. + * + * Returns: + * propattr: -> propvalue mapping + */ + let properties = HashMap{}; + + let (input, header) = RecordHeader::parse(input)?; + while header.tag != ENDEL::tag() { + if header.tag == PROPATTR::tag() { + let (input, key) = PROPATTR::read_data(input, header.data_size)?; + let (input, value) = PROPVALUE::read(input)?; + assert!(!properties.contains_key(key), format!{"Duplicate property key: {}", key}); + properties.insert(key, value); + } + let (input, header) = RecordHeader::parse(input)?; + } + Ok((input, properties)) +} + + +fn write_properties(ww: W, properties: &HashMap::>) -> OWResult { + /* + * Write element properties. + * + * This is does _not_ write the ENDEL record. + * + * Args: + * stream: Stream to write to. + */ + let mut size = 0; + for (key, value) in &properties { + size += PROPATTR::write(ww, key)?; + size += PROPVALUE::write(ww, value)?; + } + Ok(size) +} + +trait Element { + fn parse(input: &[u8]) -> Self; + /* + * Read from a stream to construct this object. + * Consumes up to (and including) the ENDEL record. + */ + + fn write(&self, ww: W) -> OWResult; + /* + * Write this element to a stream. + * Finishes with an ENDEL record. + */ +} + +struct Reference { + /* + * Datastructure representing + * an instance of a structure (SREF / structure reference) or + * an array of instances (AREF / array reference). + * Type is determined by the presence of the `colrow` tuple. + * + * Transforms are applied to each individual instance (_not_ + * to the instance's origin location || array vectors). + */ + struct_name: Vec, // Name of the structure being referenced. + invert_y: bool, // Whether to mirror the pattern (negate y-values / flip across x-axis). Default false. + mag: f64, // Scaling factor (default 1) """ + angle_deg: f64, // Rotation (degrees counterclockwise) + xy: Vec, + /* + * (For SREF) Location in the parent structure corresponding to the instance's origin (0, 0). + * (For AREF) 3 locations: + * [`offset`, + * `offset + col_basis_vector * colrow[0]`, + * `offset + row_basis_vector * colrow[1]`] + * which define the first instance's offset and the array's basis vectors. + * Note that many GDS implementations only support manhattan basis vectors, and some + * assume a certain axis mapping (e.g. x->columns, y->rows) and "reinterpret" the + * basis vectors to match it. + */ + + colrow: Option<(i32, i32)>, // Number of columns and rows (AREF) || None (SREF) + properties: HashMap::>, // Properties associated with this reference. +} + +impl Element for Reference { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut invert_y = false; + let mut mag = 1; + let mut angle_deg = 0; + let mut colrow = None; + let (input, mut struct_name) = SNAME::skip_and_read(input)?; + + let (input, mut header) = RecordHeader::parse(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_STRANS => + {let (input, invert_y) = STRANS::read_data(input, header.data_size)?[0];}, + records::RTAG_MAG => + {let (input, mag) = MAG::read_data(input, header.data_size)?;}, + records::RTAG_ANGLE => + {let (input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, + records::RTAG_COLROW => + {let (input, colrow) = COLROW::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + }; + let (input, header) = RecordHeader::parse(input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + let (input, properties) = read_properties(input)?; + Reference{ + struct_name: struct_name, + xy: xy, + properties: properties, + colrow: colrow, + invert_y: invert_y, + mag: mag, + angle_deg: angle_deg + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += match self.colrow { + None => SREF::write(ww)?, + Some(_) => AREF::write(ww)?, + }; + + size += SNAME::write(ww, self.struct_name)?; + if self.angle_deg != 0 || self.mag != 1 || self.invert_y { + size += STRANS::write(ww, (self.invert_y as u16) << 15)?; + if self.mag != 1 { + size += MAG::write(ww, self.mag)?; + } + if self.angle_deg !=0 { + size += ANGLE::write(ww, self.angle_deg)?; + } + } + + if self.colrow.is_some() { + size += COLROW::write(ww, self.colrow)?; + } + + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww, None)?; + Ok(size) + } +} + +impl Reference { + pub fn check(&self) { + if self.colrow.is_some() { + assert!(self.xy.len() != 6, format!("colrow is Some, so expected size-6 xy. Got {}", self.xy)); + } else { + assert!(self.xy.len() != 2, format!("Expected size-2 xy. Got {}", self.xy)); + } + } +} + + +struct Boundary { + /* + * Datastructure representing a Boundary element. + */ + layer: (i16, i16), // (layer, data_type) tuple + xy: Vec, // Ordered vertices of the shape. First and last points should be identical. Order x0, y0, x1,... + properties: HashMap::>, // Properties for the element. +} + +impl Element for Boundary { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + Boundary{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += BOUNDARY::write(ww)?; + size += LAYER::write(ww, self.layer.0)?; + size += DATATYPE::write(ww, self.layer.1)?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct Path { + /* + * Datastructure representing a Path element. + + * If `path_type < 4`, `extension` values are not written. + * During read, `exension` defaults to (0, 0) even if unused. + */ + layer: (i16, i16), // (layer, data_type) tuple + path_type: i16, // End-cap type (0: flush, 1: circle, 2: square, 4: custom) + width: i16, // Path width + extension: (i32, i32), // Extension when using path_type=4. Ignored otherwise. + xy: Vec, // Path centerline coordinates. [x0, y0, x1, y1,...] + properties: HashMap::>, //Properties for the element. +} + +impl Element for Path { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut path_type = 0; + let mut width = 0; + let mut bgn_ext = 0; + let mut end_ext = 0; + let (input, mut layer) = LAYER::skip_and_read(input)?; + let (input, mut dtype) = DATATYPE::read(input)?; + + let (input, mut header) = RecordHeader::parse(&input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PATHTYPE => + {let (input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, + records::RTAG_WIDTH => + {let (input, width) = WIDTH::read_data(input, header.data_size)?;}, + records::RTAG_BGNEXTN => + {let (input, bgn_ext) = BGNEXTN::read_data(input, header.data_size)?;}, + records::RTAG_ENDEXTN => + {let (input, end_ext) = ENDEXTN::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + }; + let (input, header) = RecordHeader::parse(&input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + let (input, properties) = read_properties(input)?; + Path{ + layer: (layer, dtype), + xy: xy, + properties: properties, + extension: (bgn_ext, end_ext), + path_type: path_type, + width: width, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += PATH::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += DATATYPE::write(ww, self.layer[1])?; + if self.path_type != 0 { + size += PATHTYPE::write(ww, self.path_type)?; + } + if self.width != 0 { + size += WIDTH::write(ww, self.width)?; + } + + if self.path_type < 4 { + let (bgn_ext, end_ext) = self.extension; + if bgn_ext != 0 { + size += BGNEXTN::write(ww, bgn_ext)?; + } + if end_ext != 0 { + size += ENDEXTN::write(ww, end_ext)?; + } + } + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct GDSBox { + /* + * Datastructure representing a Box element. Rarely used. + */ + layer: (i16, i16), // (layer, box_type) tuple + xy: Vec, // Box coordinates (5 pairs) + properties: HashMap::>, // Properties for the element. +} + +impl Element for GDSBox { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = BOXTYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + GDSBox{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += BOX::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += BOXTYPE::write(ww, self.layer[1])?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct Node { + /* + * Datastructure representing a Node element. Rarely used. + */ + layer: (i16, i16), // (layer, box_type) tuple + xy: Vec, // 1-50 pairs of coordinates. + properties: HashMap::>, // Properties for the element. +} + +impl Element for Node { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = NODETYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + Node{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += NODE::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += NODETYPE::write(ww, self.layer[1])?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + +struct Text { + /* + * Datastructure representing a text label. + */ + layer: (i16, i16), // (layer, node_type) tuple + presentation: [bool; 16], + /* + * Bit array. Default all zeros. + * bits 0-1: 00 left/01 center/10 right + * bits 2-3: 00 top/01 middle/10 bottom + * bits 4-5: font number + */ + path_type: i16, // Default 0 + width: i32, // Default 0 + invert_y: bool, // Vertical inversion. Default false. + mag: f64, // Scaling factor. Default 1. + angle_deg: f64, // Rotation (ccw). Default 0. + xy: Vec, // Position (1 pair only) + string: Vec, // Text content + properties: HashMap::> // Properties for the element. +} + +impl Element for Text { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut path_type = 0; + let mut presentation = 0; + let mut invert_y = false; + let mut width = 0; + let mut mag = 1; + let mut angle_deg = 0; + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = TEXTTYPE::read(input)?; + + let mut header = RecordHeader::parse(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PRESENTATION => + {let (input, presentation) = PRESENTATION::read_data(input, header.data_size)?;}, + records::RTAG_PATHTYPE => + {let (input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, + records::RTAG_WIDTH => + {let (input, width) = WIDTH::read_data(input, header.data_size)?;}, + records::RTAG_STRANS => { + let (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = strans[0]; + }, + records::RTAG_MAG => + {let (input, mag) = MAG::read_data(input, header.data_size)?;}, + records::RTAG_ANGLE => + {let (input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + } + let (input, header) = RecordHeader::parse(input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + + let (input, string) = STRING::read(input)?; + let (input, properties) = read_properties(input)?; + Text{ + layer: (layer, dtype), + xy: xy, + properties: properties, + string: string, + presentation: presentation, + path_type: path_type, + width: width, + invert_y: invert_y, + mag: mag, + angle_deg: angle_deg, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += TEXT::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += TEXTTYPE::write(ww, self.layer[1])?; + if self.presentation != 0 { + size += PRESENTATION::write(ww, self.presentation)?; + } + if self.path_type != 0 { + size += PATHTYPE::write(ww, self.path_type)?; + } + if self.width != 0 { + size += WIDTH::write(ww, self.width)? + } + if self.angle_deg != 0 || self.mag != 1 || self.invert_y { + let strans = [false; 16]; + strans[0] = self.invert_y; + size += STRANS::write(ww, strans)?; + if self.mag != 1 { + size += MAG::write(ww, self.mag)?; + } + if self.angle_deg !=0 { + size += ANGLE::write(ww, self.angle_deg)?; + } + } + size += XY::write(ww, self.xy)?; + size += STRING::write(ww, self.string)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d1c5fd4 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,7 @@ +extern crate nom; +extern crate byteorder; + +pub mod basic; +pub mod record; +pub mod records; +pub mod elements; diff --git a/src/library.rs b/src/library.rs new file mode 100644 index 0000000..7a8573a --- /dev/null +++ b/src/library.rs @@ -0,0 +1,282 @@ +/* + *File-level read/write functionality. + */ + +use nom; +use std::io::Write; +use std::collections:HashMap; + +use record; +use records; +use elements; +//from .records import HEADER, BGNLIB, ENDLIB, UNITS, LIBNAME +//from .records import BGNSTR, STRNAME, ENDSTR, SNAME, COLROW, ENDEL +//from .records import BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF +//from .elements import Element, Reference, Text, Box, Boundary, Path, Node + + +pub struct FileHeader { + /* + * Representation of the GDS file header. + * + * File header records: HEADER BGNLIB LIBNAME UNITS + * Optional records are ignored if present and never written. + * + * Version is written as `600`. + */ + name: Vec, // Library name + user_units_per_db_unit: f64, // Number of user units in one database unit + meters_per_db_unit: f64, // Number of meters in one database unit + mod_time: [u16; 6], // Last-modified time [y, m, d, h, m, s] + acc_time: [u16; 6], // Last-accessed time [y, m, d, h, m, s] +} + +impl FileHeader { + pub fn new(name: &[u8], meters_per_db_unit: f64, user_units_per_db_unit: f64) -> Self { + FileHeader{ + mod_time: [0, 1, 1, 0, 0, 0]; + acc_time: [0, 1, 1, 0, 0, 0]; + name: name.to_owned(), + user_units_per_db_unit: user_units_per_db_unit, + meters_per_db_unit: meters_per_db_unit, + } + } + + pub fn read(input: &[u8]) -> IResult<&[u8], Self> { + /* + * Read and construct a header from the provided stream. + * + * Args: + * stream: Seekable stream to read from + * + * Returns: + * FileHeader object + */ + let (input, version) = records::HEADER.read(input)?; + let (input, (mod_time, acc_time)) = records::BGNLIB.read(input)?; + let (input, name) = records::LIBNAME.skip_and_read(input)?; + let (input, (uu, dbu)) = records::UNITS.skip_and_read(input)?; + + FileHeader{ + mod_time: mod_time, + acc_time: acc_time, + name: name, + user_units_per_db_unit: uu, + meters_per_db_unit: dbu, + } + } + + pub fn write(&self, ww: W) -> OWResult { + /* + * Write the header to a stream + * + * Args: + * stream: Stream to write to + * + * Returns: + * number of bytes written + */ + let mut size = 0; + size += records::HEADER.write(stream, 600) + size += records::BGNLIB.write(stream, [self.mod_time, self.acc_time]) + size += records::LIBNAME.write(stream, self.name) + size += records::UNITS.write(stream, (self.user_units_per_db_unit, self.meters_per_db_unit)) + Ok(size) + } +} + + +pub fn scan_structs(input: &[u8]) -> HashMap::, usize> { + /* + * Scan through a GDS file, building a table of + * {b'structure_name': byte_offset}. + * The intent of this function is to enable random access + * and/or partial (structure-by-structure) reads. + * + * Args: + * stream: Seekable stream to read from. Should be positioned + * before the first structure record, but possibly + * already past the file header. + */ + let input_size = input.len(); + let positions = HashMap{}; + + let (input, header) = RecordHeader.parse(input)?; + while header.tag != records::RTAG_ENDLIB { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + if tag == records::RTAG_BGNSTR { + let (input, name) = records::STRNAME.read(input)?; + if positions.conains_key(name) { + return Err(format!("Duplicate structure name: {}", name)); + } + let position = input_size - input.len(); + positions.insert(name, position); + } + let (input, header) = RecordHeader.parse(input)?; + } + positions +} + + +pub struct Cell { + +} + +impl Cell { + pub fn write( + &self, + ww: Write, + name: &[u8], + cre_time: Option<[i16; 6]>, + mod_time: Option<[i16; 6]>, + ) -> OWResult { + /* + * Write a structure to the provided stream. + * + * Args: + * name: Structure name (ascii-encoded). + * elements: List of Elements containing the geometry and text in this struct. + * cre_time: Creation time (optional). + * mod_time: Modification time (optional). + * + * Return: + * Number of bytes written + */ + let mut size = 0; + size += BGNSTR.write(ww, (cre_time, mod_time)) + size += STRNAME.write(ww, name) + size += cell.write(ww) + size += ENDSTR.write(ww) + Ok(size) + } +} + +pub fn try_read_struct(input: &[u8]) -> IResult<&[u8], Option<(Vec, Cell>)> { + /* + * Skip to the next structure and attempt to read it. + * + * Args: + * stream: Seekable stream to read from. + * + * Returns: + * (name, elements) if a structure was found. + * None if no structure was found before the end of the library. + */ + let (input, success) = records::BGNSTR.skip_past(input)?; + if !success { + return None + } + + let (input, name) = records::STRNAME.read(input)?; + let (input, elements) = Cell::read_elements(input)?; + Some((name, elements)) +} + + + +pub fn read_elements(stream: BinaryIO) -> List[Element] { + /* + * Read elements from the stream until an ENDSTR + * record is encountered. The ENDSTR record is also + * consumed. + * + * Args: + * stream: Seekable stream to read from. + * + * Returns: + * List of element objects. + */ + let (input, header) = RecordHeader.parse(input)?; + while header.tag != records::RTAG_ENDSTR { + match header.tag { + records::RTAG_BOUNDARY => { + let (input, boundary) = records::BOUNDARY.read(input)?; + cell.boundaries.insert(boundary); + }, + records::RTAG_PATH => { + let (input, path) = records::PATH.read(input)?; + cell.paths.insert(path); + }, + records::RTAG_NODE => { + let (input, node) = records::NODE.read(input)?; + cell.nodes.insert(node); + }, + records::RTAG_BOX => { + let (input, gds_box) = records::BOX.read(input)?; + cell.boxes.insert(gds_box); + }, + records::RTAG_TEXT => { + let (input, txt) = records::TEXT.read(input)?; + cell.texts.insert(txt); + }, + records::RTAG_SREF => { + let (input, sref) = records::SREF.read(input)?; + cell.refs.insert(sref); + }, + records::RTAG_AREF => { + let (input, aref) = records::AREF.read(input)?; + cell.refs.insert(aref); + }, + _ => { + // don't care, skip + let (input, _) = nom::bytes::streaming::take(size)(input)?; + } + } + let (input, header) = RecordHeader.parse(input)?; + } + Ok((input, data)) +} + + +pub fn scan_hierarchy(input: &[u8]) -> IResult<&[u8], HashMap::, HashMap::, u32>>> { + /* + * Scan through a GDS file, building a table of instance counts + * `{b'structure_name': {b'ref_name': count}}`. + * + * This is intended to provide a fast overview of the file's + * contents without performing a full read of all elements. + * + * Args: + * stream: Seekable stream to read from. Should be positioned + * before the first structure record, but possibly + * already past the file header. + */ + let structures = HashMap{}; + + let mut ref_name = None + let mut ref_count = None + let mut cur_structure = HashMap{}; + + let (input, header) = Record.read_header(stream) + while header.tag != records::RTAG_ENDLIB { + match header.tag { + records::RTAG_BGNSTR => { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + let (input, name) = records::STRNAME.read(input)?; + if structures.contains_key(name) { + return Err(format!("Duplicate structure name: {}", name)); + } + cur_structure = HashMap{}; + structures.insert(name, cur_structure); + ref_name = None; + ref_count = None; + }, + records::RTAG_SNAME => { + let (input, sname) = SNAME.read_data(input, header.data_size)?; + ref_name = Some(sname); + }, + records::RTAG_COLROW => { + let (input, colrow) = COLROW.read_data(input, header.data_size); + ref_count = colrow[0] * colrow[1]; + }, + records::RTAG_ENDEL => { + *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); + }, + _ => { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + }, + } + let (input, header) = RecordHeader.parse(input)?; + } + structures +} diff --git a/src/record.rs b/src/record.rs new file mode 100644 index 0000000..5f6775b --- /dev/null +++ b/src/record.rs @@ -0,0 +1,219 @@ +/* + * Generic record-level read/write functionality. + */ +use nom::IResult; +use std::io::Write; +use byteorder::BigEndian; + + +use basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use basic::{OWResult}; +use records; +//from .basic import parse_int2, parse_int4, parse_real8, parse_datetime, parse_bitarray +//from .basic import pack_int2, pack_int4, pack_real8, pack_datetime, pack_bitarray +//from .basic import parse_ascii, read + + +//#[no_mangle] +//pub extern "C" fn write_record_header( + + +#[repr(C)] +pub struct RecordHeader { + pub tag: u16, + pub data_size: u16, +} + +impl RecordHeader { + pub fn parse(input: &[u8]) -> IResult<&[u8], RecordHeader> { + let (_, size) = nom::number::streaming::be_u16(input[0..])?; + let (_, tag) = nom::number::streaming::be_u16(input[2..])?; + Ok((input[4..], RecordHeader{tag:tag, data_size:size - 4})) + } + + pub fn pack(self) -> [u8; 4] { + assert!(self.size < 0xffff - 4, "Record too big!"); + let vals = [self.size, self.tag]; + let mut buf = [0x77; 4]; + BigEndian::write_u16_into(&vals, &mut buf); + buf + } + + pub fn write(&self, ww: W) -> OWResult { + let bytes = self.pack(); + ww.write(bytes) + } +} + + +pub trait Record { + fn tag() -> u32; + fn expected_size() -> Option; + + //fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Self>; +} + +impl Record { + pub fn check_size(&self, actual_size: usize) -> Result<(), &str> { + match self.expected_size() { + Some(size) => if size == actual_size { + Ok(()) + } else { + Err(format!("Expected record size {}, got {}", size, actual_size)) + }, + None => Ok(()), + } + } + + pub fn parse_header(input: &[u8]) -> IResult<&[u8], RecordHeader> { + RecordHeader::parse(input) + } + + pub fn write_header(ww: W, data_size: usize) -> OWResult { + RecordHeader{tag: Self.tag(), size: data_size}.write(ww) + } + + pub fn skip_past(input: &[u8]) -> IResult<&[u8], bool> { + /* + * Skip to the end of the next occurence of this record. + * + * Return: + * True if the record was encountered and skipped. + * False if the end of the library was reached. + */ + let (input, header) = RecordHeader::parse(input)?; + while header.tag != Self.tag() { + let (input, _) = nom::bytes::streaming::take(header.size)?; + if header.tag == records::RTAG_ENDLIB { + return Ok((input, false)) + } + let (input, header) = RecordHeader::parse(input)?; + } + let (input, _) = nom::bytes::streaming::take(header.size)?; + Ok((input, true)) + } + + /* + pub fn skip_and_read(input: &[u8]) -> IResult<&[u8], bool>{ + size, tag = Record.read_header(stream) + while tag != cls.tag{ + stream.seek(size, io.SEEK_CUR) + size, tag = Record.read_header(stream) + } + data = cls.read_data(stream, size) + return data + } + + def read(cls: Type[R], stream: BinaryIO){ + size = expect_record(stream, cls.tag) + data = cls.read_data(stream, size) + return data + } + + def write(cls, stream: BinaryIO, data) -> int { + data_bytes = cls.pack_data(data) + b = cls.write_header(stream, len(data_bytes)) + b += stream.write(data_bytes) + return b + } + */ +} + + +pub trait BitArray { + fn parse_data(input: &[u8]) -> IResult<&[u8], [bool; 16]> { + parse_bitarray(input) + } + + fn pack_data(buf: &mut [u8], vals: &[bool; 16]) { + pack_bitarray(&mut buf, vals) + } +} + +pub trait Int2 { + fn parse_data(input: &[u8]) -> IResult<&[u8], i16> { + parse_int2(input) + } + + fn pack_data(buf: &mut [u8], val: i16) { + pack_int2(&mut buf, val) + } +} + +pub trait Int4 { + fn parse_data(input: &[u8]) -> IResult<&[u8], i32> { + parse_int4(input) + } + + fn pack_data(buf: &mut [u8], val: i32) { + pack_int4(&mut buf, val) + } +} + +pub trait Int2Array { + fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Vec> { + assert!(size % 2 == 0, "Record must contain an integer quantity of integers"); + nom::multi::count(parse_int2, size / 2)(input) + } + + fn pack_data(buf: &mut [u8], vals: &[i16]) { + BigEndian::write_i16_into(&vals, &mut buf) + } +} + + +pub trait Int4Array { + fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Vec> { + assert!(size % 4 == 0, "Record must contain an integer quantity of integers"); + nom::multi::count(parse_int4, size / 4)(input) + } + + fn pack_data(buf: &mut [u8], vals: &[i32]) { + BigEndian::write_i32_into(&vals, &mut buf) + } +} + +pub trait Real8 { + fn parse_data(input: &[u8]) -> IResult<&[u8], f64> { + parse_real8(input) + } + + fn pack_data(buf: &mut [u8], val: f64) { + pack_real8(&mut buf, val) + } +} + +pub trait ASCII { + fn parse_data(input: &[u8]) -> IResult<&[u8], Vec> { + parse_ascii(input) + } + + fn pack_data(buf: &mut [u8], data: &[u8]) { + pack_ascii(&mut buf, data) + } +} + +pub trait DateTime { + fn parse_data(input: &[u8]) -> IResult<&[u8], [u16; 6]> { + parse_datetime(input) + } + + fn pack_data(buf: &mut [u8], data: [u16; 6]) { + pack_datetime(&mut buf, data) + } +} + +impl DTR { + fn skip_and_read(input: &[u8]) -> IResult<&[u8], [DTR; 2]> { + let mut header = Self.read_header(input)?; + while header.tag != Self.tag() { + nom::bytes::streaming::take(header.data_size)?; + header = Self.read_header(input)?; + } + assert!(header.data_size == 6 * 2); + let data0 = Self.read_data(&input)?; + let data1 = Self.read_data(&input)?; + Ok([data0, data1]) + } +} diff --git a/src/records.rs b/src/records.rs new file mode 100644 index 0000000..92c5ad4 --- /dev/null +++ b/src/records.rs @@ -0,0 +1,595 @@ +/* + * Record type and tag definitions + */ + +use record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, DateTime, BitArray, ASCII}; +//use basic::{OWResult}; + +//use std::io::Write; + +// record tags +pub const RTAG_HEADER: u16 = 0x0002; +pub const RTAG_BGNLIB: u16 = 0x0102; +pub const RTAG_LIBNAME: u16 = 0x0206; +pub const RTAG_UNITS: u16 = 0x0305; // (user_units_per_db_unit, db_units_per_meter) +pub const RTAG_ENDLIB: u16 = 0x0400; +pub const RTAG_BGNSTR: u16 = 0x0502; +pub const RTAG_STRNAME: u16 = 0x0606; +pub const RTAG_ENDSTR: u16 = 0x0700; +pub const RTAG_BOUNDARY: u16 = 0x0800; +pub const RTAG_PATH: u16 = 0x0900; +pub const RTAG_SREF: u16 = 0x0a00; +pub const RTAG_AREF: u16 = 0x0b00; +pub const RTAG_TEXT: u16 = 0x0c00; +pub const RTAG_LAYER: u16 = 0x0d02; +pub const RTAG_DATATYPE: u16 = 0x0e02; +pub const RTAG_WIDTH: u16 = 0x0f03; +pub const RTAG_XY: u16 = 0x1003; +pub const RTAG_ENDEL: u16 = 0x1100; +pub const RTAG_SNAME: u16 = 0x1206; +pub const RTAG_COLROW: u16 = 0x1302; +pub const RTAG_NODE: u16 = 0x1500; +pub const RTAG_TEXTTYPE: u16 = 0x1602; +pub const RTAG_PRESENTATION: u16 = 0x1701; +pub const RTAG_SPACING: u16 = 0x1802; // unused; not sure about 02 +pub const RTAG_STRING: u16 = 0x1906; +pub const RTAG_STRANS: u16 = 0x1a01; +pub const RTAG_MAG: u16 = 0x1b05; +pub const RTAG_ANGLE: u16 = 0x1c05; +pub const RTAG_UINTEGER: u16 = 0x1d02; // unused; not sure about 02 +pub const RTAG_USTRING: u16 = 0x1e06; // unused; not sure about 06 +pub const RTAG_REFLIBS: u16 = 0x1f06; +pub const RTAG_FONTS: u16 = 0x2006; +pub const RTAG_PATHTYPE: u16 = 0x2102; +pub const RTAG_GENERATIONS: u16 = 0x2202; +pub const RTAG_ATTRTABLE: u16 = 0x2306; +pub const RTAG_STYPTABLE: u16 = 0x2406; // unused; not sure about 06 +pub const RTAG_STRTYPE: u16 = 0x2502; // unused +pub const RTAG_ELFLAGS: u16 = 0x2601; +pub const RTAG_ELKEY: u16 = 0x2703; // unused +pub const RTAG_LINKTYPE: u16 = 0x2803; // unused +pub const RTAG_LINKKEYS: u16 = 0x2903; // unused +pub const RTAG_NODETYPE: u16 = 0x2a02; +pub const RTAG_PROPATTR: u16 = 0x2b02; +pub const RTAG_PROPVALUE: u16 = 0x2c06; +pub const RTAG_BOX: u16 = 0x2d00; +pub const RTAG_BOXTYPE: u16 = 0x2e02; +pub const RTAG_PLEX: u16 = 0x2f03; +pub const RTAG_BGNEXTN: u16 = 0x3003; +pub const RTAG_ENDEXTN: u16 = 0x3103; +pub const RTAG_TAPENUM: u16 = 0x3202; +pub const RTAG_TAPECODE: u16 = 0x3302; +pub const RTAG_STRCLASS: u16 = 0x3401; +pub const RTAG_RESERVED: u16 = 0x3503; +pub const RTAG_FORMAT: u16 = 0x3602; +pub const RTAG_MASK: u16 = 0x3706; // list of Layers and dtypes +pub const RTAG_ENDMASKS: u16 = 0x3800; // end of MASKS records +pub const RTAG_LIBDIRSIZE: u16 = 0x3902; +pub const RTAG_SRFNAME: u16 = 0x3a06; +pub const RTAG_LIBSECUR: u16 = 0x3b02; +pub const RTAG_BORDER: u16 = 0x3c00; +pub const RTAG_SOFTFENCE: u16 = 0x3d00; +pub const RTAG_HARDFENCE: u16 = 0x3f00; +pub const RTAG_SOFTWIRE: u16 = 0x3f00; +pub const RTAG_HARDWIRE: u16 = 0x4000; +pub const RTAG_PATHPORT: u16 = 0x4100; +pub const RTAG_NODEPORT: u16 = 0x4200; +pub const RTAG_USERCONSTRAINT: u16 = 0x4300; +pub const RTAG_SPACERERROR: u16 = 0x4400; +pub const RTAG_CONTACT: u16 = 0x4500; + +/* +// data types +pub const DATA_TYPE_NONE: u16 = 0x00; +pub const DATA_TYPE_BIT: u16 = 0x01; +pub const DATA_TYPE_INT16: u16 = 0x02; +pub const DATA_TYPE_INT32: u16 = 0x03; +pub const DATA_TYPE_REAL32: u16 = 0x04; +pub const DATA_TYPE_REAL64: u16 = 0x05; +pub const DATA_TYPE_STR: u16 = 0x06; + +pub const MAX_DATA_SIZE: usize = 8; + +/// Returns the size of the given data type in bytes. +pub fn data_size(t: u16) -> usize { + match t { + x if x == DATA_TYPE_NONE => 0, + x if x == DATA_TYPE_BIT => 2, + x if x == DATA_TYPE_INT16 => 2, + x if x == DATA_TYPE_INT32 => 4, + x if x == DATA_TYPE_REAL32 => 4, + x if x == DATA_TYPE_REAL64 => 8, + _ => 0 + } +*/ + +pub struct HEADER {} +impl Record for HEADER { + fn tag() -> u16 { RTAG_HEADER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for HEADER {} + +pub struct BGNLIB {} +impl Record for BGNLIB { + fn tag() -> u16 { RTAG_BGNLIB } + fn expected_size() -> usize { Some(2 * 6) } +} +impl DateTime for BGNLIB {} + +pub struct LIBNAME {} +impl Record for LIBNAME { + fn tag() -> u16 { RTAG_LIBNAME } + fn expected_size() -> usize { None } +} +impl ASCII for LIBNAME {} + +pub struct UNITS {} +impl Record for UNITS { + // (user_units_per_db_unit, db_units_per_meter) + fn tag() -> u16 { RTAG_UNITS } + fn expected_size() -> usize { Some(2 * 8) } +} +impl Real8 for UNITS {} + +pub struct ENDLIB {} +impl Record for ENDLIB { + fn tag() -> u16 { RTAG_ENDLIB } + fn expected_size() -> usize { Some(0) } +} + +pub struct BGNSTR {} +impl Record for BGNSTR { + fn tag() -> u16 { RTAG_BGNSTR } + fn expected_size() -> usize { Some(2 * 6) } +} +impl DateTime for ENDLIB {} + +pub struct STRNAME {} +impl Record for STRNAME { + fn tag() -> u16 { RTAG_STRNAME } + fn expected_size() -> usize { Some(2 * 6) } +} +impl ASCII for STRNAME {} + +pub struct ENDSTR {} +impl Record for ENDSTR { + fn tag() -> u16 { RTAG_ENDSTR } + fn expected_size() -> usize { Some(0) } +} + +pub struct BOUNDARY {} +impl Record for BOUNDARY { + fn tag() -> u16 { RTAG_BOUNDARY } + fn expected_size() -> usize { Some(0) } +} + +pub struct PATH {} +impl Record for PATH { + fn tag() -> u16 { RTAG_PATH } + fn expected_size() -> usize { Some(0) } +} + +pub struct SREF {} +impl Record for SREF { + fn tag() -> u16 { RTAG_SREF } + fn expected_size() -> usize { Some(0) } +} + +pub struct AREF {} +impl Record for AREF { + fn tag() -> u16 { RTAG_AREF } + fn expected_size() -> usize { Some(0) } +} + +pub struct TEXT {} +impl Record for TEXT { + fn tag() -> u16 { RTAG_TEXT } + fn expected_size() -> usize { Some(0) } +} + +pub struct LAYER {} +impl Record for LAYER { + fn tag() -> u16 { RTAG_LAYER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LAYER {} + +pub struct DATATYPE {} +impl Record for DATATYPE { + fn tag() -> u16 { RTAG_DATATYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for DATATYPE {} + +pub struct WIDTH {} +impl Record for WIDTH { + fn tag() -> u16 { RTAG_WIDTH } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for WIDTH {} + +pub struct XY {} +impl Record for XY { + fn tag() -> u16 { RTAG_XY } + fn expected_size() -> usize { None } +} +impl Int4Array for XY {} + +pub struct ENDEL {} +impl Record for ENDEL { + fn tag() -> u16 { RTAG_ENDEL } + fn expected_size() -> usize { Some(0) } +} + +pub struct SNAME {} +impl Record for SNAME { + fn tag() -> u16 { RTAG_SNAME } + fn expected_size() -> usize { None } +} +impl ASCII for SNAME {} + +pub struct COLROW {} +impl Record for COLROW { + fn tag() -> u16 { RTAG_COLROW } + fn expected_size() -> usize { Some(4) } +} +impl Int2Array for COLROW {} + +pub struct NODE {} +impl Record for NODE { + fn tag() -> u16 { RTAG_NODE } + fn expected_size() -> usize { Some(0) } +} + +pub struct TEXTTYPE {} +impl Record for TEXTTYPE { + fn tag() -> u16 { RTAG_TEXTTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for TEXTTYPE {} + +pub struct PRESENTATION {} +impl Record for PRESENTATION { + fn tag() -> u16 { RTAG_PRESENTATION } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for PRESENTATION {} + +pub struct SPACING {} +impl Record for SPACING { + fn tag() -> u16 { RTAG_SPACING } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for SPACING {} + +pub struct STRING {} +impl Record for STRING { + fn tag() -> u16 { RTAG_STRING } + fn expected_size() -> usize { None } +} +impl ASCII for STRING {} + +pub struct STRANS {} +impl Record for STRANS { + fn tag() -> u16 { RTAG_STRANS } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for STRANS {} + +pub struct MAG {} +impl Record for MAG { + fn tag() -> u16 { RTAG_MAG } + fn expected_size() -> usize { Some(8) } +} +impl Real8 for MAG {} + +pub struct ANGLE {} +impl Record for ANGLE { + fn tag() -> u16 { RTAG_ANGLE } + fn expected_size() -> usize { Some(8) } +} +impl Real8 for ANGLE {} + +pub struct UINTEGER {} +impl Record for UINTEGER { + fn tag() -> u16 { RTAG_UINTEGER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for UINTEGER {} + +pub struct USTRING {} +impl Record for USTRING { + fn tag() -> u16 { RTAG_USTRING } + fn expected_size() -> usize { None } +} +impl ASCII for USTRING {} + +pub struct REFLIBS {} +impl Record for REFLIBS { + fn tag() -> u16 { RTAG_REFLIBS } + fn expected_size() -> usize { None } +} +impl REFLIBS { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for REFLIBS {} + +pub struct FONTS {} +impl Record for FONTS { + fn tag() -> u16 { RTAG_FONTS } + fn expected_size() -> usize { None } +} +impl FONTS { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for FONTS {} + +pub struct PATHTYPE {} +impl Record for PATHTYPE { + fn tag() -> u16 { RTAG_PATHTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for PATHTYPE {} + +pub struct GENERATIONS {} +impl Record for GENERATIONS { + fn tag() -> u16 { RTAG_GENERATIONS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for GENERATIONS {} + +pub struct ATTRTABLE {} +impl Record for ATTRTABLE { + fn tag() -> u16 { RTAG_ATTRTABLE } + fn expected_size() -> usize { None } +} +impl ATTRTABLE { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for ATTRTABLE {} + +pub struct STYPTABLE {} +impl Record for STYPTABLE { + fn tag() -> u16 { RTAG_STYPTABLE } + fn expected_size() -> usize { None } +} +impl ASCII for STYPTABLE {} + +pub struct STRTYPE {} +impl Record for STRTYPE { + fn tag() -> u16 { RTAG_STRTYPE } + fn expected_size() -> usize { None } +} +impl Int2 for STRTYPE {} + +pub struct ELFLAGS {} +impl Record for ELFLAGS { + fn tag() -> u16 { RTAG_ELFLAGS } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for ELFLAGS {} + +pub struct ELKEY {} +impl Record for ELKEY { + fn tag() -> u16 { RTAG_ELKEY } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for ELKEY {} + +pub struct LINKTYPE {} +impl Record for LINKTYPE { + fn tag() -> u16 { RTAG_LINKTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LINKTYPE {} + +pub struct LINKKEYS {} +impl Record for LINKKEYS { + fn tag() -> u16 { RTAG_LINKKEYS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LINKKEYS {} + +pub struct NODETYPE {} +impl Record for NODETYPE { + fn tag() -> u16 { RTAG_NODETYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for NODETYPE {} + +pub struct PROPATTR {} +impl Record for PROPATTR { + fn tag() -> u16 { RTAG_PROPATTR } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for PROPATTR {} + +pub struct PROPVALUE {} +impl Record for PROPVALUE { + fn tag() -> u16 { RTAG_PROPVALUE } + fn expected_size() -> usize { Some(2) } +} +impl ASCII for PROPVALUE {} + +pub struct BOX {} +impl Record for BOX { + fn tag() -> u16 { RTAG_BOX } + fn expected_size() -> usize { Some(0) } +} + +pub struct BOXTYPE {} +impl Record for BOXTYPE { + fn tag() -> u16 { RTAG_BOXTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for BOXTYPE {} + +pub struct PLEX {} +impl Record for PLEX { + fn tag() -> u16 { RTAG_PLEX } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for PLEX {} + +pub struct BGNEXTN {} +impl Record for BGNEXTN { + fn tag() -> u16 { RTAG_BGNEXTN } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for BGNEXTN {} + +pub struct ENDEXTN {} +impl Record for ENDEXTN { + fn tag() -> u16 { RTAG_ENDEXTN } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for ENDEXTN {} + +pub struct TAPENUM {} +impl Record for TAPENUM { + fn tag() -> u16 { RTAG_TAPENUM } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for TAPENUM {} + +pub struct TAPECODE {} +impl Record for TAPECODE { + fn tag() -> u16 { RTAG_TAPECODE } + fn expected_size() -> usize { Some(2 * 6) } +} +impl Int2Array for TAPECODE {} + +pub struct STRCLASS {} +impl Record for STRCLASS { + fn tag() -> u16 { RTAG_STRCLASS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for STRCLASS {} + +pub struct RESERVED {} +impl Record for RESERVED { + fn tag() -> u16 { RTAG_RESERVED } + fn expected_size() -> usize { Some(2) } +} +impl Int2Array for RESERVED {} + +pub struct FORMAT {} +impl Record for FORMAT { + fn tag() -> u16 { RTAG_FORMAT } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for FORMAT {} + +pub struct MASK {} +impl Record for MASK { + fn tag() -> u16 { RTAG_MASK } + fn expected_size() -> usize { None } +} +impl ASCII for MASK {} + +pub struct ENDMASKS {} +impl Record for ENDMASKS { + // End of MASKS records + fn tag() -> u16 { RTAG_ENDMASKS } + fn expected_size() -> usize { Some(0) } +} + +pub struct LIBDIRSIZE {} +impl Record for LIBDIRSIZE { + fn tag() -> u16 { RTAG_LIBDIRSIZE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LIBDIRSIZE {} + +pub struct SRFNAME {} +impl Record for SRFNAME { + fn tag() -> u16 { RTAG_SRFNAME } + fn expected_size() -> usize { None } +} +impl ASCII for SRFNAME {} + +pub struct LIBSECUR {} +impl Record for LIBSECUR { + fn tag() -> u16 { RTAG_LIBSECUR } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LIBSECUR {} + +pub struct BORDER {} +impl Record for BORDER { + fn tag() -> u16 { RTAG_BORDER } + fn expected_size() -> usize { Some(0) } +} + +pub struct SOFTFENCE {} +impl Record for SOFTFENCE { + fn tag() -> u16 { RTAG_SOFTFENCE } + fn expected_size() -> usize { Some(0) } +} + +pub struct HARDFENCE {} +impl Record for HARDFENCE { + fn tag() -> u16 { RTAG_HARDFENCE } + fn expected_size() -> usize { Some(0) } +} + +pub struct SOFTWIRE {} +impl Record for SOFTWIRE { + fn tag() -> u16 { RTAG_SOFTWIRE } + fn expected_size() -> usize { Some(0) } +} + +pub struct HARDWIRE {} +impl Record for HARDWIRE { + fn tag() -> u16 { RTAG_HARDWIRE } + fn expected_size() -> usize { Some(0) } +} + +pub struct PATHPORT {} +impl Record for PATHPORT { + fn tag() -> u16 { RTAG_PATHPORT } + fn expected_size() -> usize { Some(0) } +} + +pub struct NODEPORT {} +impl Record for NODEPORT { + fn tag() -> u16 { RTAG_NODEPORT } + fn expected_size() -> usize { Some(0) } +} + +pub struct USERCONSTRAINT {} +impl Record for USERCONSTRAINT { + fn tag() -> u16 { RTAG_USERCONSTRAINT } + fn expected_size() -> usize { Some(0) } +} + +pub struct SPACERERROR {} +impl Record for SPACERERROR { + fn tag() -> u16 { RTAG_SPACERERROR } + fn expected_size() -> usize { Some(0) } +} + +pub struct CONTACT {} +impl Record for CONTACT { + fn tag() -> u16 { RTAG_CONTACT } + fn expected_size() -> usize { Some(0) } +} diff --git a/src/test_basic.rs b/src/test_basic.rs new file mode 100644 index 0000000..6bd8bee --- /dev/null +++ b/src/test_basic.rs @@ -0,0 +1,121 @@ +/* +import struct + +import pytest # type: ignore +import numpy # type: ignore +from numpy.testing import assert_array_equal # type: ignore + +from .basic import parse_bitarray, parse_int2, parse_int4, parse_real8, parse_ascii +from .basic import pack_bitarray, pack_int2, pack_int4, pack_real8, pack_ascii +from .basic import decode_real8, encode_real8 + +from .basic import KlamathError + + +def test_parse_bitarray(): + assert(parse_bitarray(b'59') == 13625) + assert(parse_bitarray(b'\0\0') == 0) + assert(parse_bitarray(b'\xff\xff') == 65535) + + # 4 bytes (too long) + with pytest.raises(KlamathError): + parse_bitarray(b'4321') + + # empty data + with pytest.raises(KlamathError): + parse_bitarray(b'') + + +def test_parse_int2(): + assert_array_equal(parse_int2(b'59\xff\xff\0\0'), (13625, -1, 0)) + + # odd length + with pytest.raises(KlamathError): + parse_int2(b'54321') + + # empty data + with pytest.raises(KlamathError): + parse_int2(b'') + + +def test_parse_int4(): + assert_array_equal(parse_int4(b'4321'), (875770417,)) + + # length % 4 != 0 + with pytest.raises(KlamathError): + parse_int4(b'654321') + + # empty data + with pytest.raises(KlamathError): + parse_int4(b'') + + +def test_decode_real8(): + # zeroes + assert(decode_real8(numpy.array([0x0])) == 0) + assert(decode_real8(numpy.array([1<<63])) == 0) # negative + assert(decode_real8(numpy.array([0xff << 56])) == 0) # denormalized + + assert(decode_real8(numpy.array([0x4110 << 48])) == 1.0) + assert(decode_real8(numpy.array([0xC120 << 48])) == -2.0) + + +def test_parse_real8(): + packed = struct.pack('>3Q', 0x0, 0x4110_0000_0000_0000, 0xC120_0000_0000_0000) + assert_array_equal(parse_real8(packed), (0.0, 1.0, -2.0)) + + # length % 8 != 0 + with pytest.raises(KlamathError): + parse_real8(b'0987654321') + + # empty data + with pytest.raises(KlamathError): + parse_real8(b'') + + +def test_parse_ascii(): + # empty data + with pytest.raises(KlamathError): + parse_ascii(b'') + + assert(parse_ascii(b'12345') == b'12345') + assert(parse_ascii(b'12345\0') == b'12345') # strips trailing null byte + + +def test_pack_bitarray(): + packed = pack_bitarray(321) + assert(len(packed) == 2) + assert(packed == struct.pack('>H', 321)) + + +def test_pack_int2(): + packed = pack_int2((3, 2, 1)) + assert(len(packed) == 3*2) + assert(packed == struct.pack('>3h', 3, 2, 1)) + assert(pack_int2([-3, 2, -1]) == struct.pack('>3h', -3, 2, -1)) + + +def test_pack_int4(): + packed = pack_int4((3, 2, 1)) + assert(len(packed) == 3*4) + assert(packed == struct.pack('>3l', 3, 2, 1)) + assert(pack_int4([-3, 2, -1]) == struct.pack('>3l', -3, 2, -1)) + + +def test_encode_real8(): + assert(encode_real8(numpy.array([0.0])) == 0) + arr = numpy.array((1.0, -2.0, 1e-9, 1e-3, 1e-12)) + assert_array_equal(decode_real8(encode_real8(arr)), arr) + + +def test_pack_real8(): + reals = (0, 1, -1, 0.5, 1e-9, 1e-3, 1e-12) + packed = pack_real8(reals) + assert(len(packed) == len(reals) * 8) + assert_array_equal(parse_real8(packed), reals) + + +def test_pack_ascii(): + assert(pack_ascii(b'4321') == b'4321') + assert(pack_ascii(b'321') == b'321\0') +*/ From e0a634f47665427046499f8afd3c1cbfadef058a Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 18 Dec 2021 21:17:43 -0800 Subject: [PATCH 02/18] add gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d80173 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target/ +**/*.rs.bk +Cargo.lock + +*.swp +*.swo From 00a29941b7f393a05f03a3f4760e627f1972511d Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 20 Mar 2022 16:28:34 -0700 Subject: [PATCH 03/18] index on (no branch): 48db472 snapshot 2021-12-18 21:05:00.635887 From 02e483c8cdaba081518076aeeecaba41265225f7 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 8 May 2022 16:41:43 -0700 Subject: [PATCH 04/18] update to 2021 edition --- Cargo.toml | 3 ++- src/__init__.rs | 29 ----------------------------- src/elements.rs | 8 ++++---- src/lib.rs | 1 - src/library.rs | 27 ++++++++++++++------------- src/record.rs | 8 ++++---- src/records.rs | 2 +- 7 files changed, 25 insertions(+), 53 deletions(-) delete mode 100644 src/__init__.rs diff --git a/Cargo.toml b/Cargo.toml index 414196a..01429e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,8 @@ name = "rs-klamath" version = "0.1.0" authors = ["jan "] +edition = "2021" [dependencies] byteorder = "^1" -nom = "^7" +#nom = "^7" diff --git a/src/__init__.rs b/src/__init__.rs deleted file mode 100644 index 44d6c6b..0000000 --- a/src/__init__.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * `klamath` is a Python module for reading and writing to the GDSII file format. - * - * The goal is to keep this library simple: - * - Map data types directly wherever possible. - * * Presents an accurate representation of what is saved to the file. - * * Avoids excess copies / allocations for speed. - * * No "automatic" error checking, except when casting datatypes. - * If data integrity checks are provided at all, they must be - * explicitly run by the caller. - * - Low-level functionality is first-class. - * * Meant for use-cases where the caller wants to read or write - * individual GDS records. - * * Offers complete control over the written file. - * - Opinionated and limited high-level functionality. - * * Discards or ignores rarely-encountered data types. - * * Keeps functions simple and reusable. - * * Only de/encodes the file format, doesn't provide tools to modify - * the data itself. - * * Still requires explicit values for most fields. - * - No compilation - * * Uses `numpy` for speed, since it's commonly available / pre-built. - * * Building this library should not require a compiler. - * - * `klamath` was built to provide a fast and versatile GDS interface for - * [masque](https://mpxd.net/code/jan/masque), which provides higher-level - * tools for working with hierarchical design data and supports multiple - * file formats. - */ diff --git a/src/elements.rs b/src/elements.rs index 8c2e35b..516a622 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -3,16 +3,16 @@ /// structure references) and associated properties. /// -use records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, +use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, ENDEL, BGNEXTN, ENDEXTN, SNAME, }; -use records; -use record::{RecordHeader, Record}; -use basic::{OResult, IResult, fail}; +use crate::records; +use crate::record::{RecordHeader, Record}; +use crate::basic::{OResult, IResult, fail}; use std::collections::HashMap; use std::io::Write; diff --git a/src/lib.rs b/src/lib.rs index b17d455..dbaf483 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,4 @@ //#![feature(generic_associated_types)] -#![feature(destructuring_assignment)] extern crate byteorder; diff --git a/src/library.rs b/src/library.rs index 06ff7ba..84d74dc 100644 --- a/src/library.rs +++ b/src/library.rs @@ -5,12 +5,12 @@ use std::io::Write; use std::collections::HashMap; -use record; -use record::{RecordHeader, Record}; -use records; -use elements; -use elements::{Element}; -use basic::{IResult, OResult, take_bytes, fail}; +use crate::record; +use crate::record::{RecordHeader, Record}; +use crate::records; +use crate::elements; +use crate::elements::{Element}; +use crate::basic::{IResult, OResult, take_bytes, fail}; const DEFAULT_DATE: [i16; 6] = [1900, 0, 0, 0, 0, 0]; @@ -289,6 +289,7 @@ impl Cell { } +/* /// /// Scan through a GDS file, building a table of instance counts /// `{b'structure_name': {b'ref_name': count}}`. @@ -304,8 +305,10 @@ impl Cell { pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap::, u32>>> { let mut structures = HashMap::new(); - + let mut ref_name = None; + let mut ref_count = None; let (mut input, mut header) = RecordHeader::read(input)?; + let mut cur_structure = HashMap::new(); while header.tag != records::RTAG_ENDLIB { match header.tag { records::RTAG_BGNSTR => { @@ -317,8 +320,6 @@ pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap:: IResult, HashMap:: IResult, u32))> { let (input, found_struc) = records::BGNSTR.skip_past(input)?; - if not found_struc { + if !found_struc { return Ok((input, None)) } let mut cur_structure = HashMap::new(); @@ -390,3 +390,4 @@ pub fn count_ref(input: &[u8]) -> IResult, u32))> { structures.insert(name, cur_structure); (input, header) = RecordHeader::read(input1)?; } +*/ diff --git a/src/record.rs b/src/record.rs index 8d01c93..7d04085 100644 --- a/src/record.rs +++ b/src/record.rs @@ -6,10 +6,10 @@ use std::convert::TryInto; use byteorder::{ByteOrder, BigEndian}; -use basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use basic::{OResult, IResult, fail, parse_u16, take_bytes}; -use records; +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; +use crate::records; //#[no_mangle] diff --git a/src/records.rs b/src/records.rs index 972cc78..21a5bb0 100644 --- a/src/records.rs +++ b/src/records.rs @@ -2,7 +2,7 @@ /// Record type and tag definitions /// -use record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; +use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; //use std::io::Write; From 921afa556958054e60fe86ccb706b9ec4439bb32 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 20 May 2022 19:44:17 -0700 Subject: [PATCH 05/18] whitespace --- src/elements.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/elements.rs b/src/elements.rs index 516a622..b711da2 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,11 +4,11 @@ /// use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, - DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, - LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, - STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, - ENDEL, BGNEXTN, ENDEXTN, SNAME, - }; + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; use crate::records; use crate::record::{RecordHeader, Record}; From e5e0adab7119bd0899059a4b20057e9326a52f3e Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:38:40 -0800 Subject: [PATCH 06/18] fixup imports/exports --- src/basic.rs | 12 ++++++------ src/library.rs | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index d8fa7da..cd22d6f 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -232,7 +232,7 @@ pub fn encode_real8(fnum: f64) -> u64 { mod tests { #[test] fn test_parse_bitarray() { - use basic::parse_bitarray; + use crate::basic::parse_bitarray; //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); @@ -251,7 +251,7 @@ mod tests { #[test] fn test_parse_int2() { - use basic::parse_int2; + use crate::basic::parse_int2; assert_eq!(parse_int2(b"59").unwrap().1, 13625); assert_eq!(parse_int2(b"\0\0").unwrap().1, 0); assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1); @@ -259,13 +259,13 @@ mod tests { #[test] fn test_parse_int4() { - use basic::parse_int4; + use crate::basic::parse_int4; assert_eq!(parse_int4(b"4321").unwrap().1, 875770417); } #[test] fn test_decode_real8() { - use basic::decode_real8; + use crate::basic::decode_real8; // zeroes assert_eq!(decode_real8(0x0), 0.0); @@ -280,7 +280,7 @@ mod tests { #[test] fn test_parse_real8() { - use basic:: parse_real8; + use crate::basic:: parse_real8; assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1); assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1); @@ -289,7 +289,7 @@ mod tests { #[test] fn test_parse_ascii() { - use basic::parse_ascii; + use crate::basic::parse_ascii; assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345"); assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte diff --git a/src/library.rs b/src/library.rs index 84d74dc..e5b5758 100644 --- a/src/library.rs +++ b/src/library.rs @@ -5,12 +5,12 @@ use std::io::Write; use std::collections::HashMap; -use crate::record; -use crate::record::{RecordHeader, Record}; -use crate::records; -use crate::elements; -use crate::elements::{Element}; -use crate::basic::{IResult, OResult, take_bytes, fail}; +pub use crate::record; +pub use crate::record::{RecordHeader, Record}; +pub use crate::records; +pub use crate::elements; +pub use crate::elements::{Element}; +pub use crate::basic::{IResult, OResult, take_bytes, fail}; const DEFAULT_DATE: [i16; 6] = [1900, 0, 0, 0, 0, 0]; From 24d8e32173ee8538a9110ca2d4c2129c345a4542 Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:38:50 -0800 Subject: [PATCH 07/18] add leading zeros to mask --- src/basic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic.rs b/src/basic.rs index cd22d6f..0899fa9 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -161,7 +161,7 @@ pub fn encode_real8(fnum: f64) -> u64 { let ieee = fnum.to_bits(); let sign = ieee & 0x8000_0000_0000_0000; let ieee_exp = (ieee >> 52) as i32 & 0x7ff; - let ieee_mant = ieee & 0xf_ffff_ffff_ffff; + let ieee_mant = ieee & 0x000f_ffff_ffff_ffff; let subnorm = (ieee_exp == 0) & (ieee_mant != 0); if (ieee_exp == 0) & (ieee_mant == 0) { From 6531bec6e5282e86a7e9738b3d65fc2d828958ae Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:39:02 -0800 Subject: [PATCH 08/18] ErrType should be Debug --- src/basic.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/basic.rs b/src/basic.rs index 0899fa9..0fa8392 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -8,6 +8,7 @@ pub type OResult = Result; pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>; +#[derive(Debug)] pub enum ErrType { Incomplete(Option), Failed(String), From 6186f1d5613a9bb94652598da6f57317e307b95e Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:43:41 -0800 Subject: [PATCH 09/18] enable more tests --- src/basic.rs | 81 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 0fa8392..72fd03a 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -297,43 +297,80 @@ mod tests { assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456"); } -/* + #[test] fn test_pack_bitarray() { - packed = pack_bitarray(321) - assert_eq!(len(packed), 2); - assert_eq!(packed, struct.pack('>H', 321)); + use crate::basic::pack_bitarray; + let mut buf = [10; 3]; + let mut bools = [false; 16]; + bools[1] = true; + bools[2] = true; + bools[11] = true; + + pack_bitarray(&mut buf, &bools); + assert_eq!(buf[0], 0b0110_0000); + assert_eq!(buf[1], 0b0001_0000); + assert_eq!(buf[2], 10); } + #[test] fn test_pack_int2() { - packed = pack_int2((3, 2, 1)) - assert(len(packed) == 3*2) - assert(packed == struct.pack('>3h', 3, 2, 1)) - assert(pack_int2([-3, 2, -1]) == struct.pack('>3h', -3, 2, -1)) + use crate::basic::pack_int2; + let mut buf = [10; 3 * 2]; + pack_int2(&mut buf, -3); + pack_int2(&mut buf[2..], 2); + pack_int2(&mut buf[4..], -1); + assert_eq!(buf[0..2], [0xFF, 0xFD]); + assert_eq!(buf[2..4], [0x00, 0x02]); + assert_eq!(buf[4..6], [0xFF, 0xFF]); } + #[test] fn test_pack_int4() { - packed = pack_int4((3, 2, 1)) - assert(len(packed) == 3*4) - assert(packed == struct.pack('>3l', 3, 2, 1)) - assert(pack_int4([-3, 2, -1]) == struct.pack('>3l', -3, 2, -1)) + use crate::basic::pack_int4; + let mut buf = [10; 3 * 4]; + pack_int4(&mut buf, -3); + pack_int4(&mut buf[4..], 2); + pack_int4(&mut buf[8..], -1); + assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]); + assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]); + assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]); } + #[test] fn test_encode_real8() { - assert(encode_real8(numpy.array([0.0])) == 0) - arr = numpy.array((1.0, -2.0, 1e-9, 1e-3, 1e-12)) - assert_array_equal(decode_real8(encode_real8(arr)), arr) + use crate::basic::{encode_real8, decode_real8}; + const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; + for vv in REALS { + print!("{vv}\n"); + assert!((decode_real8(encode_real8(vv)) - vv).abs() < f64::EPSILON); + } } + #[test] fn test_pack_real8() { - reals = (0, 1, -1, 0.5, 1e-9, 1e-3, 1e-12) - packed = pack_real8(reals) - assert(len(packed) == len(reals) * 8) - assert_array_equal(parse_real8(packed), reals) + use crate::basic::{pack_real8, parse_real8}; + const COUNT: usize = 7; + const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; + let mut buf = [10; 8 * COUNT]; + for (ii, &vv) in REALS.iter().enumerate() { + pack_real8(&mut buf[ii * 8..], vv); + } + for (ii, &vv) in REALS.iter().enumerate() { + print!("{vv}\n"); + let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1; + assert!((parsed_val - vv).abs() < f64::EPSILON); + } } + #[test] fn test_pack_ascii() { - assert(pack_ascii(b'4321') == b'4321') - assert(pack_ascii(b'321') == b'321\0') + use crate::basic::pack_ascii; + let mut buf = [10; 12]; + pack_ascii(&mut buf[0..], "4321".as_bytes()); + pack_ascii(&mut buf[6..], "321".as_bytes()); + assert_eq!(&buf[0..4], "4321".as_bytes()); + assert_eq!(&buf[4..6], [10, 10]); + assert_eq!(&buf[6..9], "321".as_bytes()); + assert_eq!(&buf[9..], [0, 10, 10]); } -*/ } From b049f70dce1a20899340f33e060fa0c60e0e1876 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:43:50 -0800 Subject: [PATCH 10/18] check_size should be public --- src/records.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/records.rs b/src/records.rs index 21a5bb0..4fe21fa 100644 --- a/src/records.rs +++ b/src/records.rs @@ -290,7 +290,7 @@ impl Record for REFLIBS { fn expected_size() -> Option { None } } impl REFLIBS { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { @@ -305,7 +305,7 @@ impl Record for FONTS { fn expected_size() -> Option { None } } impl FONTS { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { @@ -332,7 +332,7 @@ impl Record for ATTRTABLE { fn expected_size() -> Option { None } } impl ATTRTABLE { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { From a1e9a3f90ca0e8d832ef2ad39618c0096e9f4c28 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:44:05 -0800 Subject: [PATCH 11/18] fix incomplete condition --- src/basic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic.rs b/src/basic.rs index 72fd03a..0bd0636 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -25,7 +25,7 @@ pub fn incomplete(input: &[u8], size: Option) -> IResult { pub fn take_bytes>(input: &[u8], count: CC) -> IResult<&[u8]> { let cc = count.into(); - if input.len() > cc { + if input.len() < cc { incomplete(input, Some(cc)) } else { let (taken, input) = input.split_at(cc); From babd7f14025b15a6aeab621e1d812b3026fba609 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:51:42 -0800 Subject: [PATCH 12/18] test for panic if float is too large for real8 --- src/basic.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 0bd0636..016cc6f 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -275,10 +275,16 @@ mod tests { assert_eq!(decode_real8(0x4110 << 48), 1.0); assert_eq!(decode_real8(0xC120 << 48), -2.0); - - //TODO panics on invalid? } + #[test] + #[should_panic] + fn test_encode_real8_panic() { + use crate::basic::encode_real8; + encode_real8(1e80); + } + + #[test] fn test_parse_real8() { use crate::basic:: parse_real8; From e42ac03c95aa96d38d5637f3e639fe3d8703ad3e Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 17:27:10 -0800 Subject: [PATCH 13/18] add FloatTooBigError --- src/basic.rs | 39 +++++++++++++++++++++++++-------------- src/record.rs | 6 +++--- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 016cc6f..1019be1 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -3,6 +3,7 @@ /// use byteorder::{ByteOrder, BigEndian}; use std::io; +use std::fmt; pub type OResult = Result; @@ -14,7 +15,6 @@ pub enum ErrType { Failed(String), } - pub fn fail(input: &[u8], msg: String) -> IResult { Err((input, ErrType::Failed(msg))) } @@ -118,20 +118,21 @@ pub fn bitarray2int(bits: &[bool; 16]) -> u16 { } pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { - BigEndian::write_u16(buf, bitarray2int(bits)) + BigEndian::write_u16(buf, bitarray2int(bits)); } pub fn pack_int2(buf: &mut [u8], int: i16) { - BigEndian::write_i16(buf, int) + BigEndian::write_i16(buf, int); } pub fn pack_int4(buf: &mut [u8], int: i32) { - BigEndian::write_i32(buf, int) + BigEndian::write_i32(buf, int); } -pub fn pack_real8(buf: &mut [u8], fnum: f64) { - BigEndian::write_u64(buf, encode_real8(fnum)) +pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { + BigEndian::write_u64(buf, encode_real8(fnum)?); + Ok(()) } pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { @@ -156,8 +157,18 @@ pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) { } +#[derive(Debug, Clone)] +pub struct FloatTooBigError { + float_value: f64, +} +impl fmt::Display for FloatTooBigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Float {0} is too large for Real8", self.float_value) + } +} + /// Convert from float64 to GDS REAL8 representation. -pub fn encode_real8(fnum: f64) -> u64 { +pub fn encode_real8(fnum: f64) -> Result { // Split the ieee float bitfields let ieee = fnum.to_bits(); let sign = ieee & 0x8000_0000_0000_0000; @@ -166,7 +177,7 @@ pub fn encode_real8(fnum: f64) -> u64 { let subnorm = (ieee_exp == 0) & (ieee_mant != 0); if (ieee_exp == 0) & (ieee_mant == 0) { - return 0 + return Ok(0) } // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) @@ -207,7 +218,7 @@ pub fn encode_real8(fnum: f64) -> u64 { if gds_exp < -14 { // number is too small - return 0 + return Ok(0) } let neg_biased = gds_exp < 0; @@ -218,13 +229,13 @@ pub fn encode_real8(fnum: f64) -> u64 { let too_big = (gds_exp > 0x7f) & !subnorm; if too_big { - panic!("Number too big for real8 format"); //TODO error handling + return Err(FloatTooBigError{float_value: fnum}); } let gds_exp_bits = (gds_exp as u64) << 56; let real8 = sign | gds_exp_bits | gds_mant; - real8 + Ok(real8) } @@ -281,7 +292,7 @@ mod tests { #[should_panic] fn test_encode_real8_panic() { use crate::basic::encode_real8; - encode_real8(1e80); + encode_real8(1e80).unwrap(); } @@ -348,7 +359,7 @@ mod tests { const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; for vv in REALS { print!("{vv}\n"); - assert!((decode_real8(encode_real8(vv)) - vv).abs() < f64::EPSILON); + assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON); } } @@ -359,7 +370,7 @@ mod tests { const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; let mut buf = [10; 8 * COUNT]; for (ii, &vv) in REALS.iter().enumerate() { - pack_real8(&mut buf[ii * 8..], vv); + pack_real8(&mut buf[ii * 8..], vv).unwrap(); } for (ii, &vv) in REALS.iter().enumerate() { print!("{vv}\n"); diff --git a/src/record.rs b/src/record.rs index 7d04085..41921c4 100644 --- a/src/record.rs +++ b/src/record.rs @@ -279,7 +279,7 @@ impl RecordData for Real8 { } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(buf, *data) + pack_real8(buf, *data).expect(&format!("Float {0} too big for Real8", data)) } fn pack(data: &Self::InData) -> Self::ByteData { @@ -303,8 +303,8 @@ impl RecordData for Real8Pair { } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(&mut buf[8 * 0..], data.0); - pack_real8(&mut buf[8 * 1..], data.1); + pack_real8(&mut buf[8 * 0..], data.0).expect(&format!("Float.0 {0} too big for Real8", data.0)); + pack_real8(&mut buf[8 * 1..], data.1).expect(&format!("Float.1 {0} too big for Real8", data.1)); } fn pack(data: &Self::InData) -> Self::ByteData { From ad1c2f1c35ed96b9878631706b59f60acafd720b Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 19 Dec 2024 19:02:55 -0800 Subject: [PATCH 14/18] remove commented code --- src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dbaf483..2a34034 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,3 @@ -//#![feature(generic_associated_types)] - extern crate byteorder; pub mod basic; From ba07d253d266b0eb19b64e2663975b772b7198c0 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 21 Dec 2024 09:50:19 -0800 Subject: [PATCH 15/18] pyo3 variant working --- Cargo.toml | 11 ++- klamath_rs_ext/__init__.py | 6 ++ klamath_rs_ext/basic.py | 56 +++++++++++++ klamath_rs_ext/py.typed | 0 pyproject.toml | 7 ++ src/lib.rs | 156 +++++++++++++++++++++++++++++++++++++ 6 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 klamath_rs_ext/__init__.py create mode 100644 klamath_rs_ext/basic.py create mode 100644 klamath_rs_ext/py.typed create mode 100644 pyproject.toml diff --git a/Cargo.toml b/Cargo.toml index 01429e9..41cbb68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,16 @@ [package] -name = "rs-klamath" +name = "klamath_rs_ext" version = "0.1.0" authors = ["jan "] edition = "2021" + +[lib] +name = "klamath_rs_ext" +crate-type = ["cdylib", "rlib"] + + [dependencies] byteorder = "^1" -#nom = "^7" +pyo3 = "^0" +numpy = "^0" diff --git a/klamath_rs_ext/__init__.py b/klamath_rs_ext/__init__.py new file mode 100644 index 0000000..fa02fb7 --- /dev/null +++ b/klamath_rs_ext/__init__.py @@ -0,0 +1,6 @@ +from .basic import pack_int2 as pack_int2 +from .basic import pack_int4 as pack_int4 + + +__version__ = 0.1 + diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py new file mode 100644 index 0000000..7237b1c --- /dev/null +++ b/klamath_rs_ext/basic.py @@ -0,0 +1,56 @@ +from collections.abc import Sequence + +import numpy +from numpy.typing import NDArray + +from .klamath_rs_ext import arr_to_int2, arr_to_int4 + + +def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + arr = numpy.asarray(data) + + if arr.dtype in ( + numpy.float64, numpy.float32, + numpy.int64, numpy.uint64, + numpy.int32, numpy.uint32, + numpy.int16, numpy.uint16, + ): + arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) + if arr is data: + arr = numpy.array(arr, copy=True) + arr_to_int2(arr) + i2arr = arr.view('>i2')[::arr.itemsize // 2] + return i2arr.tobytes() + + if arr.dtype == numpy.dtype('>i2'): + return arr.tobytes() + + if (arr > 32767).any() or (arr < -32768).any(): + raise Exception(f'int2 data out of range: {arr}') + + return arr.astype('>i2').tobytes() + + +def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + arr = numpy.asarray(data) + + if arr.dtype in ( + numpy.float64, numpy.float32, + numpy.int64, numpy.uint64, + numpy.int32, numpy.uint32, + ): + arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) + if arr is data: + arr = numpy.array(arr, copy=True) + arr_to_int4(arr) + i4arr = arr.view('>i4')[::arr.itemsize // 4] + return i4arr.tobytes() + + if arr.dtype == numpy.dtype('>i4'): + return arr.tobytes() + + if (arr > 2147483647).any() or (arr < -2147483648).any(): + raise Exception(f'int4 data out of range: {arr}') + + return arr.astype('>i4').tobytes() + diff --git a/klamath_rs_ext/py.typed b/klamath_rs_ext/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9704ea5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["maturin>1.0,<2.0"] +build-backend = "maturin" + + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/src/lib.rs b/src/lib.rs index 2a34034..b64ab5d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,3 +5,159 @@ pub mod record; pub mod records; pub mod elements; pub mod library; + + +//use ndarray; +use numpy::{PyArray1, PyUntypedArray, PyUntypedArrayMethods, PyArrayDescrMethods, PyArrayMethods, dtype}; +use pyo3::prelude::{Python, pymodule, PyModule, PyResult, Bound, wrap_pyfunction, pyfunction, PyModuleMethods, PyAnyMethods}; +use pyo3::exceptions::{PyValueError, PyTypeError}; + + +#[pymodule] +fn klamath_rs_ext(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(arr_to_int2, m)?)?; + m.add_function(wrap_pyfunction!(arr_to_int4, m)?)?; + Ok(()) +} + + +#[pyfunction] +fn arr_to_int2(py: Python<'_>, pyarr: &Bound<'_, PyUntypedArray>) -> PyResult<()> { + use rust_util::ToInt2BE; + + assert!(pyarr.is_c_contiguous(), "Array must be c-contiguous!"); + + macro_rules! i2if { + ( $el_type:expr, $tt:ty ) => { + if $el_type.is_equiv_to(&dtype::<$tt>(py)) { + let arr = pyarr.downcast::>()?; + let mut array = unsafe { arr.as_array_mut() }; + for xx in array.iter_mut() { + *xx = <$tt>::convert_to_i2be(*xx).map_err( + |e| PyValueError::new_err(format!("Invalid value for 2-byte int: {}", e)) + )?; + } + return Ok(()) + } + } + } + + + let el_type = pyarr.dtype(); + i2if!(el_type, f64); + i2if!(el_type, f32); + i2if!(el_type, i64); + i2if!(el_type, u64); + i2if!(el_type, i32); + i2if!(el_type, u32); + i2if!(el_type, i16); + i2if!(el_type, u16); + + Err(PyTypeError::new_err(format!("arr_to_int2 not implemented for type {:?}", el_type))) +} + + +#[pyfunction] +fn arr_to_int4(py: Python<'_>, pyarr: &Bound<'_, PyUntypedArray>) -> PyResult<()> { + use rust_util::ToInt4BE; + + assert!(pyarr.is_c_contiguous(), "Array must be c-contiguous!"); + + macro_rules! i4if { + ( $el_type:expr, $tt:ty ) => { + if $el_type.is_equiv_to(&dtype::<$tt>(py)) { + let arr = pyarr.downcast::>()?; + let mut array = unsafe { arr.as_array_mut() }; + for xx in array.iter_mut() { + *xx = <$tt>::convert_to_i4be(*xx).map_err( + |e| PyValueError::new_err(format!("Invalid value for 4-byte int: {}", e)) + )?; + } + return Ok(()) + } + } + } + + let el_type = pyarr.dtype(); + i4if!(el_type, f64); + i4if!(el_type, f32); + i4if!(el_type, i64); + i4if!(el_type, u64); + i4if!(el_type, i32); + i4if!(el_type, u32); + + Err(PyTypeError::new_err(format!("arr_to_int4 not implemented for type {:?}", el_type))) +} + + +mod rust_util { + use byteorder::{ByteOrder, BigEndian}; + use std::mem::size_of; + + pub trait ToInt2BE { + fn convert_to_i2be(ii: Self) -> Result where Self: Sized; + } + + pub trait ToInt4BE { + fn convert_to_i4be(ii: Self) -> Result where Self: Sized; + } + + macro_rules! impl_i2be { + ( $tt:ty ) => { + impl ToInt2BE for $tt { + fn convert_to_i2be(ii: $tt) -> Result<$tt, $tt> { + if ii < i16::MIN as $tt { return Err(ii); } + if ii > i16::MAX as $tt { return Err(ii); } + + let mut buf = [0; size_of::<$tt>()]; + BigEndian::write_i16(&mut buf, ii as i16); + Ok(<$tt>::from_le_bytes(buf)) + } + } + } + } + + macro_rules! impl_i4be { + ( $tt:ty ) => { + impl ToInt4BE for $tt { + fn convert_to_i4be(ii: $tt) -> Result<$tt, $tt> { + if ii < i32::MIN as $tt { return Err(ii); } + if ii > i32::MAX as $tt { return Err(ii); } + + let mut buf = [0; size_of::<$tt>()]; + BigEndian::write_i32(&mut buf, ii as i32); + Ok(<$tt>::from_le_bytes(buf)) + } + } + } + } + + impl_i2be!(f64); + impl_i4be!(f64); + + impl_i2be!(f32); + impl_i4be!(f32); + + impl_i2be!(i64); + impl_i4be!(i64); + impl_i2be!(u64); + impl_i4be!(u64); + + impl_i2be!(i32); + impl_i4be!(i32); + impl_i2be!(u32); + impl_i4be!(u32); + + impl_i2be!(i16); + impl_i2be!(u16); + + // Does not fit + //impl_i4be!(i16); + //impl_i4be!(u16); + // + //impl_i2be!(i8); + //impl_i4be!(i8); + //impl_i2be!(u8); + //impl_i4be!(u8); + +} From 320958d888fcd05b36d9f0fa8b9b29aeddb84582 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 21 Dec 2024 13:56:51 -0800 Subject: [PATCH 16/18] ctypes approach --- Cargo.toml | 4 +- klamath_rs_ext/__init__.py | 2 +- klamath_rs_ext/basic.py | 71 +++++++++++++++++++++------ pyproject.toml | 32 +++++++++++-- src/lib.rs | 98 +++++++++++--------------------------- 5 files changed, 115 insertions(+), 92 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 41cbb68..a211306 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "klamath_rs_ext" -version = "0.1.0" +version = "0.2.0" authors = ["jan "] edition = "2021" @@ -12,5 +12,3 @@ crate-type = ["cdylib", "rlib"] [dependencies] byteorder = "^1" -pyo3 = "^0" -numpy = "^0" diff --git a/klamath_rs_ext/__init__.py b/klamath_rs_ext/__init__.py index fa02fb7..d042d92 100644 --- a/klamath_rs_ext/__init__.py +++ b/klamath_rs_ext/__init__.py @@ -2,5 +2,5 @@ from .basic import pack_int2 as pack_int2 from .basic import pack_int4 as pack_int4 -__version__ = 0.1 +__version__ = 0.2 diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py index 7237b1c..838b42f 100644 --- a/klamath_rs_ext/basic.py +++ b/klamath_rs_ext/basic.py @@ -1,24 +1,68 @@ from collections.abc import Sequence +import ctypes +from pathlib import Path +from itertools import chain import numpy from numpy.typing import NDArray -from .klamath_rs_ext import arr_to_int2, arr_to_int4 +so_path = Path(__file__).resolve().parent / 'libklamath_rs_ext.so' +clib = ctypes.CDLL(so_path) + + +CONV_TABLE_i16 = { + numpy.float64: clib.f64_to_i16, + numpy.float32: clib.f32_to_i16, + numpy.int64: clib.i64_to_i16, + numpy.int32: clib.i32_to_i16, + numpy.int16: clib.i16_to_i16, + numpy.uint64: clib.u64_to_i16, + numpy.uint32: clib.u32_to_i16, + numpy.uint16: clib.u16_to_i16, + } + +CONV_TABLE_i32 = { + numpy.float64: clib.f64_to_i32, + numpy.float32: clib.f32_to_i32, + numpy.int64: clib.i64_to_i32, + numpy.int32: clib.i32_to_i32, + numpy.uint64: clib.u64_to_i32, + numpy.uint32: clib.u32_to_i32, + } + +clib.f64_to_i16.restype = ctypes.c_double +clib.f32_to_i16.restype = ctypes.c_float +clib.i64_to_i16.restype = ctypes.c_int64 +clib.i32_to_i16.restype = ctypes.c_int32 +clib.i16_to_i16.restype = ctypes.c_int16 +clib.u64_to_i16.restype = ctypes.c_uint64 +clib.u32_to_i16.restype = ctypes.c_uint32 +clib.u16_to_i16.restype = ctypes.c_uint16 + +clib.f64_to_i32.restype = ctypes.c_double +clib.f32_to_i32.restype = ctypes.c_float +clib.i64_to_i32.restype = ctypes.c_int64 +clib.i32_to_i32.restype = ctypes.c_int32 +clib.u64_to_i32.restype = ctypes.c_uint64 +clib.u32_to_i32.restype = ctypes.c_uint32 + + +for fn in chain(CONV_TABLE_i16.values(), CONV_TABLE_i32.values()): + fn.argtypes = [ctypes.POINTER(fn.restype), ctypes.c_size_t] + def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.asarray(data) - if arr.dtype in ( - numpy.float64, numpy.float32, - numpy.int64, numpy.uint64, - numpy.int32, numpy.uint32, - numpy.int16, numpy.uint16, - ): + if arr.dtype in CONV_TABLE_i16.keys(): arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) if arr is data: arr = numpy.array(arr, copy=True) - arr_to_int2(arr) + + fn = CONV_TABLE_i16[arr.dtype] + result = fn(arr.ctypes.data_as(fn.argtypes[0]), arr.size) + i2arr = arr.view('>i2')[::arr.itemsize // 2] return i2arr.tobytes() @@ -34,15 +78,14 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.asarray(data) - if arr.dtype in ( - numpy.float64, numpy.float32, - numpy.int64, numpy.uint64, - numpy.int32, numpy.uint32, - ): + if arr.dtype in CONV_TABLE_i32.keys(): arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) if arr is data: arr = numpy.array(arr, copy=True) - arr_to_int4(arr) + + fn = CONV_TABLE_i32[arr.dtype] + result = fn(arr.ctypes.data_as(fn.argtypes[0]), arr.size) + i4arr = arr.view('>i4')[::arr.itemsize // 4] return i4arr.tobytes() diff --git a/pyproject.toml b/pyproject.toml index 9704ea5..97d888c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,32 @@ requires = ["maturin>1.0,<2.0"] build-backend = "maturin" - -[tool.maturin] -features = ["pyo3/extension-module"] +[project] +name = "klamath_rs_ext" +description = "Compiled extensions for klamath GDS library" +#readme = "README.md" +#license = { file = "LICENSE.md" } +authors = [ + { name="Jan Petykiewicz", email="jan@mpxd.net" }, + ] +homepage = "https://mpxd.net/code/jan/klamath-rs" +repository = "https://mpxd.net/code/jan/klamath-rs" +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", +# "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Manufacturing", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", + ] +requires-python = ">=3.11" +#include = [ +# "LICENSE.md" +# ] +dynamic = ["version"] +dependencies = [ + "cffi", + ] diff --git a/src/lib.rs b/src/lib.rs index b64ab5d..84e8551 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,87 +7,43 @@ pub mod elements; pub mod library; -//use ndarray; -use numpy::{PyArray1, PyUntypedArray, PyUntypedArrayMethods, PyArrayDescrMethods, PyArrayMethods, dtype}; -use pyo3::prelude::{Python, pymodule, PyModule, PyResult, Bound, wrap_pyfunction, pyfunction, PyModuleMethods, PyAnyMethods}; -use pyo3::exceptions::{PyValueError, PyTypeError}; +use rust_util::ToInt2BE; +use rust_util::ToInt4BE; -#[pymodule] -fn klamath_rs_ext(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(arr_to_int2, m)?)?; - m.add_function(wrap_pyfunction!(arr_to_int4, m)?)?; - Ok(()) -} - - -#[pyfunction] -fn arr_to_int2(py: Python<'_>, pyarr: &Bound<'_, PyUntypedArray>) -> PyResult<()> { - use rust_util::ToInt2BE; - - assert!(pyarr.is_c_contiguous(), "Array must be c-contiguous!"); - - macro_rules! i2if { - ( $el_type:expr, $tt:ty ) => { - if $el_type.is_equiv_to(&dtype::<$tt>(py)) { - let arr = pyarr.downcast::>()?; - let mut array = unsafe { arr.as_array_mut() }; - for xx in array.iter_mut() { - *xx = <$tt>::convert_to_i2be(*xx).map_err( - |e| PyValueError::new_err(format!("Invalid value for 2-byte int: {}", e)) - )?; +macro_rules! mkfun { + ( $fname:ident, $tt:ty, $elfn:ident ) => { + #[no_mangle] + pub extern "C" fn $fname(arr: *mut $tt, size: usize) -> $tt { + let sl = unsafe { std::slice::from_raw_parts_mut(arr, size) }; + for xx in sl.iter_mut() { + let res = <$tt>::$elfn(*xx); + match res { + Err(cc) => return cc, + Ok(cc) => { *xx = cc; }, } - return Ok(()) } + 0 as $tt } } - - - let el_type = pyarr.dtype(); - i2if!(el_type, f64); - i2if!(el_type, f32); - i2if!(el_type, i64); - i2if!(el_type, u64); - i2if!(el_type, i32); - i2if!(el_type, u32); - i2if!(el_type, i16); - i2if!(el_type, u16); - - Err(PyTypeError::new_err(format!("arr_to_int2 not implemented for type {:?}", el_type))) } -#[pyfunction] -fn arr_to_int4(py: Python<'_>, pyarr: &Bound<'_, PyUntypedArray>) -> PyResult<()> { - use rust_util::ToInt4BE; +mkfun!(f64_to_i16, f64, convert_to_i2be); +mkfun!(f32_to_i16, f32, convert_to_i2be); +mkfun!(i64_to_i16, i64, convert_to_i2be); +mkfun!(u64_to_i16, u64, convert_to_i2be); +mkfun!(i32_to_i16, i32, convert_to_i2be); +mkfun!(u32_to_i16, u32, convert_to_i2be); +mkfun!(i16_to_i16, i16, convert_to_i2be); +mkfun!(u16_to_i16, u16, convert_to_i2be); - assert!(pyarr.is_c_contiguous(), "Array must be c-contiguous!"); - - macro_rules! i4if { - ( $el_type:expr, $tt:ty ) => { - if $el_type.is_equiv_to(&dtype::<$tt>(py)) { - let arr = pyarr.downcast::>()?; - let mut array = unsafe { arr.as_array_mut() }; - for xx in array.iter_mut() { - *xx = <$tt>::convert_to_i4be(*xx).map_err( - |e| PyValueError::new_err(format!("Invalid value for 4-byte int: {}", e)) - )?; - } - return Ok(()) - } - } - } - - let el_type = pyarr.dtype(); - i4if!(el_type, f64); - i4if!(el_type, f32); - i4if!(el_type, i64); - i4if!(el_type, u64); - i4if!(el_type, i32); - i4if!(el_type, u32); - - Err(PyTypeError::new_err(format!("arr_to_int4 not implemented for type {:?}", el_type))) -} +mkfun!(f64_to_i32, f64, convert_to_i4be); +mkfun!(f32_to_i32, f32, convert_to_i4be); +mkfun!(i64_to_i32, i64, convert_to_i4be); +mkfun!(u64_to_i32, u64, convert_to_i4be); +mkfun!(i32_to_i32, i32, convert_to_i4be); +mkfun!(u32_to_i32, u32, convert_to_i4be); mod rust_util { From bac31e2ce634af549f876a1414d2aa40251209f9 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 21 Dec 2024 15:42:50 -0800 Subject: [PATCH 17/18] CFFI + maturin approach --- klamath_rs_ext/basic.py | 75 +++++++++---------- pyproject.toml | 4 + src/lib.rs | 157 ++++++++++++++++------------------------ 3 files changed, 99 insertions(+), 137 deletions(-) diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py index 838b42f..4540fe6 100644 --- a/klamath_rs_ext/basic.py +++ b/klamath_rs_ext/basic.py @@ -6,62 +6,47 @@ from itertools import chain import numpy from numpy.typing import NDArray -so_path = Path(__file__).resolve().parent / 'libklamath_rs_ext.so' -clib = ctypes.CDLL(so_path) +from .klamath_rs_ext import lib, ffi CONV_TABLE_i16 = { - numpy.float64: clib.f64_to_i16, - numpy.float32: clib.f32_to_i16, - numpy.int64: clib.i64_to_i16, - numpy.int32: clib.i32_to_i16, - numpy.int16: clib.i16_to_i16, - numpy.uint64: clib.u64_to_i16, - numpy.uint32: clib.u32_to_i16, - numpy.uint16: clib.u16_to_i16, + numpy.float64: lib.f64_to_i16, + numpy.float32: lib.f32_to_i16, + numpy.int64: lib.i64_to_i16, + numpy.int32: lib.i32_to_i16, + numpy.int16: lib.i16_to_i16, + numpy.uint64: lib.u64_to_i16, + numpy.uint32: lib.u32_to_i16, + numpy.uint16: lib.u16_to_i16, } CONV_TABLE_i32 = { - numpy.float64: clib.f64_to_i32, - numpy.float32: clib.f32_to_i32, - numpy.int64: clib.i64_to_i32, - numpy.int32: clib.i32_to_i32, - numpy.uint64: clib.u64_to_i32, - numpy.uint32: clib.u32_to_i32, + numpy.float64: lib.f64_to_i32, + numpy.float32: lib.f32_to_i32, + numpy.int64: lib.i64_to_i32, + numpy.int32: lib.i32_to_i32, + numpy.uint64: lib.u64_to_i32, + numpy.uint32: lib.u32_to_i32, } -clib.f64_to_i16.restype = ctypes.c_double -clib.f32_to_i16.restype = ctypes.c_float -clib.i64_to_i16.restype = ctypes.c_int64 -clib.i32_to_i16.restype = ctypes.c_int32 -clib.i16_to_i16.restype = ctypes.c_int16 -clib.u64_to_i16.restype = ctypes.c_uint64 -clib.u32_to_i16.restype = ctypes.c_uint32 -clib.u16_to_i16.restype = ctypes.c_uint16 - -clib.f64_to_i32.restype = ctypes.c_double -clib.f32_to_i32.restype = ctypes.c_float -clib.i64_to_i32.restype = ctypes.c_int64 -clib.i32_to_i32.restype = ctypes.c_int32 -clib.u64_to_i32.restype = ctypes.c_uint64 -clib.u32_to_i32.restype = ctypes.c_uint32 - - -for fn in chain(CONV_TABLE_i16.values(), CONV_TABLE_i32.values()): - fn.argtypes = [ctypes.POINTER(fn.restype), ctypes.c_size_t] - def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.asarray(data) - if arr.dtype in CONV_TABLE_i16.keys(): + for dtype in CONV_TABLE_i16.keys(): + if arr.dtype != dtype: + continue + arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) if arr is data: arr = numpy.array(arr, copy=True) - fn = CONV_TABLE_i16[arr.dtype] - result = fn(arr.ctypes.data_as(fn.argtypes[0]), arr.size) + fn = CONV_TABLE_i16[dtype] + result = fn(ffi.from_buffer(arr), arr.size) + + if result != 0: + raise ValueError(f'Invalid value for conversion to Int2: {result}') i2arr = arr.view('>i2')[::arr.itemsize // 2] return i2arr.tobytes() @@ -78,13 +63,19 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.asarray(data) - if arr.dtype in CONV_TABLE_i32.keys(): + for dtype in CONV_TABLE_i32.keys(): + if arr.dtype != dtype: + continue + arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) if arr is data: arr = numpy.array(arr, copy=True) - fn = CONV_TABLE_i32[arr.dtype] - result = fn(arr.ctypes.data_as(fn.argtypes[0]), arr.size) + fn = CONV_TABLE_i32[dtype] + result = fn(ffi.from_buffer(arr), arr.size) + + if result != 0: + raise ValueError(f'Invalid value for conversion to Int4: {result}') i4arr = arr.view('>i4')[::arr.itemsize // 4] return i4arr.tobytes() diff --git a/pyproject.toml b/pyproject.toml index 97d888c..06aa1af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,3 +31,7 @@ dynamic = ["version"] dependencies = [ "cffi", ] + +[tool.maturin] +bindings = "cffi" + diff --git a/src/lib.rs b/src/lib.rs index 84e8551..cf8365c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,21 +7,21 @@ pub mod elements; pub mod library; -use rust_util::ToInt2BE; -use rust_util::ToInt4BE; +use byteorder::{ByteOrder, BigEndian}; +use std::mem::size_of; -macro_rules! mkfun { - ( $fname:ident, $tt:ty, $elfn:ident ) => { - #[no_mangle] - pub extern "C" fn $fname(arr: *mut $tt, size: usize) -> $tt { - let sl = unsafe { std::slice::from_raw_parts_mut(arr, size) }; +macro_rules! impl_i16be { + ( $tt:ty, $arr:ident, $size:ident ) => { + { + let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; for xx in sl.iter_mut() { - let res = <$tt>::$elfn(*xx); - match res { - Err(cc) => return cc, - Ok(cc) => { *xx = cc; }, - } + if *xx < i16::MIN as $tt { return *xx } + if *xx > i16::MAX as $tt { return *xx } + + let mut buf = [0; size_of::<$tt>()]; + BigEndian::write_i16(&mut buf, *xx as i16); + *xx = <$tt>::from_le_bytes(buf); } 0 as $tt } @@ -29,91 +29,58 @@ macro_rules! mkfun { } -mkfun!(f64_to_i16, f64, convert_to_i2be); -mkfun!(f32_to_i16, f32, convert_to_i2be); -mkfun!(i64_to_i16, i64, convert_to_i2be); -mkfun!(u64_to_i16, u64, convert_to_i2be); -mkfun!(i32_to_i16, i32, convert_to_i2be); -mkfun!(u32_to_i16, u32, convert_to_i2be); -mkfun!(i16_to_i16, i16, convert_to_i2be); -mkfun!(u16_to_i16, u16, convert_to_i2be); +macro_rules! impl_i32be { + ( $tt:ty, $arr:ident, $size:ident ) => { + { + let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; + for xx in sl.iter_mut() { + if *xx < i32::MIN as $tt { return *xx } + if *xx > i32::MAX as $tt { return *xx } -mkfun!(f64_to_i32, f64, convert_to_i4be); -mkfun!(f32_to_i32, f32, convert_to_i4be); -mkfun!(i64_to_i32, i64, convert_to_i4be); -mkfun!(u64_to_i32, u64, convert_to_i4be); -mkfun!(i32_to_i32, i32, convert_to_i4be); -mkfun!(u32_to_i32, u32, convert_to_i4be); - - -mod rust_util { - use byteorder::{ByteOrder, BigEndian}; - use std::mem::size_of; - - pub trait ToInt2BE { - fn convert_to_i2be(ii: Self) -> Result where Self: Sized; - } - - pub trait ToInt4BE { - fn convert_to_i4be(ii: Self) -> Result where Self: Sized; - } - - macro_rules! impl_i2be { - ( $tt:ty ) => { - impl ToInt2BE for $tt { - fn convert_to_i2be(ii: $tt) -> Result<$tt, $tt> { - if ii < i16::MIN as $tt { return Err(ii); } - if ii > i16::MAX as $tt { return Err(ii); } - - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i16(&mut buf, ii as i16); - Ok(<$tt>::from_le_bytes(buf)) - } + let mut buf = [0; size_of::<$tt>()]; + BigEndian::write_i32(&mut buf, *xx as i32); + *xx = <$tt>::from_le_bytes(buf); } + 0 as $tt } } - - macro_rules! impl_i4be { - ( $tt:ty ) => { - impl ToInt4BE for $tt { - fn convert_to_i4be(ii: $tt) -> Result<$tt, $tt> { - if ii < i32::MIN as $tt { return Err(ii); } - if ii > i32::MAX as $tt { return Err(ii); } - - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i32(&mut buf, ii as i32); - Ok(<$tt>::from_le_bytes(buf)) - } - } - } - } - - impl_i2be!(f64); - impl_i4be!(f64); - - impl_i2be!(f32); - impl_i4be!(f32); - - impl_i2be!(i64); - impl_i4be!(i64); - impl_i2be!(u64); - impl_i4be!(u64); - - impl_i2be!(i32); - impl_i4be!(i32); - impl_i2be!(u32); - impl_i4be!(u32); - - impl_i2be!(i16); - impl_i2be!(u16); - - // Does not fit - //impl_i4be!(i16); - //impl_i4be!(u16); - // - //impl_i2be!(i8); - //impl_i4be!(i8); - //impl_i2be!(u8); - //impl_i4be!(u8); - } + + +#[no_mangle] +pub extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_i16be!(f64, arr, size) } +#[no_mangle] +pub extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_i32be!(f64, arr, size) } + +#[no_mangle] +pub extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_i16be!(f32, arr, size) } +#[no_mangle] +pub extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_i32be!(f32, arr, size) } + +#[no_mangle] +pub extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_i16be!(u64, arr, size) } +#[no_mangle] +pub extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_i32be!(u64, arr, size) } + +#[no_mangle] +pub extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_i16be!(i64, arr, size) } +#[no_mangle] +pub extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_i32be!(i64, arr, size) } + +#[no_mangle] +pub extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_i16be!(u32, arr, size) } +#[no_mangle] +pub extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_i32be!(u32, arr, size) } + +#[no_mangle] +pub extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_i16be!(i32, arr, size) } +#[no_mangle] +pub extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_i32be!(i32, arr, size) } + + +#[no_mangle] +pub extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_i16be!(u16, arr, size) } + +#[no_mangle] +pub extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_i16be!(i16, arr, size) } + From ec659b559d1b380debea840cab7442e048041110 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 21 Dec 2024 16:38:56 -0800 Subject: [PATCH 18/18] fixes for cffi approach --- klamath_rs_ext/basic.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py index 4540fe6..aabc246 100644 --- a/klamath_rs_ext/basic.py +++ b/klamath_rs_ext/basic.py @@ -32,7 +32,7 @@ CONV_TABLE_i32 = { def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.asarray(data) + arr = numpy.atleast_1d(data) for dtype in CONV_TABLE_i16.keys(): if arr.dtype != dtype: @@ -43,7 +43,8 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.array(arr, copy=True) fn = CONV_TABLE_i16[dtype] - result = fn(ffi.from_buffer(arr), arr.size) + buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) + result = fn(buf, arr.size) if result != 0: raise ValueError(f'Invalid value for conversion to Int2: {result}') @@ -61,7 +62,7 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.asarray(data) + arr = numpy.atleast_1d(data) for dtype in CONV_TABLE_i32.keys(): if arr.dtype != dtype: @@ -72,7 +73,8 @@ def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: arr = numpy.array(arr, copy=True) fn = CONV_TABLE_i32[dtype] - result = fn(ffi.from_buffer(arr), arr.size) + buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) + result = fn(buf, arr.size) if result != 0: raise ValueError(f'Invalid value for conversion to Int4: {result}')