From 48db47217afffd8f489c21dd9e01f57b1df65978 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 18 Dec 2021 21:05:00 -0800 Subject: [PATCH 01/41] snapshot 2021-12-18 21:05:00.635887 --- .gitignore | 6 + Cargo.toml | 8 + src/__init__.rs | 29 +++ src/basic.rs | 198 +++++++++++++++ src/elements.rs | 481 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 7 + src/library.rs | 282 ++++++++++++++++++++++ src/record.rs | 219 +++++++++++++++++ src/records.rs | 595 ++++++++++++++++++++++++++++++++++++++++++++++ src/test_basic.rs | 121 ++++++++++ 10 files changed, 1946 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/__init__.rs create mode 100644 src/basic.rs create mode 100644 src/elements.rs create mode 100644 src/lib.rs create mode 100644 src/library.rs create mode 100644 src/record.rs create mode 100644 src/records.rs create mode 100644 src/test_basic.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d80173 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target/ +**/*.rs.bk +Cargo.lock + +*.swp +*.swo diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..414196a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rs-klamath" +version = "0.1.0" +authors = ["jan "] + +[dependencies] +byteorder = "^1" +nom = "^7" diff --git a/src/__init__.rs b/src/__init__.rs new file mode 100644 index 0000000..44d6c6b --- /dev/null +++ b/src/__init__.rs @@ -0,0 +1,29 @@ +/* + * `klamath` is a Python module for reading and writing to the GDSII file format. + * + * The goal is to keep this library simple: + * - Map data types directly wherever possible. + * * Presents an accurate representation of what is saved to the file. + * * Avoids excess copies / allocations for speed. + * * No "automatic" error checking, except when casting datatypes. + * If data integrity checks are provided at all, they must be + * explicitly run by the caller. + * - Low-level functionality is first-class. + * * Meant for use-cases where the caller wants to read or write + * individual GDS records. + * * Offers complete control over the written file. + * - Opinionated and limited high-level functionality. + * * Discards or ignores rarely-encountered data types. + * * Keeps functions simple and reusable. + * * Only de/encodes the file format, doesn't provide tools to modify + * the data itself. + * * Still requires explicit values for most fields. + * - No compilation + * * Uses `numpy` for speed, since it's commonly available / pre-built. + * * Building this library should not require a compiler. + * + * `klamath` was built to provide a fast and versatile GDS interface for + * [masque](https://mpxd.net/code/jan/masque), which provides higher-level + * tools for working with hierarchical design data and supports multiple + * file formats. + */ diff --git a/src/basic.rs b/src/basic.rs new file mode 100644 index 0000000..0cadc0d --- /dev/null +++ b/src/basic.rs @@ -0,0 +1,198 @@ +/* + * Functionality for parsing and writing basic data types + */ +use nom; +use nom::{IResult}; +use byteorder::BigEndian; +//use std::io::Write; +use std::io; + + +pub type OWResult = Result; + +/* + * Parse functions + */ +//pub fn parse_byte_as_bits(input: &[u8]) -> IResult<&[u8], (u8, u8, u8, u8, u8, u8, u8, u8)> { +// nom::bits::bits(nom::sequence::tuple(( +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// nom::bits::complete::take::<_, _, _, CustomError<_>>(1_usize), +// )))(input) +//} + + +pub fn parse_int2(input: &[u8]) -> IResult<&[u8], i16> { + nom::number::streaming::be_i16(input)? +} + +pub fn parse_int4(input: &[u8]) -> IResult<&[u8], i32> { + nom::number::streaming::be_i32(input)? +} + +pub fn decode_real8(int: u64) -> f64 { + // Convert GDS REAL8 to IEEE float64 + let neg = int & 0x8000_0000_0000_0000; + let exp = (int >> 56) & 0x7f; + let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64; + if neg != 0 { + mant *= -1 + } + mant * 2_f64.powi(4 * (exp - 64) - 56) +} + +pub fn parse_real8(input: &[u8]) -> IResult<&[u8], f64> { + let data = nom::number::streaming::be_u64(input)?; + IResult::Ok(decode_real8(data)) +} + + +pub fn parse_datetime(input: &[u8]) -> IResult<&[u8], [u16; 6]> { + let mut buf = [0_u16; 6]; + let mut parts = nom::multi::fill(parse_int2, &mut buf)(input); + parts[0] += 1900; // Year is from 1900 + IResult::Ok(parts) +} + + +pub fn parse_bitarray(input: &[u8]) -> IResult<&[u8], [bool; 16]> { + let bits = [false; 16]; + let (input, val) = parse_int2(input)?; + for ii in 0..16 { + bits[ii] = ((val >> (16 - 1 - ii)) & 0x01) == 1; + } + bits +} + + +pub fn parse_ascii(input: &[u8], length: usize) -> IResult<&[u8], Vec> { + let last = input[length - 1]; + let true_length = if last == '\0' { length - 1 } else { length }; + let vec = input[..true_length].to_vec(); + IResult::Ok((input[length..], vec)) +} + + +/* + * Pack functions + */ + +pub fn pack_bitarray(bits: &[bool; 16]) -> u16 { + let mut int: u16 = 0; + for ii in 0..16 { + int |= (bits[ii] as u16) << (16 - 1 - ii); + } + int +} + + +pub fn pack_int2(buf: &mut [u8], int: i16) { + BigEndian::write_i16(&mut buf, int) +} + +pub fn pack_int4(buf: &mut [u8], int: i32) { + BigEndian::write_i32(&mut buf, int) +} + +pub fn pack_real8(buf: &mut [u8], fnum: f64) { + BigEndian::write_u64(&mut buf, encode_real8(fnum)) +} + +pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { + let len = data.len(); + buf[..len].copy_from_slice(data); + if len % 2 == 1 { + buf[len] = '\0'; + len + 1 + } else { + len + } +} + + +pub fn pack_datetime(buf: &mut [u8], date: &[u16; 6]) { + assert!(buf.len() >= 6 * 2); + let year = date[0] - 1900; + pack_int2(&mut buf, year); + for ii in 1..6 { + pack_int2(&mut buf[(2 * ii)..], date[ii]); + } +} + + +pub fn encode_real8(fnum: f64) -> u64 { + // Convert from float64 to GDS REAL8 representation. + + // Split the ieee float bitfields + let ieee = fnum.to_bits(); + let sign = ieee & 0x8000_0000_0000_0000; + let ieee_exp = (ieee >> 52) as i32 & 0x7ff; + let ieee_mant = ieee & 0xf_ffff_ffff_ffff; + + let subnorm = (ieee_exp == 0) & (ieee_mant != 0); + if (ieee_exp == 0) & (ieee_mant == 0) { + return 0 + } + + // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) + // IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022) + // GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64) + // = (gds_mant / 2^56) * 2^(4 * gds_exp - 256) + + // Convert exponent. + let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII) + let mut exp16 = exp2 / 4; + let rest = exp2 % 4; + + // Compensate for exponent coarseness + let comp = rest != 0; + let mut shift; + if comp { + exp16 += 1; + shift = 4 - rest; + } else { + shift = rest; + } + shift -= 3; // account for gds bit position + + // add leading one + let mut gds_mant_unshifted = ieee_mant; + if !subnorm { + gds_mant_unshifted += 0x10_0000_0000_0000; + } + + let mut gds_mant = if shift > 0 { + gds_mant_unshifted >> shift + } else { + gds_mant_unshifted << -shift + }; + + // add gds exponent bias + let mut gds_exp = exp16 + 64; + + if gds_exp < -14 { + // number is too small + return 0 + } + + let neg_biased = gds_exp < 0; + if neg_biased { + gds_mant >>= gds_exp * 4; + gds_exp = 0; + } + + let too_big = (gds_exp > 0x7f) & !subnorm; + if too_big { + panic!("Number too big for real8 format"); //TODO error handling + } + + let gds_exp_bits = gds_exp << 56; + + let real8 = sign | gds_exp_bits | gds_mant; + real8 +} diff --git a/src/elements.rs b/src/elements.rs new file mode 100644 index 0000000..e972b77 --- /dev/null +++ b/src/elements.rs @@ -0,0 +1,481 @@ +/* + * Functionality for reading/writing elements (geometry, text labels, + * structure references) and associated properties. + */ + +//from .record import Record +// +use records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; + +use records; +use record::RecordHeader; +use basic::{OWResult}; + +use std::collections::HashMap; +use std::io::Write; +use nom::IResult; + + +pub fn read_properties(input: &[u8]) -> IResult<&[u8], HashMap::>> { + /* + * Read element properties. + * + * Assumes PROPATTR records have unique values. + * Stops reading after consuming ENDEL record. + * + * Args: + * stream: Stream to read from. + * + * Returns: + * propattr: -> propvalue mapping + */ + let properties = HashMap{}; + + let (input, header) = RecordHeader::parse(input)?; + while header.tag != ENDEL::tag() { + if header.tag == PROPATTR::tag() { + let (input, key) = PROPATTR::read_data(input, header.data_size)?; + let (input, value) = PROPVALUE::read(input)?; + assert!(!properties.contains_key(key), format!{"Duplicate property key: {}", key}); + properties.insert(key, value); + } + let (input, header) = RecordHeader::parse(input)?; + } + Ok((input, properties)) +} + + +fn write_properties(ww: W, properties: &HashMap::>) -> OWResult { + /* + * Write element properties. + * + * This is does _not_ write the ENDEL record. + * + * Args: + * stream: Stream to write to. + */ + let mut size = 0; + for (key, value) in &properties { + size += PROPATTR::write(ww, key)?; + size += PROPVALUE::write(ww, value)?; + } + Ok(size) +} + +trait Element { + fn parse(input: &[u8]) -> Self; + /* + * Read from a stream to construct this object. + * Consumes up to (and including) the ENDEL record. + */ + + fn write(&self, ww: W) -> OWResult; + /* + * Write this element to a stream. + * Finishes with an ENDEL record. + */ +} + +struct Reference { + /* + * Datastructure representing + * an instance of a structure (SREF / structure reference) or + * an array of instances (AREF / array reference). + * Type is determined by the presence of the `colrow` tuple. + * + * Transforms are applied to each individual instance (_not_ + * to the instance's origin location || array vectors). + */ + struct_name: Vec, // Name of the structure being referenced. + invert_y: bool, // Whether to mirror the pattern (negate y-values / flip across x-axis). Default false. + mag: f64, // Scaling factor (default 1) """ + angle_deg: f64, // Rotation (degrees counterclockwise) + xy: Vec, + /* + * (For SREF) Location in the parent structure corresponding to the instance's origin (0, 0). + * (For AREF) 3 locations: + * [`offset`, + * `offset + col_basis_vector * colrow[0]`, + * `offset + row_basis_vector * colrow[1]`] + * which define the first instance's offset and the array's basis vectors. + * Note that many GDS implementations only support manhattan basis vectors, and some + * assume a certain axis mapping (e.g. x->columns, y->rows) and "reinterpret" the + * basis vectors to match it. + */ + + colrow: Option<(i32, i32)>, // Number of columns and rows (AREF) || None (SREF) + properties: HashMap::>, // Properties associated with this reference. +} + +impl Element for Reference { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut invert_y = false; + let mut mag = 1; + let mut angle_deg = 0; + let mut colrow = None; + let (input, mut struct_name) = SNAME::skip_and_read(input)?; + + let (input, mut header) = RecordHeader::parse(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_STRANS => + {let (input, invert_y) = STRANS::read_data(input, header.data_size)?[0];}, + records::RTAG_MAG => + {let (input, mag) = MAG::read_data(input, header.data_size)?;}, + records::RTAG_ANGLE => + {let (input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, + records::RTAG_COLROW => + {let (input, colrow) = COLROW::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + }; + let (input, header) = RecordHeader::parse(input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + let (input, properties) = read_properties(input)?; + Reference{ + struct_name: struct_name, + xy: xy, + properties: properties, + colrow: colrow, + invert_y: invert_y, + mag: mag, + angle_deg: angle_deg + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += match self.colrow { + None => SREF::write(ww)?, + Some(_) => AREF::write(ww)?, + }; + + size += SNAME::write(ww, self.struct_name)?; + if self.angle_deg != 0 || self.mag != 1 || self.invert_y { + size += STRANS::write(ww, (self.invert_y as u16) << 15)?; + if self.mag != 1 { + size += MAG::write(ww, self.mag)?; + } + if self.angle_deg !=0 { + size += ANGLE::write(ww, self.angle_deg)?; + } + } + + if self.colrow.is_some() { + size += COLROW::write(ww, self.colrow)?; + } + + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww, None)?; + Ok(size) + } +} + +impl Reference { + pub fn check(&self) { + if self.colrow.is_some() { + assert!(self.xy.len() != 6, format!("colrow is Some, so expected size-6 xy. Got {}", self.xy)); + } else { + assert!(self.xy.len() != 2, format!("Expected size-2 xy. Got {}", self.xy)); + } + } +} + + +struct Boundary { + /* + * Datastructure representing a Boundary element. + */ + layer: (i16, i16), // (layer, data_type) tuple + xy: Vec, // Ordered vertices of the shape. First and last points should be identical. Order x0, y0, x1,... + properties: HashMap::>, // Properties for the element. +} + +impl Element for Boundary { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + Boundary{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += BOUNDARY::write(ww)?; + size += LAYER::write(ww, self.layer.0)?; + size += DATATYPE::write(ww, self.layer.1)?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct Path { + /* + * Datastructure representing a Path element. + + * If `path_type < 4`, `extension` values are not written. + * During read, `exension` defaults to (0, 0) even if unused. + */ + layer: (i16, i16), // (layer, data_type) tuple + path_type: i16, // End-cap type (0: flush, 1: circle, 2: square, 4: custom) + width: i16, // Path width + extension: (i32, i32), // Extension when using path_type=4. Ignored otherwise. + xy: Vec, // Path centerline coordinates. [x0, y0, x1, y1,...] + properties: HashMap::>, //Properties for the element. +} + +impl Element for Path { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut path_type = 0; + let mut width = 0; + let mut bgn_ext = 0; + let mut end_ext = 0; + let (input, mut layer) = LAYER::skip_and_read(input)?; + let (input, mut dtype) = DATATYPE::read(input)?; + + let (input, mut header) = RecordHeader::parse(&input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PATHTYPE => + {let (input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, + records::RTAG_WIDTH => + {let (input, width) = WIDTH::read_data(input, header.data_size)?;}, + records::RTAG_BGNEXTN => + {let (input, bgn_ext) = BGNEXTN::read_data(input, header.data_size)?;}, + records::RTAG_ENDEXTN => + {let (input, end_ext) = ENDEXTN::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + }; + let (input, header) = RecordHeader::parse(&input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + let (input, properties) = read_properties(input)?; + Path{ + layer: (layer, dtype), + xy: xy, + properties: properties, + extension: (bgn_ext, end_ext), + path_type: path_type, + width: width, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += PATH::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += DATATYPE::write(ww, self.layer[1])?; + if self.path_type != 0 { + size += PATHTYPE::write(ww, self.path_type)?; + } + if self.width != 0 { + size += WIDTH::write(ww, self.width)?; + } + + if self.path_type < 4 { + let (bgn_ext, end_ext) = self.extension; + if bgn_ext != 0 { + size += BGNEXTN::write(ww, bgn_ext)?; + } + if end_ext != 0 { + size += ENDEXTN::write(ww, end_ext)?; + } + } + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct GDSBox { + /* + * Datastructure representing a Box element. Rarely used. + */ + layer: (i16, i16), // (layer, box_type) tuple + xy: Vec, // Box coordinates (5 pairs) + properties: HashMap::>, // Properties for the element. +} + +impl Element for GDSBox { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = BOXTYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + GDSBox{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += BOX::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += BOXTYPE::write(ww, self.layer[1])?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + + +struct Node { + /* + * Datastructure representing a Node element. Rarely used. + */ + layer: (i16, i16), // (layer, box_type) tuple + xy: Vec, // 1-50 pairs of coordinates. + properties: HashMap::>, // Properties for the element. +} + +impl Element for Node { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = NODETYPE::read(input)?; + let (input, xy) = XY::read(input)?; + let (input, properties) = read_properties(input)?; + Node{ + layer: (layer, dtype), + xy: xy, + properties: properties, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += NODE::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += NODETYPE::write(ww, self.layer[1])?; + size += XY::write(ww, self.xy)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} + +struct Text { + /* + * Datastructure representing a text label. + */ + layer: (i16, i16), // (layer, node_type) tuple + presentation: [bool; 16], + /* + * Bit array. Default all zeros. + * bits 0-1: 00 left/01 center/10 right + * bits 2-3: 00 top/01 middle/10 bottom + * bits 4-5: font number + */ + path_type: i16, // Default 0 + width: i32, // Default 0 + invert_y: bool, // Vertical inversion. Default false. + mag: f64, // Scaling factor. Default 1. + angle_deg: f64, // Rotation (ccw). Default 0. + xy: Vec, // Position (1 pair only) + string: Vec, // Text content + properties: HashMap::> // Properties for the element. +} + +impl Element for Text { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let mut path_type = 0; + let mut presentation = 0; + let mut invert_y = false; + let mut width = 0; + let mut mag = 1; + let mut angle_deg = 0; + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = TEXTTYPE::read(input)?; + + let mut header = RecordHeader::parse(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PRESENTATION => + {let (input, presentation) = PRESENTATION::read_data(input, header.data_size)?;}, + records::RTAG_PATHTYPE => + {let (input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, + records::RTAG_WIDTH => + {let (input, width) = WIDTH::read_data(input, header.data_size)?;}, + records::RTAG_STRANS => { + let (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = strans[0]; + }, + records::RTAG_MAG => + {let (input, mag) = MAG::read_data(input, header.data_size)?;}, + records::RTAG_ANGLE => + {let (input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, + _ => + return Err(format!("Unexpected tag {:04x}", header.tag)), + } + let (input, header) = RecordHeader::parse(input)?; + } + let (input, xy) = XY::read_data(input, header.data_size)?; + + let (input, string) = STRING::read(input)?; + let (input, properties) = read_properties(input)?; + Text{ + layer: (layer, dtype), + xy: xy, + properties: properties, + string: string, + presentation: presentation, + path_type: path_type, + width: width, + invert_y: invert_y, + mag: mag, + angle_deg: angle_deg, + } + } + + fn write(&self, ww: W) -> OWResult { + let mut size = 0; + size += TEXT::write(ww)?; + size += LAYER::write(ww, self.layer[0])?; + size += TEXTTYPE::write(ww, self.layer[1])?; + if self.presentation != 0 { + size += PRESENTATION::write(ww, self.presentation)?; + } + if self.path_type != 0 { + size += PATHTYPE::write(ww, self.path_type)?; + } + if self.width != 0 { + size += WIDTH::write(ww, self.width)? + } + if self.angle_deg != 0 || self.mag != 1 || self.invert_y { + let strans = [false; 16]; + strans[0] = self.invert_y; + size += STRANS::write(ww, strans)?; + if self.mag != 1 { + size += MAG::write(ww, self.mag)?; + } + if self.angle_deg !=0 { + size += ANGLE::write(ww, self.angle_deg)?; + } + } + size += XY::write(ww, self.xy)?; + size += STRING::write(ww, self.string)?; + size += write_properties(ww, self.properties)?; + size += ENDEL::write(ww)?; + Ok(size) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d1c5fd4 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,7 @@ +extern crate nom; +extern crate byteorder; + +pub mod basic; +pub mod record; +pub mod records; +pub mod elements; diff --git a/src/library.rs b/src/library.rs new file mode 100644 index 0000000..7a8573a --- /dev/null +++ b/src/library.rs @@ -0,0 +1,282 @@ +/* + *File-level read/write functionality. + */ + +use nom; +use std::io::Write; +use std::collections:HashMap; + +use record; +use records; +use elements; +//from .records import HEADER, BGNLIB, ENDLIB, UNITS, LIBNAME +//from .records import BGNSTR, STRNAME, ENDSTR, SNAME, COLROW, ENDEL +//from .records import BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF +//from .elements import Element, Reference, Text, Box, Boundary, Path, Node + + +pub struct FileHeader { + /* + * Representation of the GDS file header. + * + * File header records: HEADER BGNLIB LIBNAME UNITS + * Optional records are ignored if present and never written. + * + * Version is written as `600`. + */ + name: Vec, // Library name + user_units_per_db_unit: f64, // Number of user units in one database unit + meters_per_db_unit: f64, // Number of meters in one database unit + mod_time: [u16; 6], // Last-modified time [y, m, d, h, m, s] + acc_time: [u16; 6], // Last-accessed time [y, m, d, h, m, s] +} + +impl FileHeader { + pub fn new(name: &[u8], meters_per_db_unit: f64, user_units_per_db_unit: f64) -> Self { + FileHeader{ + mod_time: [0, 1, 1, 0, 0, 0]; + acc_time: [0, 1, 1, 0, 0, 0]; + name: name.to_owned(), + user_units_per_db_unit: user_units_per_db_unit, + meters_per_db_unit: meters_per_db_unit, + } + } + + pub fn read(input: &[u8]) -> IResult<&[u8], Self> { + /* + * Read and construct a header from the provided stream. + * + * Args: + * stream: Seekable stream to read from + * + * Returns: + * FileHeader object + */ + let (input, version) = records::HEADER.read(input)?; + let (input, (mod_time, acc_time)) = records::BGNLIB.read(input)?; + let (input, name) = records::LIBNAME.skip_and_read(input)?; + let (input, (uu, dbu)) = records::UNITS.skip_and_read(input)?; + + FileHeader{ + mod_time: mod_time, + acc_time: acc_time, + name: name, + user_units_per_db_unit: uu, + meters_per_db_unit: dbu, + } + } + + pub fn write(&self, ww: W) -> OWResult { + /* + * Write the header to a stream + * + * Args: + * stream: Stream to write to + * + * Returns: + * number of bytes written + */ + let mut size = 0; + size += records::HEADER.write(stream, 600) + size += records::BGNLIB.write(stream, [self.mod_time, self.acc_time]) + size += records::LIBNAME.write(stream, self.name) + size += records::UNITS.write(stream, (self.user_units_per_db_unit, self.meters_per_db_unit)) + Ok(size) + } +} + + +pub fn scan_structs(input: &[u8]) -> HashMap::, usize> { + /* + * Scan through a GDS file, building a table of + * {b'structure_name': byte_offset}. + * The intent of this function is to enable random access + * and/or partial (structure-by-structure) reads. + * + * Args: + * stream: Seekable stream to read from. Should be positioned + * before the first structure record, but possibly + * already past the file header. + */ + let input_size = input.len(); + let positions = HashMap{}; + + let (input, header) = RecordHeader.parse(input)?; + while header.tag != records::RTAG_ENDLIB { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + if tag == records::RTAG_BGNSTR { + let (input, name) = records::STRNAME.read(input)?; + if positions.conains_key(name) { + return Err(format!("Duplicate structure name: {}", name)); + } + let position = input_size - input.len(); + positions.insert(name, position); + } + let (input, header) = RecordHeader.parse(input)?; + } + positions +} + + +pub struct Cell { + +} + +impl Cell { + pub fn write( + &self, + ww: Write, + name: &[u8], + cre_time: Option<[i16; 6]>, + mod_time: Option<[i16; 6]>, + ) -> OWResult { + /* + * Write a structure to the provided stream. + * + * Args: + * name: Structure name (ascii-encoded). + * elements: List of Elements containing the geometry and text in this struct. + * cre_time: Creation time (optional). + * mod_time: Modification time (optional). + * + * Return: + * Number of bytes written + */ + let mut size = 0; + size += BGNSTR.write(ww, (cre_time, mod_time)) + size += STRNAME.write(ww, name) + size += cell.write(ww) + size += ENDSTR.write(ww) + Ok(size) + } +} + +pub fn try_read_struct(input: &[u8]) -> IResult<&[u8], Option<(Vec, Cell>)> { + /* + * Skip to the next structure and attempt to read it. + * + * Args: + * stream: Seekable stream to read from. + * + * Returns: + * (name, elements) if a structure was found. + * None if no structure was found before the end of the library. + */ + let (input, success) = records::BGNSTR.skip_past(input)?; + if !success { + return None + } + + let (input, name) = records::STRNAME.read(input)?; + let (input, elements) = Cell::read_elements(input)?; + Some((name, elements)) +} + + + +pub fn read_elements(stream: BinaryIO) -> List[Element] { + /* + * Read elements from the stream until an ENDSTR + * record is encountered. The ENDSTR record is also + * consumed. + * + * Args: + * stream: Seekable stream to read from. + * + * Returns: + * List of element objects. + */ + let (input, header) = RecordHeader.parse(input)?; + while header.tag != records::RTAG_ENDSTR { + match header.tag { + records::RTAG_BOUNDARY => { + let (input, boundary) = records::BOUNDARY.read(input)?; + cell.boundaries.insert(boundary); + }, + records::RTAG_PATH => { + let (input, path) = records::PATH.read(input)?; + cell.paths.insert(path); + }, + records::RTAG_NODE => { + let (input, node) = records::NODE.read(input)?; + cell.nodes.insert(node); + }, + records::RTAG_BOX => { + let (input, gds_box) = records::BOX.read(input)?; + cell.boxes.insert(gds_box); + }, + records::RTAG_TEXT => { + let (input, txt) = records::TEXT.read(input)?; + cell.texts.insert(txt); + }, + records::RTAG_SREF => { + let (input, sref) = records::SREF.read(input)?; + cell.refs.insert(sref); + }, + records::RTAG_AREF => { + let (input, aref) = records::AREF.read(input)?; + cell.refs.insert(aref); + }, + _ => { + // don't care, skip + let (input, _) = nom::bytes::streaming::take(size)(input)?; + } + } + let (input, header) = RecordHeader.parse(input)?; + } + Ok((input, data)) +} + + +pub fn scan_hierarchy(input: &[u8]) -> IResult<&[u8], HashMap::, HashMap::, u32>>> { + /* + * Scan through a GDS file, building a table of instance counts + * `{b'structure_name': {b'ref_name': count}}`. + * + * This is intended to provide a fast overview of the file's + * contents without performing a full read of all elements. + * + * Args: + * stream: Seekable stream to read from. Should be positioned + * before the first structure record, but possibly + * already past the file header. + */ + let structures = HashMap{}; + + let mut ref_name = None + let mut ref_count = None + let mut cur_structure = HashMap{}; + + let (input, header) = Record.read_header(stream) + while header.tag != records::RTAG_ENDLIB { + match header.tag { + records::RTAG_BGNSTR => { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + let (input, name) = records::STRNAME.read(input)?; + if structures.contains_key(name) { + return Err(format!("Duplicate structure name: {}", name)); + } + cur_structure = HashMap{}; + structures.insert(name, cur_structure); + ref_name = None; + ref_count = None; + }, + records::RTAG_SNAME => { + let (input, sname) = SNAME.read_data(input, header.data_size)?; + ref_name = Some(sname); + }, + records::RTAG_COLROW => { + let (input, colrow) = COLROW.read_data(input, header.data_size); + ref_count = colrow[0] * colrow[1]; + }, + records::RTAG_ENDEL => { + *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); + }, + _ => { + let (input, _) = nom::bytes::streaming::take(size)(input)?; + }, + } + let (input, header) = RecordHeader.parse(input)?; + } + structures +} diff --git a/src/record.rs b/src/record.rs new file mode 100644 index 0000000..5f6775b --- /dev/null +++ b/src/record.rs @@ -0,0 +1,219 @@ +/* + * Generic record-level read/write functionality. + */ +use nom::IResult; +use std::io::Write; +use byteorder::BigEndian; + + +use basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use basic::{OWResult}; +use records; +//from .basic import parse_int2, parse_int4, parse_real8, parse_datetime, parse_bitarray +//from .basic import pack_int2, pack_int4, pack_real8, pack_datetime, pack_bitarray +//from .basic import parse_ascii, read + + +//#[no_mangle] +//pub extern "C" fn write_record_header( + + +#[repr(C)] +pub struct RecordHeader { + pub tag: u16, + pub data_size: u16, +} + +impl RecordHeader { + pub fn parse(input: &[u8]) -> IResult<&[u8], RecordHeader> { + let (_, size) = nom::number::streaming::be_u16(input[0..])?; + let (_, tag) = nom::number::streaming::be_u16(input[2..])?; + Ok((input[4..], RecordHeader{tag:tag, data_size:size - 4})) + } + + pub fn pack(self) -> [u8; 4] { + assert!(self.size < 0xffff - 4, "Record too big!"); + let vals = [self.size, self.tag]; + let mut buf = [0x77; 4]; + BigEndian::write_u16_into(&vals, &mut buf); + buf + } + + pub fn write(&self, ww: W) -> OWResult { + let bytes = self.pack(); + ww.write(bytes) + } +} + + +pub trait Record { + fn tag() -> u32; + fn expected_size() -> Option; + + //fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Self>; +} + +impl Record { + pub fn check_size(&self, actual_size: usize) -> Result<(), &str> { + match self.expected_size() { + Some(size) => if size == actual_size { + Ok(()) + } else { + Err(format!("Expected record size {}, got {}", size, actual_size)) + }, + None => Ok(()), + } + } + + pub fn parse_header(input: &[u8]) -> IResult<&[u8], RecordHeader> { + RecordHeader::parse(input) + } + + pub fn write_header(ww: W, data_size: usize) -> OWResult { + RecordHeader{tag: Self.tag(), size: data_size}.write(ww) + } + + pub fn skip_past(input: &[u8]) -> IResult<&[u8], bool> { + /* + * Skip to the end of the next occurence of this record. + * + * Return: + * True if the record was encountered and skipped. + * False if the end of the library was reached. + */ + let (input, header) = RecordHeader::parse(input)?; + while header.tag != Self.tag() { + let (input, _) = nom::bytes::streaming::take(header.size)?; + if header.tag == records::RTAG_ENDLIB { + return Ok((input, false)) + } + let (input, header) = RecordHeader::parse(input)?; + } + let (input, _) = nom::bytes::streaming::take(header.size)?; + Ok((input, true)) + } + + /* + pub fn skip_and_read(input: &[u8]) -> IResult<&[u8], bool>{ + size, tag = Record.read_header(stream) + while tag != cls.tag{ + stream.seek(size, io.SEEK_CUR) + size, tag = Record.read_header(stream) + } + data = cls.read_data(stream, size) + return data + } + + def read(cls: Type[R], stream: BinaryIO){ + size = expect_record(stream, cls.tag) + data = cls.read_data(stream, size) + return data + } + + def write(cls, stream: BinaryIO, data) -> int { + data_bytes = cls.pack_data(data) + b = cls.write_header(stream, len(data_bytes)) + b += stream.write(data_bytes) + return b + } + */ +} + + +pub trait BitArray { + fn parse_data(input: &[u8]) -> IResult<&[u8], [bool; 16]> { + parse_bitarray(input) + } + + fn pack_data(buf: &mut [u8], vals: &[bool; 16]) { + pack_bitarray(&mut buf, vals) + } +} + +pub trait Int2 { + fn parse_data(input: &[u8]) -> IResult<&[u8], i16> { + parse_int2(input) + } + + fn pack_data(buf: &mut [u8], val: i16) { + pack_int2(&mut buf, val) + } +} + +pub trait Int4 { + fn parse_data(input: &[u8]) -> IResult<&[u8], i32> { + parse_int4(input) + } + + fn pack_data(buf: &mut [u8], val: i32) { + pack_int4(&mut buf, val) + } +} + +pub trait Int2Array { + fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Vec> { + assert!(size % 2 == 0, "Record must contain an integer quantity of integers"); + nom::multi::count(parse_int2, size / 2)(input) + } + + fn pack_data(buf: &mut [u8], vals: &[i16]) { + BigEndian::write_i16_into(&vals, &mut buf) + } +} + + +pub trait Int4Array { + fn parse_data(input: &[u8], size: usize) -> IResult<&[u8], Vec> { + assert!(size % 4 == 0, "Record must contain an integer quantity of integers"); + nom::multi::count(parse_int4, size / 4)(input) + } + + fn pack_data(buf: &mut [u8], vals: &[i32]) { + BigEndian::write_i32_into(&vals, &mut buf) + } +} + +pub trait Real8 { + fn parse_data(input: &[u8]) -> IResult<&[u8], f64> { + parse_real8(input) + } + + fn pack_data(buf: &mut [u8], val: f64) { + pack_real8(&mut buf, val) + } +} + +pub trait ASCII { + fn parse_data(input: &[u8]) -> IResult<&[u8], Vec> { + parse_ascii(input) + } + + fn pack_data(buf: &mut [u8], data: &[u8]) { + pack_ascii(&mut buf, data) + } +} + +pub trait DateTime { + fn parse_data(input: &[u8]) -> IResult<&[u8], [u16; 6]> { + parse_datetime(input) + } + + fn pack_data(buf: &mut [u8], data: [u16; 6]) { + pack_datetime(&mut buf, data) + } +} + +impl DTR { + fn skip_and_read(input: &[u8]) -> IResult<&[u8], [DTR; 2]> { + let mut header = Self.read_header(input)?; + while header.tag != Self.tag() { + nom::bytes::streaming::take(header.data_size)?; + header = Self.read_header(input)?; + } + assert!(header.data_size == 6 * 2); + let data0 = Self.read_data(&input)?; + let data1 = Self.read_data(&input)?; + Ok([data0, data1]) + } +} diff --git a/src/records.rs b/src/records.rs new file mode 100644 index 0000000..92c5ad4 --- /dev/null +++ b/src/records.rs @@ -0,0 +1,595 @@ +/* + * Record type and tag definitions + */ + +use record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, DateTime, BitArray, ASCII}; +//use basic::{OWResult}; + +//use std::io::Write; + +// record tags +pub const RTAG_HEADER: u16 = 0x0002; +pub const RTAG_BGNLIB: u16 = 0x0102; +pub const RTAG_LIBNAME: u16 = 0x0206; +pub const RTAG_UNITS: u16 = 0x0305; // (user_units_per_db_unit, db_units_per_meter) +pub const RTAG_ENDLIB: u16 = 0x0400; +pub const RTAG_BGNSTR: u16 = 0x0502; +pub const RTAG_STRNAME: u16 = 0x0606; +pub const RTAG_ENDSTR: u16 = 0x0700; +pub const RTAG_BOUNDARY: u16 = 0x0800; +pub const RTAG_PATH: u16 = 0x0900; +pub const RTAG_SREF: u16 = 0x0a00; +pub const RTAG_AREF: u16 = 0x0b00; +pub const RTAG_TEXT: u16 = 0x0c00; +pub const RTAG_LAYER: u16 = 0x0d02; +pub const RTAG_DATATYPE: u16 = 0x0e02; +pub const RTAG_WIDTH: u16 = 0x0f03; +pub const RTAG_XY: u16 = 0x1003; +pub const RTAG_ENDEL: u16 = 0x1100; +pub const RTAG_SNAME: u16 = 0x1206; +pub const RTAG_COLROW: u16 = 0x1302; +pub const RTAG_NODE: u16 = 0x1500; +pub const RTAG_TEXTTYPE: u16 = 0x1602; +pub const RTAG_PRESENTATION: u16 = 0x1701; +pub const RTAG_SPACING: u16 = 0x1802; // unused; not sure about 02 +pub const RTAG_STRING: u16 = 0x1906; +pub const RTAG_STRANS: u16 = 0x1a01; +pub const RTAG_MAG: u16 = 0x1b05; +pub const RTAG_ANGLE: u16 = 0x1c05; +pub const RTAG_UINTEGER: u16 = 0x1d02; // unused; not sure about 02 +pub const RTAG_USTRING: u16 = 0x1e06; // unused; not sure about 06 +pub const RTAG_REFLIBS: u16 = 0x1f06; +pub const RTAG_FONTS: u16 = 0x2006; +pub const RTAG_PATHTYPE: u16 = 0x2102; +pub const RTAG_GENERATIONS: u16 = 0x2202; +pub const RTAG_ATTRTABLE: u16 = 0x2306; +pub const RTAG_STYPTABLE: u16 = 0x2406; // unused; not sure about 06 +pub const RTAG_STRTYPE: u16 = 0x2502; // unused +pub const RTAG_ELFLAGS: u16 = 0x2601; +pub const RTAG_ELKEY: u16 = 0x2703; // unused +pub const RTAG_LINKTYPE: u16 = 0x2803; // unused +pub const RTAG_LINKKEYS: u16 = 0x2903; // unused +pub const RTAG_NODETYPE: u16 = 0x2a02; +pub const RTAG_PROPATTR: u16 = 0x2b02; +pub const RTAG_PROPVALUE: u16 = 0x2c06; +pub const RTAG_BOX: u16 = 0x2d00; +pub const RTAG_BOXTYPE: u16 = 0x2e02; +pub const RTAG_PLEX: u16 = 0x2f03; +pub const RTAG_BGNEXTN: u16 = 0x3003; +pub const RTAG_ENDEXTN: u16 = 0x3103; +pub const RTAG_TAPENUM: u16 = 0x3202; +pub const RTAG_TAPECODE: u16 = 0x3302; +pub const RTAG_STRCLASS: u16 = 0x3401; +pub const RTAG_RESERVED: u16 = 0x3503; +pub const RTAG_FORMAT: u16 = 0x3602; +pub const RTAG_MASK: u16 = 0x3706; // list of Layers and dtypes +pub const RTAG_ENDMASKS: u16 = 0x3800; // end of MASKS records +pub const RTAG_LIBDIRSIZE: u16 = 0x3902; +pub const RTAG_SRFNAME: u16 = 0x3a06; +pub const RTAG_LIBSECUR: u16 = 0x3b02; +pub const RTAG_BORDER: u16 = 0x3c00; +pub const RTAG_SOFTFENCE: u16 = 0x3d00; +pub const RTAG_HARDFENCE: u16 = 0x3f00; +pub const RTAG_SOFTWIRE: u16 = 0x3f00; +pub const RTAG_HARDWIRE: u16 = 0x4000; +pub const RTAG_PATHPORT: u16 = 0x4100; +pub const RTAG_NODEPORT: u16 = 0x4200; +pub const RTAG_USERCONSTRAINT: u16 = 0x4300; +pub const RTAG_SPACERERROR: u16 = 0x4400; +pub const RTAG_CONTACT: u16 = 0x4500; + +/* +// data types +pub const DATA_TYPE_NONE: u16 = 0x00; +pub const DATA_TYPE_BIT: u16 = 0x01; +pub const DATA_TYPE_INT16: u16 = 0x02; +pub const DATA_TYPE_INT32: u16 = 0x03; +pub const DATA_TYPE_REAL32: u16 = 0x04; +pub const DATA_TYPE_REAL64: u16 = 0x05; +pub const DATA_TYPE_STR: u16 = 0x06; + +pub const MAX_DATA_SIZE: usize = 8; + +/// Returns the size of the given data type in bytes. +pub fn data_size(t: u16) -> usize { + match t { + x if x == DATA_TYPE_NONE => 0, + x if x == DATA_TYPE_BIT => 2, + x if x == DATA_TYPE_INT16 => 2, + x if x == DATA_TYPE_INT32 => 4, + x if x == DATA_TYPE_REAL32 => 4, + x if x == DATA_TYPE_REAL64 => 8, + _ => 0 + } +*/ + +pub struct HEADER {} +impl Record for HEADER { + fn tag() -> u16 { RTAG_HEADER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for HEADER {} + +pub struct BGNLIB {} +impl Record for BGNLIB { + fn tag() -> u16 { RTAG_BGNLIB } + fn expected_size() -> usize { Some(2 * 6) } +} +impl DateTime for BGNLIB {} + +pub struct LIBNAME {} +impl Record for LIBNAME { + fn tag() -> u16 { RTAG_LIBNAME } + fn expected_size() -> usize { None } +} +impl ASCII for LIBNAME {} + +pub struct UNITS {} +impl Record for UNITS { + // (user_units_per_db_unit, db_units_per_meter) + fn tag() -> u16 { RTAG_UNITS } + fn expected_size() -> usize { Some(2 * 8) } +} +impl Real8 for UNITS {} + +pub struct ENDLIB {} +impl Record for ENDLIB { + fn tag() -> u16 { RTAG_ENDLIB } + fn expected_size() -> usize { Some(0) } +} + +pub struct BGNSTR {} +impl Record for BGNSTR { + fn tag() -> u16 { RTAG_BGNSTR } + fn expected_size() -> usize { Some(2 * 6) } +} +impl DateTime for ENDLIB {} + +pub struct STRNAME {} +impl Record for STRNAME { + fn tag() -> u16 { RTAG_STRNAME } + fn expected_size() -> usize { Some(2 * 6) } +} +impl ASCII for STRNAME {} + +pub struct ENDSTR {} +impl Record for ENDSTR { + fn tag() -> u16 { RTAG_ENDSTR } + fn expected_size() -> usize { Some(0) } +} + +pub struct BOUNDARY {} +impl Record for BOUNDARY { + fn tag() -> u16 { RTAG_BOUNDARY } + fn expected_size() -> usize { Some(0) } +} + +pub struct PATH {} +impl Record for PATH { + fn tag() -> u16 { RTAG_PATH } + fn expected_size() -> usize { Some(0) } +} + +pub struct SREF {} +impl Record for SREF { + fn tag() -> u16 { RTAG_SREF } + fn expected_size() -> usize { Some(0) } +} + +pub struct AREF {} +impl Record for AREF { + fn tag() -> u16 { RTAG_AREF } + fn expected_size() -> usize { Some(0) } +} + +pub struct TEXT {} +impl Record for TEXT { + fn tag() -> u16 { RTAG_TEXT } + fn expected_size() -> usize { Some(0) } +} + +pub struct LAYER {} +impl Record for LAYER { + fn tag() -> u16 { RTAG_LAYER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LAYER {} + +pub struct DATATYPE {} +impl Record for DATATYPE { + fn tag() -> u16 { RTAG_DATATYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for DATATYPE {} + +pub struct WIDTH {} +impl Record for WIDTH { + fn tag() -> u16 { RTAG_WIDTH } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for WIDTH {} + +pub struct XY {} +impl Record for XY { + fn tag() -> u16 { RTAG_XY } + fn expected_size() -> usize { None } +} +impl Int4Array for XY {} + +pub struct ENDEL {} +impl Record for ENDEL { + fn tag() -> u16 { RTAG_ENDEL } + fn expected_size() -> usize { Some(0) } +} + +pub struct SNAME {} +impl Record for SNAME { + fn tag() -> u16 { RTAG_SNAME } + fn expected_size() -> usize { None } +} +impl ASCII for SNAME {} + +pub struct COLROW {} +impl Record for COLROW { + fn tag() -> u16 { RTAG_COLROW } + fn expected_size() -> usize { Some(4) } +} +impl Int2Array for COLROW {} + +pub struct NODE {} +impl Record for NODE { + fn tag() -> u16 { RTAG_NODE } + fn expected_size() -> usize { Some(0) } +} + +pub struct TEXTTYPE {} +impl Record for TEXTTYPE { + fn tag() -> u16 { RTAG_TEXTTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for TEXTTYPE {} + +pub struct PRESENTATION {} +impl Record for PRESENTATION { + fn tag() -> u16 { RTAG_PRESENTATION } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for PRESENTATION {} + +pub struct SPACING {} +impl Record for SPACING { + fn tag() -> u16 { RTAG_SPACING } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for SPACING {} + +pub struct STRING {} +impl Record for STRING { + fn tag() -> u16 { RTAG_STRING } + fn expected_size() -> usize { None } +} +impl ASCII for STRING {} + +pub struct STRANS {} +impl Record for STRANS { + fn tag() -> u16 { RTAG_STRANS } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for STRANS {} + +pub struct MAG {} +impl Record for MAG { + fn tag() -> u16 { RTAG_MAG } + fn expected_size() -> usize { Some(8) } +} +impl Real8 for MAG {} + +pub struct ANGLE {} +impl Record for ANGLE { + fn tag() -> u16 { RTAG_ANGLE } + fn expected_size() -> usize { Some(8) } +} +impl Real8 for ANGLE {} + +pub struct UINTEGER {} +impl Record for UINTEGER { + fn tag() -> u16 { RTAG_UINTEGER } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for UINTEGER {} + +pub struct USTRING {} +impl Record for USTRING { + fn tag() -> u16 { RTAG_USTRING } + fn expected_size() -> usize { None } +} +impl ASCII for USTRING {} + +pub struct REFLIBS {} +impl Record for REFLIBS { + fn tag() -> u16 { RTAG_REFLIBS } + fn expected_size() -> usize { None } +} +impl REFLIBS { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for REFLIBS {} + +pub struct FONTS {} +impl Record for FONTS { + fn tag() -> u16 { RTAG_FONTS } + fn expected_size() -> usize { None } +} +impl FONTS { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for FONTS {} + +pub struct PATHTYPE {} +impl Record for PATHTYPE { + fn tag() -> u16 { RTAG_PATHTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for PATHTYPE {} + +pub struct GENERATIONS {} +impl Record for GENERATIONS { + fn tag() -> u16 { RTAG_GENERATIONS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for GENERATIONS {} + +pub struct ATTRTABLE {} +impl Record for ATTRTABLE { + fn tag() -> u16 { RTAG_ATTRTABLE } + fn expected_size() -> usize { None } +} +impl ATTRTABLE { + fn check_size(&self, actual_size: usize) -> Result<(), &str> { + if actual_size % 44 == 0 { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} +impl ASCII for ATTRTABLE {} + +pub struct STYPTABLE {} +impl Record for STYPTABLE { + fn tag() -> u16 { RTAG_STYPTABLE } + fn expected_size() -> usize { None } +} +impl ASCII for STYPTABLE {} + +pub struct STRTYPE {} +impl Record for STRTYPE { + fn tag() -> u16 { RTAG_STRTYPE } + fn expected_size() -> usize { None } +} +impl Int2 for STRTYPE {} + +pub struct ELFLAGS {} +impl Record for ELFLAGS { + fn tag() -> u16 { RTAG_ELFLAGS } + fn expected_size() -> usize { Some(2) } +} +impl BitArray for ELFLAGS {} + +pub struct ELKEY {} +impl Record for ELKEY { + fn tag() -> u16 { RTAG_ELKEY } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for ELKEY {} + +pub struct LINKTYPE {} +impl Record for LINKTYPE { + fn tag() -> u16 { RTAG_LINKTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LINKTYPE {} + +pub struct LINKKEYS {} +impl Record for LINKKEYS { + fn tag() -> u16 { RTAG_LINKKEYS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LINKKEYS {} + +pub struct NODETYPE {} +impl Record for NODETYPE { + fn tag() -> u16 { RTAG_NODETYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for NODETYPE {} + +pub struct PROPATTR {} +impl Record for PROPATTR { + fn tag() -> u16 { RTAG_PROPATTR } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for PROPATTR {} + +pub struct PROPVALUE {} +impl Record for PROPVALUE { + fn tag() -> u16 { RTAG_PROPVALUE } + fn expected_size() -> usize { Some(2) } +} +impl ASCII for PROPVALUE {} + +pub struct BOX {} +impl Record for BOX { + fn tag() -> u16 { RTAG_BOX } + fn expected_size() -> usize { Some(0) } +} + +pub struct BOXTYPE {} +impl Record for BOXTYPE { + fn tag() -> u16 { RTAG_BOXTYPE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for BOXTYPE {} + +pub struct PLEX {} +impl Record for PLEX { + fn tag() -> u16 { RTAG_PLEX } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for PLEX {} + +pub struct BGNEXTN {} +impl Record for BGNEXTN { + fn tag() -> u16 { RTAG_BGNEXTN } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for BGNEXTN {} + +pub struct ENDEXTN {} +impl Record for ENDEXTN { + fn tag() -> u16 { RTAG_ENDEXTN } + fn expected_size() -> usize { Some(4) } +} +impl Int4 for ENDEXTN {} + +pub struct TAPENUM {} +impl Record for TAPENUM { + fn tag() -> u16 { RTAG_TAPENUM } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for TAPENUM {} + +pub struct TAPECODE {} +impl Record for TAPECODE { + fn tag() -> u16 { RTAG_TAPECODE } + fn expected_size() -> usize { Some(2 * 6) } +} +impl Int2Array for TAPECODE {} + +pub struct STRCLASS {} +impl Record for STRCLASS { + fn tag() -> u16 { RTAG_STRCLASS } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for STRCLASS {} + +pub struct RESERVED {} +impl Record for RESERVED { + fn tag() -> u16 { RTAG_RESERVED } + fn expected_size() -> usize { Some(2) } +} +impl Int2Array for RESERVED {} + +pub struct FORMAT {} +impl Record for FORMAT { + fn tag() -> u16 { RTAG_FORMAT } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for FORMAT {} + +pub struct MASK {} +impl Record for MASK { + fn tag() -> u16 { RTAG_MASK } + fn expected_size() -> usize { None } +} +impl ASCII for MASK {} + +pub struct ENDMASKS {} +impl Record for ENDMASKS { + // End of MASKS records + fn tag() -> u16 { RTAG_ENDMASKS } + fn expected_size() -> usize { Some(0) } +} + +pub struct LIBDIRSIZE {} +impl Record for LIBDIRSIZE { + fn tag() -> u16 { RTAG_LIBDIRSIZE } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LIBDIRSIZE {} + +pub struct SRFNAME {} +impl Record for SRFNAME { + fn tag() -> u16 { RTAG_SRFNAME } + fn expected_size() -> usize { None } +} +impl ASCII for SRFNAME {} + +pub struct LIBSECUR {} +impl Record for LIBSECUR { + fn tag() -> u16 { RTAG_LIBSECUR } + fn expected_size() -> usize { Some(2) } +} +impl Int2 for LIBSECUR {} + +pub struct BORDER {} +impl Record for BORDER { + fn tag() -> u16 { RTAG_BORDER } + fn expected_size() -> usize { Some(0) } +} + +pub struct SOFTFENCE {} +impl Record for SOFTFENCE { + fn tag() -> u16 { RTAG_SOFTFENCE } + fn expected_size() -> usize { Some(0) } +} + +pub struct HARDFENCE {} +impl Record for HARDFENCE { + fn tag() -> u16 { RTAG_HARDFENCE } + fn expected_size() -> usize { Some(0) } +} + +pub struct SOFTWIRE {} +impl Record for SOFTWIRE { + fn tag() -> u16 { RTAG_SOFTWIRE } + fn expected_size() -> usize { Some(0) } +} + +pub struct HARDWIRE {} +impl Record for HARDWIRE { + fn tag() -> u16 { RTAG_HARDWIRE } + fn expected_size() -> usize { Some(0) } +} + +pub struct PATHPORT {} +impl Record for PATHPORT { + fn tag() -> u16 { RTAG_PATHPORT } + fn expected_size() -> usize { Some(0) } +} + +pub struct NODEPORT {} +impl Record for NODEPORT { + fn tag() -> u16 { RTAG_NODEPORT } + fn expected_size() -> usize { Some(0) } +} + +pub struct USERCONSTRAINT {} +impl Record for USERCONSTRAINT { + fn tag() -> u16 { RTAG_USERCONSTRAINT } + fn expected_size() -> usize { Some(0) } +} + +pub struct SPACERERROR {} +impl Record for SPACERERROR { + fn tag() -> u16 { RTAG_SPACERERROR } + fn expected_size() -> usize { Some(0) } +} + +pub struct CONTACT {} +impl Record for CONTACT { + fn tag() -> u16 { RTAG_CONTACT } + fn expected_size() -> usize { Some(0) } +} diff --git a/src/test_basic.rs b/src/test_basic.rs new file mode 100644 index 0000000..6bd8bee --- /dev/null +++ b/src/test_basic.rs @@ -0,0 +1,121 @@ +/* +import struct + +import pytest # type: ignore +import numpy # type: ignore +from numpy.testing import assert_array_equal # type: ignore + +from .basic import parse_bitarray, parse_int2, parse_int4, parse_real8, parse_ascii +from .basic import pack_bitarray, pack_int2, pack_int4, pack_real8, pack_ascii +from .basic import decode_real8, encode_real8 + +from .basic import KlamathError + + +def test_parse_bitarray(): + assert(parse_bitarray(b'59') == 13625) + assert(parse_bitarray(b'\0\0') == 0) + assert(parse_bitarray(b'\xff\xff') == 65535) + + # 4 bytes (too long) + with pytest.raises(KlamathError): + parse_bitarray(b'4321') + + # empty data + with pytest.raises(KlamathError): + parse_bitarray(b'') + + +def test_parse_int2(): + assert_array_equal(parse_int2(b'59\xff\xff\0\0'), (13625, -1, 0)) + + # odd length + with pytest.raises(KlamathError): + parse_int2(b'54321') + + # empty data + with pytest.raises(KlamathError): + parse_int2(b'') + + +def test_parse_int4(): + assert_array_equal(parse_int4(b'4321'), (875770417,)) + + # length % 4 != 0 + with pytest.raises(KlamathError): + parse_int4(b'654321') + + # empty data + with pytest.raises(KlamathError): + parse_int4(b'') + + +def test_decode_real8(): + # zeroes + assert(decode_real8(numpy.array([0x0])) == 0) + assert(decode_real8(numpy.array([1<<63])) == 0) # negative + assert(decode_real8(numpy.array([0xff << 56])) == 0) # denormalized + + assert(decode_real8(numpy.array([0x4110 << 48])) == 1.0) + assert(decode_real8(numpy.array([0xC120 << 48])) == -2.0) + + +def test_parse_real8(): + packed = struct.pack('>3Q', 0x0, 0x4110_0000_0000_0000, 0xC120_0000_0000_0000) + assert_array_equal(parse_real8(packed), (0.0, 1.0, -2.0)) + + # length % 8 != 0 + with pytest.raises(KlamathError): + parse_real8(b'0987654321') + + # empty data + with pytest.raises(KlamathError): + parse_real8(b'') + + +def test_parse_ascii(): + # empty data + with pytest.raises(KlamathError): + parse_ascii(b'') + + assert(parse_ascii(b'12345') == b'12345') + assert(parse_ascii(b'12345\0') == b'12345') # strips trailing null byte + + +def test_pack_bitarray(): + packed = pack_bitarray(321) + assert(len(packed) == 2) + assert(packed == struct.pack('>H', 321)) + + +def test_pack_int2(): + packed = pack_int2((3, 2, 1)) + assert(len(packed) == 3*2) + assert(packed == struct.pack('>3h', 3, 2, 1)) + assert(pack_int2([-3, 2, -1]) == struct.pack('>3h', -3, 2, -1)) + + +def test_pack_int4(): + packed = pack_int4((3, 2, 1)) + assert(len(packed) == 3*4) + assert(packed == struct.pack('>3l', 3, 2, 1)) + assert(pack_int4([-3, 2, -1]) == struct.pack('>3l', -3, 2, -1)) + + +def test_encode_real8(): + assert(encode_real8(numpy.array([0.0])) == 0) + arr = numpy.array((1.0, -2.0, 1e-9, 1e-3, 1e-12)) + assert_array_equal(decode_real8(encode_real8(arr)), arr) + + +def test_pack_real8(): + reals = (0, 1, -1, 0.5, 1e-9, 1e-3, 1e-12) + packed = pack_real8(reals) + assert(len(packed) == len(reals) * 8) + assert_array_equal(parse_real8(packed), reals) + + +def test_pack_ascii(): + assert(pack_ascii(b'4321') == b'4321') + assert(pack_ascii(b'321') == b'321\0') +*/ From e0a634f47665427046499f8afd3c1cbfadef058a Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 18 Dec 2021 21:17:43 -0800 Subject: [PATCH 02/41] add gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d80173 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target/ +**/*.rs.bk +Cargo.lock + +*.swp +*.swo From 00a29941b7f393a05f03a3f4760e627f1972511d Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 20 Mar 2022 16:28:34 -0700 Subject: [PATCH 03/41] index on (no branch): 48db472 snapshot 2021-12-18 21:05:00.635887 From 02e483c8cdaba081518076aeeecaba41265225f7 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 8 May 2022 16:41:43 -0700 Subject: [PATCH 04/41] update to 2021 edition --- Cargo.toml | 3 ++- src/__init__.rs | 29 ----------------------------- src/elements.rs | 8 ++++---- src/lib.rs | 1 - src/library.rs | 27 ++++++++++++++------------- src/record.rs | 8 ++++---- src/records.rs | 2 +- 7 files changed, 25 insertions(+), 53 deletions(-) delete mode 100644 src/__init__.rs diff --git a/Cargo.toml b/Cargo.toml index 414196a..01429e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,8 @@ name = "rs-klamath" version = "0.1.0" authors = ["jan "] +edition = "2021" [dependencies] byteorder = "^1" -nom = "^7" +#nom = "^7" diff --git a/src/__init__.rs b/src/__init__.rs deleted file mode 100644 index 44d6c6b..0000000 --- a/src/__init__.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * `klamath` is a Python module for reading and writing to the GDSII file format. - * - * The goal is to keep this library simple: - * - Map data types directly wherever possible. - * * Presents an accurate representation of what is saved to the file. - * * Avoids excess copies / allocations for speed. - * * No "automatic" error checking, except when casting datatypes. - * If data integrity checks are provided at all, they must be - * explicitly run by the caller. - * - Low-level functionality is first-class. - * * Meant for use-cases where the caller wants to read or write - * individual GDS records. - * * Offers complete control over the written file. - * - Opinionated and limited high-level functionality. - * * Discards or ignores rarely-encountered data types. - * * Keeps functions simple and reusable. - * * Only de/encodes the file format, doesn't provide tools to modify - * the data itself. - * * Still requires explicit values for most fields. - * - No compilation - * * Uses `numpy` for speed, since it's commonly available / pre-built. - * * Building this library should not require a compiler. - * - * `klamath` was built to provide a fast and versatile GDS interface for - * [masque](https://mpxd.net/code/jan/masque), which provides higher-level - * tools for working with hierarchical design data and supports multiple - * file formats. - */ diff --git a/src/elements.rs b/src/elements.rs index 8c2e35b..516a622 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -3,16 +3,16 @@ /// structure references) and associated properties. /// -use records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, +use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, ENDEL, BGNEXTN, ENDEXTN, SNAME, }; -use records; -use record::{RecordHeader, Record}; -use basic::{OResult, IResult, fail}; +use crate::records; +use crate::record::{RecordHeader, Record}; +use crate::basic::{OResult, IResult, fail}; use std::collections::HashMap; use std::io::Write; diff --git a/src/lib.rs b/src/lib.rs index b17d455..dbaf483 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,4 @@ //#![feature(generic_associated_types)] -#![feature(destructuring_assignment)] extern crate byteorder; diff --git a/src/library.rs b/src/library.rs index 06ff7ba..84d74dc 100644 --- a/src/library.rs +++ b/src/library.rs @@ -5,12 +5,12 @@ use std::io::Write; use std::collections::HashMap; -use record; -use record::{RecordHeader, Record}; -use records; -use elements; -use elements::{Element}; -use basic::{IResult, OResult, take_bytes, fail}; +use crate::record; +use crate::record::{RecordHeader, Record}; +use crate::records; +use crate::elements; +use crate::elements::{Element}; +use crate::basic::{IResult, OResult, take_bytes, fail}; const DEFAULT_DATE: [i16; 6] = [1900, 0, 0, 0, 0, 0]; @@ -289,6 +289,7 @@ impl Cell { } +/* /// /// Scan through a GDS file, building a table of instance counts /// `{b'structure_name': {b'ref_name': count}}`. @@ -304,8 +305,10 @@ impl Cell { pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap::, u32>>> { let mut structures = HashMap::new(); - + let mut ref_name = None; + let mut ref_count = None; let (mut input, mut header) = RecordHeader::read(input)?; + let mut cur_structure = HashMap::new(); while header.tag != records::RTAG_ENDLIB { match header.tag { records::RTAG_BGNSTR => { @@ -317,8 +320,6 @@ pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap:: IResult, HashMap:: IResult, u32))> { let (input, found_struc) = records::BGNSTR.skip_past(input)?; - if not found_struc { + if !found_struc { return Ok((input, None)) } let mut cur_structure = HashMap::new(); @@ -390,3 +390,4 @@ pub fn count_ref(input: &[u8]) -> IResult, u32))> { structures.insert(name, cur_structure); (input, header) = RecordHeader::read(input1)?; } +*/ diff --git a/src/record.rs b/src/record.rs index 8d01c93..7d04085 100644 --- a/src/record.rs +++ b/src/record.rs @@ -6,10 +6,10 @@ use std::convert::TryInto; use byteorder::{ByteOrder, BigEndian}; -use basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use basic::{OResult, IResult, fail, parse_u16, take_bytes}; -use records; +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; +use crate::records; //#[no_mangle] diff --git a/src/records.rs b/src/records.rs index 972cc78..21a5bb0 100644 --- a/src/records.rs +++ b/src/records.rs @@ -2,7 +2,7 @@ /// Record type and tag definitions /// -use record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; +use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; //use std::io::Write; From 921afa556958054e60fe86ccb706b9ec4439bb32 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 20 May 2022 19:44:17 -0700 Subject: [PATCH 05/41] whitespace --- src/elements.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/elements.rs b/src/elements.rs index 516a622..b711da2 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,11 +4,11 @@ /// use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, - DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, - LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, - STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, - ENDEL, BGNEXTN, ENDEXTN, SNAME, - }; + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; use crate::records; use crate::record::{RecordHeader, Record}; From e5e0adab7119bd0899059a4b20057e9326a52f3e Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:38:40 -0800 Subject: [PATCH 06/41] fixup imports/exports --- src/basic.rs | 12 ++++++------ src/library.rs | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index d8fa7da..cd22d6f 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -232,7 +232,7 @@ pub fn encode_real8(fnum: f64) -> u64 { mod tests { #[test] fn test_parse_bitarray() { - use basic::parse_bitarray; + use crate::basic::parse_bitarray; //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); @@ -251,7 +251,7 @@ mod tests { #[test] fn test_parse_int2() { - use basic::parse_int2; + use crate::basic::parse_int2; assert_eq!(parse_int2(b"59").unwrap().1, 13625); assert_eq!(parse_int2(b"\0\0").unwrap().1, 0); assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1); @@ -259,13 +259,13 @@ mod tests { #[test] fn test_parse_int4() { - use basic::parse_int4; + use crate::basic::parse_int4; assert_eq!(parse_int4(b"4321").unwrap().1, 875770417); } #[test] fn test_decode_real8() { - use basic::decode_real8; + use crate::basic::decode_real8; // zeroes assert_eq!(decode_real8(0x0), 0.0); @@ -280,7 +280,7 @@ mod tests { #[test] fn test_parse_real8() { - use basic:: parse_real8; + use crate::basic:: parse_real8; assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1); assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1); @@ -289,7 +289,7 @@ mod tests { #[test] fn test_parse_ascii() { - use basic::parse_ascii; + use crate::basic::parse_ascii; assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345"); assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte diff --git a/src/library.rs b/src/library.rs index 84d74dc..e5b5758 100644 --- a/src/library.rs +++ b/src/library.rs @@ -5,12 +5,12 @@ use std::io::Write; use std::collections::HashMap; -use crate::record; -use crate::record::{RecordHeader, Record}; -use crate::records; -use crate::elements; -use crate::elements::{Element}; -use crate::basic::{IResult, OResult, take_bytes, fail}; +pub use crate::record; +pub use crate::record::{RecordHeader, Record}; +pub use crate::records; +pub use crate::elements; +pub use crate::elements::{Element}; +pub use crate::basic::{IResult, OResult, take_bytes, fail}; const DEFAULT_DATE: [i16; 6] = [1900, 0, 0, 0, 0, 0]; From 24d8e32173ee8538a9110ca2d4c2129c345a4542 Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:38:50 -0800 Subject: [PATCH 07/41] add leading zeros to mask --- src/basic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic.rs b/src/basic.rs index cd22d6f..0899fa9 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -161,7 +161,7 @@ pub fn encode_real8(fnum: f64) -> u64 { let ieee = fnum.to_bits(); let sign = ieee & 0x8000_0000_0000_0000; let ieee_exp = (ieee >> 52) as i32 & 0x7ff; - let ieee_mant = ieee & 0xf_ffff_ffff_ffff; + let ieee_mant = ieee & 0x000f_ffff_ffff_ffff; let subnorm = (ieee_exp == 0) & (ieee_mant != 0); if (ieee_exp == 0) & (ieee_mant == 0) { From 6531bec6e5282e86a7e9738b3d65fc2d828958ae Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 17 Dec 2024 18:39:02 -0800 Subject: [PATCH 08/41] ErrType should be Debug --- src/basic.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/basic.rs b/src/basic.rs index 0899fa9..0fa8392 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -8,6 +8,7 @@ pub type OResult = Result; pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>; +#[derive(Debug)] pub enum ErrType { Incomplete(Option), Failed(String), From 6186f1d5613a9bb94652598da6f57317e307b95e Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:43:41 -0800 Subject: [PATCH 09/41] enable more tests --- src/basic.rs | 81 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 0fa8392..72fd03a 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -297,43 +297,80 @@ mod tests { assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456"); } -/* + #[test] fn test_pack_bitarray() { - packed = pack_bitarray(321) - assert_eq!(len(packed), 2); - assert_eq!(packed, struct.pack('>H', 321)); + use crate::basic::pack_bitarray; + let mut buf = [10; 3]; + let mut bools = [false; 16]; + bools[1] = true; + bools[2] = true; + bools[11] = true; + + pack_bitarray(&mut buf, &bools); + assert_eq!(buf[0], 0b0110_0000); + assert_eq!(buf[1], 0b0001_0000); + assert_eq!(buf[2], 10); } + #[test] fn test_pack_int2() { - packed = pack_int2((3, 2, 1)) - assert(len(packed) == 3*2) - assert(packed == struct.pack('>3h', 3, 2, 1)) - assert(pack_int2([-3, 2, -1]) == struct.pack('>3h', -3, 2, -1)) + use crate::basic::pack_int2; + let mut buf = [10; 3 * 2]; + pack_int2(&mut buf, -3); + pack_int2(&mut buf[2..], 2); + pack_int2(&mut buf[4..], -1); + assert_eq!(buf[0..2], [0xFF, 0xFD]); + assert_eq!(buf[2..4], [0x00, 0x02]); + assert_eq!(buf[4..6], [0xFF, 0xFF]); } + #[test] fn test_pack_int4() { - packed = pack_int4((3, 2, 1)) - assert(len(packed) == 3*4) - assert(packed == struct.pack('>3l', 3, 2, 1)) - assert(pack_int4([-3, 2, -1]) == struct.pack('>3l', -3, 2, -1)) + use crate::basic::pack_int4; + let mut buf = [10; 3 * 4]; + pack_int4(&mut buf, -3); + pack_int4(&mut buf[4..], 2); + pack_int4(&mut buf[8..], -1); + assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]); + assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]); + assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]); } + #[test] fn test_encode_real8() { - assert(encode_real8(numpy.array([0.0])) == 0) - arr = numpy.array((1.0, -2.0, 1e-9, 1e-3, 1e-12)) - assert_array_equal(decode_real8(encode_real8(arr)), arr) + use crate::basic::{encode_real8, decode_real8}; + const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; + for vv in REALS { + print!("{vv}\n"); + assert!((decode_real8(encode_real8(vv)) - vv).abs() < f64::EPSILON); + } } + #[test] fn test_pack_real8() { - reals = (0, 1, -1, 0.5, 1e-9, 1e-3, 1e-12) - packed = pack_real8(reals) - assert(len(packed) == len(reals) * 8) - assert_array_equal(parse_real8(packed), reals) + use crate::basic::{pack_real8, parse_real8}; + const COUNT: usize = 7; + const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; + let mut buf = [10; 8 * COUNT]; + for (ii, &vv) in REALS.iter().enumerate() { + pack_real8(&mut buf[ii * 8..], vv); + } + for (ii, &vv) in REALS.iter().enumerate() { + print!("{vv}\n"); + let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1; + assert!((parsed_val - vv).abs() < f64::EPSILON); + } } + #[test] fn test_pack_ascii() { - assert(pack_ascii(b'4321') == b'4321') - assert(pack_ascii(b'321') == b'321\0') + use crate::basic::pack_ascii; + let mut buf = [10; 12]; + pack_ascii(&mut buf[0..], "4321".as_bytes()); + pack_ascii(&mut buf[6..], "321".as_bytes()); + assert_eq!(&buf[0..4], "4321".as_bytes()); + assert_eq!(&buf[4..6], [10, 10]); + assert_eq!(&buf[6..9], "321".as_bytes()); + assert_eq!(&buf[9..], [0, 10, 10]); } -*/ } From b049f70dce1a20899340f33e060fa0c60e0e1876 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:43:50 -0800 Subject: [PATCH 10/41] check_size should be public --- src/records.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/records.rs b/src/records.rs index 21a5bb0..4fe21fa 100644 --- a/src/records.rs +++ b/src/records.rs @@ -290,7 +290,7 @@ impl Record for REFLIBS { fn expected_size() -> Option { None } } impl REFLIBS { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { @@ -305,7 +305,7 @@ impl Record for FONTS { fn expected_size() -> Option { None } } impl FONTS { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { @@ -332,7 +332,7 @@ impl Record for ATTRTABLE { fn expected_size() -> Option { None } } impl ATTRTABLE { - fn check_size(actual_size: usize) -> Result<(), String> { + pub fn check_size(actual_size: usize) -> Result<(), String> { if actual_size % 44 == 0 { Ok(()) } else { From a1e9a3f90ca0e8d832ef2ad39618c0096e9f4c28 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:44:05 -0800 Subject: [PATCH 11/41] fix incomplete condition --- src/basic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic.rs b/src/basic.rs index 72fd03a..0bd0636 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -25,7 +25,7 @@ pub fn incomplete(input: &[u8], size: Option) -> IResult { pub fn take_bytes>(input: &[u8], count: CC) -> IResult<&[u8]> { let cc = count.into(); - if input.len() > cc { + if input.len() < cc { incomplete(input, Some(cc)) } else { let (taken, input) = input.split_at(cc); From babd7f14025b15a6aeab621e1d812b3026fba609 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 16:51:42 -0800 Subject: [PATCH 12/41] test for panic if float is too large for real8 --- src/basic.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 0bd0636..016cc6f 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -275,10 +275,16 @@ mod tests { assert_eq!(decode_real8(0x4110 << 48), 1.0); assert_eq!(decode_real8(0xC120 << 48), -2.0); - - //TODO panics on invalid? } + #[test] + #[should_panic] + fn test_encode_real8_panic() { + use crate::basic::encode_real8; + encode_real8(1e80); + } + + #[test] fn test_parse_real8() { use crate::basic:: parse_real8; From e42ac03c95aa96d38d5637f3e639fe3d8703ad3e Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 18 Dec 2024 17:27:10 -0800 Subject: [PATCH 13/41] add FloatTooBigError --- src/basic.rs | 39 +++++++++++++++++++++++++-------------- src/record.rs | 6 +++--- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/basic.rs b/src/basic.rs index 016cc6f..1019be1 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -3,6 +3,7 @@ /// use byteorder::{ByteOrder, BigEndian}; use std::io; +use std::fmt; pub type OResult = Result; @@ -14,7 +15,6 @@ pub enum ErrType { Failed(String), } - pub fn fail(input: &[u8], msg: String) -> IResult { Err((input, ErrType::Failed(msg))) } @@ -118,20 +118,21 @@ pub fn bitarray2int(bits: &[bool; 16]) -> u16 { } pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { - BigEndian::write_u16(buf, bitarray2int(bits)) + BigEndian::write_u16(buf, bitarray2int(bits)); } pub fn pack_int2(buf: &mut [u8], int: i16) { - BigEndian::write_i16(buf, int) + BigEndian::write_i16(buf, int); } pub fn pack_int4(buf: &mut [u8], int: i32) { - BigEndian::write_i32(buf, int) + BigEndian::write_i32(buf, int); } -pub fn pack_real8(buf: &mut [u8], fnum: f64) { - BigEndian::write_u64(buf, encode_real8(fnum)) +pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { + BigEndian::write_u64(buf, encode_real8(fnum)?); + Ok(()) } pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { @@ -156,8 +157,18 @@ pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) { } +#[derive(Debug, Clone)] +pub struct FloatTooBigError { + float_value: f64, +} +impl fmt::Display for FloatTooBigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Float {0} is too large for Real8", self.float_value) + } +} + /// Convert from float64 to GDS REAL8 representation. -pub fn encode_real8(fnum: f64) -> u64 { +pub fn encode_real8(fnum: f64) -> Result { // Split the ieee float bitfields let ieee = fnum.to_bits(); let sign = ieee & 0x8000_0000_0000_0000; @@ -166,7 +177,7 @@ pub fn encode_real8(fnum: f64) -> u64 { let subnorm = (ieee_exp == 0) & (ieee_mant != 0); if (ieee_exp == 0) & (ieee_mant == 0) { - return 0 + return Ok(0) } // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) @@ -207,7 +218,7 @@ pub fn encode_real8(fnum: f64) -> u64 { if gds_exp < -14 { // number is too small - return 0 + return Ok(0) } let neg_biased = gds_exp < 0; @@ -218,13 +229,13 @@ pub fn encode_real8(fnum: f64) -> u64 { let too_big = (gds_exp > 0x7f) & !subnorm; if too_big { - panic!("Number too big for real8 format"); //TODO error handling + return Err(FloatTooBigError{float_value: fnum}); } let gds_exp_bits = (gds_exp as u64) << 56; let real8 = sign | gds_exp_bits | gds_mant; - real8 + Ok(real8) } @@ -281,7 +292,7 @@ mod tests { #[should_panic] fn test_encode_real8_panic() { use crate::basic::encode_real8; - encode_real8(1e80); + encode_real8(1e80).unwrap(); } @@ -348,7 +359,7 @@ mod tests { const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; for vv in REALS { print!("{vv}\n"); - assert!((decode_real8(encode_real8(vv)) - vv).abs() < f64::EPSILON); + assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON); } } @@ -359,7 +370,7 @@ mod tests { const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; let mut buf = [10; 8 * COUNT]; for (ii, &vv) in REALS.iter().enumerate() { - pack_real8(&mut buf[ii * 8..], vv); + pack_real8(&mut buf[ii * 8..], vv).unwrap(); } for (ii, &vv) in REALS.iter().enumerate() { print!("{vv}\n"); diff --git a/src/record.rs b/src/record.rs index 7d04085..41921c4 100644 --- a/src/record.rs +++ b/src/record.rs @@ -279,7 +279,7 @@ impl RecordData for Real8 { } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(buf, *data) + pack_real8(buf, *data).expect(&format!("Float {0} too big for Real8", data)) } fn pack(data: &Self::InData) -> Self::ByteData { @@ -303,8 +303,8 @@ impl RecordData for Real8Pair { } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(&mut buf[8 * 0..], data.0); - pack_real8(&mut buf[8 * 1..], data.1); + pack_real8(&mut buf[8 * 0..], data.0).expect(&format!("Float.0 {0} too big for Real8", data.0)); + pack_real8(&mut buf[8 * 1..], data.1).expect(&format!("Float.1 {0} too big for Real8", data.1)); } fn pack(data: &Self::InData) -> Self::ByteData { From ad1c2f1c35ed96b9878631706b59f60acafd720b Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 19 Dec 2024 19:02:55 -0800 Subject: [PATCH 14/41] remove commented code --- src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dbaf483..2a34034 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,3 @@ -//#![feature(generic_associated_types)] - extern crate byteorder; pub mod basic; From b3346b40a43ed1e7386f68c8bf02d35b3c8a65d9 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 16:21:21 -0800 Subject: [PATCH 15/41] Introduce Pheon GDSII parsing and layout review tools Implement native GDSII reading and writing with Arrow interchange, hierarchy scans and selective cell loading. Validate numeric packing and array pitches, and preserve normalized REAL8 encodings exactly within the supported range. Add standalone dump, structural diff, hierarchy, layer-name annotation and Git diff commands. Provide scoped text and JSON reports, deterministic comparison, and record-preserving metadata edits. Package the shared library and optional Python Arrow readers as pheon using CFFI ABI mode and Python-independent platform wheels. Include locked source builds, native API checks, cross-platform artifact workflows, documentation and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer. Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14, and 71 installed companion integration tests pass. Full, scanned and selected Arrow layouts match the previous schemas. --- .github/workflows/ci.yml | 23 + .github/workflows/packages.yml | 123 +++ .gitignore | 6 +- ARROW_SCHEMA.md | 154 +++ Cargo.lock | 587 +++++++++++ Cargo.toml | 25 + LICENSE.md | 595 +++++++++++ MANIFEST.in | 7 + README.md | 337 ++++++ contrib/git/attributes | 5 + contrib/git/config | 10 + docs/diff-report-v1.schema.json | 206 ++++ docs/diff-reports.md | 189 ++++ docs/releases.md | 51 + pheon/__init__.py | 8 + pheon/_ffi.h | 53 + pheon/_native.py | 55 + pheon/arrow.py | 105 ++ pheon/basic.py | 72 ++ pheon/py.typed | 0 pyproject.toml | 43 + rust-toolchain.toml | 2 + setup.py | 76 ++ src/basic.rs | 353 +++++++ src/bin/pheon.rs | 379 +++++++ src/elements.rs | 1004 ++++++++++++++++++ src/lib.rs | 307 ++++++ src/library.rs | 588 +++++++++++ src/misc.py | 39 + src/record.rs | 430 ++++++++ src/records.rs | 567 ++++++++++ src/review.rs | 1492 +++++++++++++++++++++++++++ src/review/changes.rs | 127 +++ src/review/names.rs | 120 +++ src/review/report.rs | 407 ++++++++ src/review/selection.rs | 233 +++++ test.py | 8 + tests/common/mod.rs | 20 + tests/ffi_errors.rs | 157 +++ tests/golden/report-equal.json | 78 ++ tests/golden/report-failed.json | 64 ++ tests/golden/report-geometry.json | 360 +++++++ tests/golden/report-incomplete.json | 105 ++ tests/golden/report-metadata.json | 87 ++ tests/golden/review-canonical.txt | 9 + tests/golden/review-diff.txt | 7 + tests/golden/review-hierarchy.mmd | 10 + tests/golden/review-hierarchy.txt | 6 + tests/numeric_packing.rs | 103 ++ tests/python/test_arrow.py | 105 ++ tests/python/test_ffi.py | 70 ++ tests/python/test_loader.py | 36 + tests/python/test_packing.py | 66 ++ tests/real8.rs | 75 ++ tests/review.rs | 1248 ++++++++++++++++++++++ tests/review/reports.rs | 633 ++++++++++++ tests/selective_reads.rs | 768 ++++++++++++++ tests/writing.rs | 138 +++ tools/check_installed.py | 26 + 59 files changed, 12956 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/packages.yml create mode 100644 ARROW_SCHEMA.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 LICENSE.md create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 contrib/git/attributes create mode 100644 contrib/git/config create mode 100644 docs/diff-report-v1.schema.json create mode 100644 docs/diff-reports.md create mode 100644 docs/releases.md create mode 100644 pheon/__init__.py create mode 100644 pheon/_ffi.h create mode 100644 pheon/_native.py create mode 100644 pheon/arrow.py create mode 100644 pheon/basic.py create mode 100644 pheon/py.typed create mode 100644 pyproject.toml create mode 100644 rust-toolchain.toml create mode 100644 setup.py create mode 100644 src/basic.rs create mode 100644 src/bin/pheon.rs create mode 100644 src/elements.rs create mode 100644 src/lib.rs create mode 100644 src/library.rs create mode 100644 src/misc.py create mode 100644 src/record.rs create mode 100644 src/records.rs create mode 100644 src/review.rs create mode 100644 src/review/changes.rs create mode 100644 src/review/names.rs create mode 100644 src/review/report.rs create mode 100644 src/review/selection.rs create mode 100644 test.py create mode 100644 tests/common/mod.rs create mode 100644 tests/ffi_errors.rs create mode 100644 tests/golden/report-equal.json create mode 100644 tests/golden/report-failed.json create mode 100644 tests/golden/report-geometry.json create mode 100644 tests/golden/report-incomplete.json create mode 100644 tests/golden/report-metadata.json create mode 100644 tests/golden/review-canonical.txt create mode 100644 tests/golden/review-diff.txt create mode 100644 tests/golden/review-hierarchy.mmd create mode 100644 tests/golden/review-hierarchy.txt create mode 100644 tests/numeric_packing.rs create mode 100644 tests/python/test_arrow.py create mode 100644 tests/python/test_ffi.py create mode 100644 tests/python/test_loader.py create mode 100644 tests/python/test_packing.py create mode 100644 tests/real8.rs create mode 100644 tests/review.rs create mode 100644 tests/review/reports.rs create mode 100644 tests/selective_reads.rs create mode 100644 tests/writing.rs create mode 100644 tools/check_installed.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2d14a76 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: Native checks + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + rust: + runs-on: ubuntu-latest + strategy: + matrix: + rust: ['1.88.0', stable] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: rustup toolchain install ${{ matrix.rust }} --profile minimal + - run: cargo +${{ matrix.rust }} test --locked + - run: cargo +${{ matrix.rust }} test --locked --release diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..3a0c0bc --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,123 @@ +name: Package artifacts + +on: + pull_request: + workflow_dispatch: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - run: python -m pip install build + - run: python -m build --sdist + - uses: actions/upload-artifact@v4 + with: + name: source + path: dist/*.tar.gz + + wheels: + needs: sdist + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: source + path: source-dist + - name: Unpack the source release + shell: bash + run: | + mkdir source-build + tar -xzf source-dist/*.tar.gz -C source-build --strip-components=1 + - name: Build and repair Python-independent wheels from the sdist + uses: pypa/cibuildwheel@v4.3.5 + with: + package-dir: source-build + env: + CIBW_BUILD: cp311-* + CIBW_SKIP: '*-musllinux_* *-win32' + CIBW_ARCHS: native + CIBW_ENVIRONMENT_LINUX: 'CARGO_HOME=/tmp/pheon-cargo RUSTUP_HOME=/tmp/pheon-rustup PATH=/tmp/pheon-cargo/bin:$PATH' + CIBW_BEFORE_ALL_LINUX: 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable' + CIBW_TEST_EXTRAS: arrow + CIBW_TEST_COMMAND: 'python -I {project}/tools/check_installed.py' + - name: Check wheel ABI tags + shell: python + run: | + from pathlib import Path + wheels = list(Path('wheelhouse').glob('*.whl')) + assert wheels + assert all('-py3-none-' in p.name for p in wheels), wheels + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse/*.whl + + same-wheel: + needs: wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + python: ['3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse + - name: Install the same artifact on another Python version + shell: bash + run: python -m pip install wheelhouse/*.whl pyarrow + - run: python -I tools/check_installed.py + + cli: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: cargo test --release --locked + - run: cargo build --release --locked --bin pheon + - name: Assemble standalone CLI artifact + shell: bash + run: | + mkdir cli-dist + if [ "$RUNNER_OS" = Windows ]; then + cp target/release/pheon.exe cli-dist/ + else + cp target/release/pheon cli-dist/ + fi + cp LICENSE.md README.md cli-dist/ + cp -R contrib cli-dist/ + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.os }} + path: cli-dist/ diff --git a/.gitignore b/.gitignore index 5d80173..e67c112 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ /target/ **/*.rs.bk -Cargo.lock *.swp *.swo + +__pycache__/ +/build/ +/dist/ +/*.egg-info/ diff --git a/ARROW_SCHEMA.md b/ARROW_SCHEMA.md new file mode 100644 index 0000000..14aa848 --- /dev/null +++ b/ARROW_SCHEMA.md @@ -0,0 +1,154 @@ +# Arrow Schema Documentation + +Pheon's native `pheon_arrow_schema_version()` returns 1 for the full-read, +scan and selected-read layouts documented here. This version is independent +of the C API version, package release and CLI JSON report schema. The stable +Arrow C Data Interface transports these application-specific layouts; consumers +must check both the native API and application schema before reading data. + +This project uses Apache Arrow to represent GDSII (Graphic Data System II) library data. The schema is optimized for memory efficiency and fast access, employing several packed representations for common data types. + +## Top-Level Structure + +The top-level object is a `StructArray` representing the entire GDSII library. + +| Field | Type | Description | +|---|---|---| +| `meters_per_db_unit` | `Float64` | Database unit size in meters. | +| `user_units_per_db_unit` | `Float64` | Number of user units in one database unit. | +| `lib_name` | `Utf8` | The name of the GDSII library. | +| `mod_time` | `FixedSizeList(Int16, 6)` | Last modification time [year, month, day, hour, minute, second]. | +| `acc_time` | `FixedSizeList(Int16, 6)` | Last access time [year, month, day, hour, minute, second]. | +| `cells` | `List(Cell)` | A list of cells (structures) in the library. | +| `cell_names` | `List(Utf8)` | Names of all cells, indexed by their ID. | +| `layers` | `List(UInt32)` | List of packed layer/datatype pairs, indexed by their ID. | + +## Cell Structure + +Each cell in the `cells` list is a `StructArray`. + +| Field | Type | Description | +|---|---|---| +| `id` | `UInt32` | Unique identifier for the cell. | +| `file_offset` | `UInt64` | Byte offset immediately after the cell's `STRNAME` record (the first element, or `ENDSTR` for an empty cell). Identical for full and selected reads. | +| `srefs` | `List(SRef)` | Simple structure references (no properties). | +| `arefs` | `List(ARef)` | Array references (no properties). | +| `sref_props` | `List(SRefProp)` | Structure references with properties. | +| `aref_props` | `List(ARefProp)` | Array references with properties. | +| `rect_batches` | `List(RectBatch)` | Batches of rectangular boundaries (optimized). | +| `boundary_batches` | `List(BoundaryBatch)` | Batches of generic boundaries. | +| `boundary_props` | `List(BoundaryProp)` | Individual boundaries with properties. | +| `paths` | `List(Path)` | Path elements. | +| `nodes` | `List(BoxNode)` | Node elements. | +| `boxes` | `List(BoxNode)` | Box elements. | +| `texts` | `List(Text)` | Text labels. | + +## Hierarchy Scan Schema + +`scan_library` and `scan_bytes` return the same library metadata and `cell_names` +as a full read, with the following `cells` schema and no `layers` field: + +| Field | Type | Description | +|---|---|---| +| `id` | `UInt32` | Cell ID indexing this result's `cell_names`. | +| `struct_start_offset` | `UInt64` | Inclusive byte offset of the `BGNSTR` record. | +| `struct_end_offset` | `UInt64` | Exclusive byte offset immediately after `ENDSTR`. | +| `refs` | `List(ScanRef)` | SREFs and AREFs, including references with properties, sorted by target ID. | + +Each scan reference has `target`, `invert_y`, `scale`, `angle_rad`, `xy`, `xy0`, +`xy1`, and `counts`, using the ARef types and packed encodings described below. +SREFs use zero basis vectors and counts `(1, 1)`. Properties are omitted from the +scan output; geometry and text are skipped without decoding. + +Pass the half-open scan ranges to `read_selected_library` or `read_cells_bytes` +to obtain the full library schema for just those cells. Each range must contain +exactly one complete cell, from `BGNSTR` through `ENDSTR`. Cells appear in the +requested order. Cell and layer IDs are assigned within each result, so use +`cell_names` and `layers` from that result when resolving them; IDs can differ +from a full read or a scan. An empty selection returns metadata and empty lists. +The Rust random-access reader returns the original input slice as its remainder. + +## Element Types + +### Common Packed Encodings + +- **Layer ID (`UInt32`)**: A packed representation of `(layer, datatype)`. + - `layer = (layer_id >> 16) as i16` + - `datatype = (layer_id & 0xFFFF) as i16` +- **XY Coordinate (`UInt64`)**: A packed representation of a single point `(x, y)`. + - `x = (xy >> 32) as i32` + - `y = (xy & 0xFFFFFFFF) as i32` +- **Array Counts (`UInt32`)**: A packed representation of `(columns, rows)`. + - `columns = (counts >> 16) as i16` + - `rows = (counts & 0xFFFF) as i16` + +### Reference Elements (SRef / ARef) + +References point to other cells in the library. + +**SRef (Simple Reference)**: +- `target`: `UInt32` (Cell ID) +- `invert_y`: `Boolean` (Vertical flip) +- `scale`: `Float64` (Magnification) +- `angle_rad`: `Float64` (Rotation in radians) +- `xy`: `UInt64` (Packed origin point) + +**ARef (Array Reference)**: +- Same as SRef, plus: +- `xy0`: `UInt64` (Packed column basis vector increment) +- `xy1`: `UInt64` (Packed row basis vector increment) +- `counts`: `UInt32` (Packed columns and rows) + +Array increments must be exact signed 32-bit integer database-unit values. Full +reads, hierarchy scans, selected reads, and their FFI entry points reject arrays +whose endpoint/count division is fractional or outside that range. They never +round or truncate placements. The review CLI retains original endpoints and can +compare fractional arrays independently of this Arrow representation. + +**SRefProp / ARefProp**: +- Same as above, plus a `properties` field. + +### Boundary Elements + +Boundaries are grouped into batches for efficiency when they share properties or are simple rectangles. + +**RectBatch**: +- `layer`: `UInt32` (Packed Layer ID) +- `rects`: `List(Int32)` (Flattened list of `[min_x, min_y, max_x, max_y]` for each rectangle) + +**BoundaryBatch**: +- `layer`: `UInt32` (Packed Layer ID) +- `vertices`: `List(Int32)` (Flattened list of all vertices for all boundaries in the batch) +- `vertex_offsets`: `List(UInt32)` (Start index for each boundary in the `vertices` list) + +**BoundaryProp**: +- Individual boundary with `properties`. + +### Other Elements + +**Path**: +- `layer`: `UInt32` +- `path_type`: `Int16` (Optional) +- `extension_start`: `Int32` (Optional) +- `extension_end`: `Int32` (Optional) +- `width`: `Int32` (Optional; null when `WIDTH` is absent) +- `xy`: `List(Int32)` (centerline coordinates) +- `properties`: `List(Property)` + +**Text**: +- `layer`: `UInt32` +- `presentation_horiz`: `UInt8` +- `presentation_vert`: `UInt8` +- `presentation_font`: `UInt8` +- `path_type`: `Int16` +- `width`: `Int32` +- `invert_y`: `Boolean` +- `mag`: `Float64` +- `angle_deg`: `Float64` +- `xy`: `UInt64` (Packed position) +- `string`: `Utf8` +- `properties`: `List(Property)` + +**Property**: +- `key`: `Int16` +- `value`: `Utf8` diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..3a525b0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,587 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pheon" +version = "0.3.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..49e7ad5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "pheon" +version = "0.3.0" +authors = ["jan "] +edition = "2024" +rust-version = "1.88" +description = "GDSII parsing, Arrow interchange, and structural layout review" +license = "GPL-3.0-only" +readme = "README.md" +repository = "https://mpxd.net/code/jan/pheon" + + +[lib] +name = "pheon" +crate-type = ["cdylib", "rlib"] + + +[dependencies] +arrow-array = {version = "^54", features = ["ffi"]} +arrow-schema = "^54" +serde_json = "1" + + +[profile.release] +lto = true diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..175443c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,595 @@ +GNU General Public License +========================== + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +* **a)** The work must carry prominent notices stating that you modified it, and giving a +relevant date. +* **b)** The work must carry prominent notices stating that it is released under this +License and any conditions added under section 7. This requirement modifies the +requirement in section 4 to “keep intact all notices”. +* **c)** You must license the entire work, as a whole, under this License to anyone who +comes into possession of a copy. This License will therefore apply, along with any +applicable section 7 additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no permission to license the +work in any other way, but it does not invalidate such permission if you have +separately received it. +* **d)** If the work has interactive user interfaces, each must display Appropriate Legal +Notices; however, if the Program has interactive interfaces that do not display +Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +* **a)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by the Corresponding Source fixed on a +durable physical medium customarily used for software interchange. +* **b)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by a written offer, valid for at least +three years and valid for as long as you offer spare parts or customer support for +that product model, to give anyone who possesses the object code either **(1)** a copy of +the Corresponding Source for all the software in the product that is covered by this +License, on a durable physical medium customarily used for software interchange, for +a price no more than your reasonable cost of physically performing this conveying of +source, or **(2)** access to copy the Corresponding Source from a network server at no +charge. +* **c)** Convey individual copies of the object code with a copy of the written offer to +provide the Corresponding Source. This alternative is allowed only occasionally and +noncommercially, and only if you received the object code with such an offer, in +accord with subsection 6b. +* **d)** Convey the object code by offering access from a designated place (gratis or for +a charge), and offer equivalent access to the Corresponding Source in the same way +through the same place at no further charge. You need not require recipients to copy +the Corresponding Source along with the object code. If the place to copy the object +code is a network server, the Corresponding Source may be on a different server +(operated by you or a third party) that supports equivalent copying facilities, +provided you maintain clear directions next to the object code saying where to find +the Corresponding Source. Regardless of what server hosts the Corresponding Source, +you remain obligated to ensure that it is available for as long as needed to satisfy +these requirements. +* **e)** Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are being +offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +* **a)** Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or +* **b)** Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed by works +containing it; or +* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that +modified versions of such material be marked in reasonable ways as different from the +original version; or +* **d)** Limiting the use for publicity purposes of names of licensors or authors of the +material; or +* **e)** Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or +* **f)** Requiring indemnification of licensors and authors of that material by anyone +who conveys the material (or modified versions of it) with contractual assumptions of +liability to the recipient, for any liability that these contractual assumptions +directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a552f5f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include Cargo.toml Cargo.lock rust-toolchain.toml README.md ARROW_SCHEMA.md LICENSE.md +recursive-include src *.rs +recursive-include pheon *.py *.h py.typed +recursive-include tests *.rs *.py *.json *.txt *.mmd +recursive-include docs *.md *.json +recursive-include contrib * +recursive-include tools *.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..885fc01 --- /dev/null +++ b/README.md @@ -0,0 +1,337 @@ +# Pheon + +A fast GDSII parser and writer in Rust, with Apache Arrow integration for efficient data handling. + +Pheon provides native GDSII codecs and a standalone layout review CLI. The Rust +crate, Python package and executable are all named `pheon`. Python Klamath and +Masque remain separate consumers. The former `klamath_rs_ext` import and +`klamath` executable are not provided as aliases. + +## Overview + +`pheon` is designed to handle large GDSII files with high performance. It uses Apache Arrow as its primary in-memory data representation, allowing for efficient memory usage and easy integration with other tools in the Arrow ecosystem (like pandas, polars, or DuckDB via Python bindings). + +## Documentation + +- [Arrow Schema Documentation](ARROW_SCHEMA.md): Detailed information about the Arrow schema used for GDSII data. + +## Features + +- Fast parsing of GDSII files. +- Efficient memory representation using Apache Arrow. +- Support for common GDSII elements: SREF, AREF, Boundary, Path, Text, etc. +- Optimized "batch" representations for rectangles and boundaries sharing the same layer. +- C-compatible FFI for integration with other languages (e.g., Python). + +REAL8 writing uses normalized GDS encodings and represents every in-range `f64` +exactly. Finite magnitudes below `2^-260` underflow to canonical zero; nonfinite +values and magnitudes at or above `2^252` are rejected by the encoder. + +## Usage + +`read_path` exports a library through the Arrow C Data Interface. Allocate +initialized `ArrowArray` and `ArrowSchema` output structures, then check the +returned status: `0` means success and `-1` means failure. Only import the Arrow +outputs after success; the caller owns their release callbacks. Failed calls +leave the outputs unchanged. `pheon/_ffi.h` declares the C API; `test.py` +demonstrates the supported Python reader. + +`last_error_message()` returns a UTF-8, null-terminated error string, or null +after a successful parser call. Errors belong to the calling thread. Copy the +message before making another parser call on that thread, and before the thread +exits; calls on other threads do not invalidate it. Parser panics that unwind +are converted into errors. + +For partial reads, call `scan_bytes` with a borrowed GDSII byte buffer to obtain +cell ranges and references without decoding geometry. Pass the desired ranges +to `read_cells_bytes` to load their full contents. Both calls use the same Arrow +outputs and status/error handling as `read_path`: + +```c +int scan_bytes(const uint8_t *data, size_t size, + struct ArrowArray *array, struct ArrowSchema *schema); +int read_cells_bytes(const uint8_t *data, size_t size, + const uint64_t *ranges, size_t range_count, + struct ArrowArray *array, struct ArrowSchema *schema); +``` + +`ranges` contains alternating inclusive start and exclusive end offsets, with +two native-endian `uint64_t` values per cell. Each range must cover one complete +cell. A null `ranges` pointer is accepted when `range_count` is zero. Likewise, +null `data` is accepted for zero bytes and produces an ordinary parse error for +the empty input. Nonempty buffers must be readable for the duration of the call; +range values must be aligned. Exported Arrow data owns its storage independently +of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan +schema, offset definitions, and ID assignment. + +## Python package and validation + +Python 3.11 or newer is required. Source builds require Rust 1.88 or newer; +installing a prebuilt wheel requires no Rust compiler. The shared library uses +a C ABI rather than Python's or Rust's native ABI, so one platform wheel serves +multiple Python minor versions. CFFI, NumPy and optional PyArrow must themselves +support the selected interpreter. A Rust application uses the crate normally +and compiles it with its own compatible toolchain. The checkout selects the +stable toolchain locally through `rust-toolchain.toml`, without changing your +global default. CI checks both Rust 1.88 and current stable. + +```sh +python3 -m pip install 'pheon[arrow]' +``` + +```python +from pheon import arrow + +library = arrow.read_path('chip.gds') # one-row StructArray; also accepts gzip +scan = arrow.scan_bytes(gds_bytes) +cells = scan.field('cells')[0].values +ranges = list(zip(cells.field('struct_start_offset').to_pylist(), + cells.field('struct_end_offset').to_pylist())) +selected = arrow.read_cells_bytes(gds_bytes, ranges[:2]) +``` + +`read_bytes` reads a complete borrowed buffer. Full, scanned and selected Arrow +results own their storage independently of that buffer. `PheonError` reports +native parse failures; malformed range shapes and types raise Python value/type +errors before calling native code. Integer packing needs no PyArrow import. + +### Discovery and companion packages + +Klamath's `native` extra installs `pheon`; Masque's `arrow` extra installs +`pheon[arrow]`. Klamath imports Pheon's packing helpers, while Masque calls +`pheon.arrow` and translates parse failures to its existing error type. + +The Python loader selects the library bundled with the imported `pheon` package, +or the exact file selected by `PHEON_LIBRARY`. It checks C ABI version 1 and +Arrow layout schema version 1 before exposing native functions. Incompatible +or unversioned libraries fail with an import error. Those versions are independent +of package releases and the CLI's JSON report schema. + +`cargo install --path . --locked --bin pheon` installs the CLI, not the Python +package or a discoverable shared library. Klamath does not search PATH or Cargo +directories for it. To reuse an external native build, install the Python +components too and set `PHEON_LIBRARY`; an editable source install is useful +during development. The old library override variables are not recognized. + +The Python packing helpers require NumPy and CFFI, both declared as runtime +dependencies. Setuptools invokes `cargo build --release --lib` and bundles the +C ABI shared library in a platform-specific wheel. Its version comes from +`Cargo.toml`. The C declarations are in `pheon/_ffi.h`; the loader uses +CFFI ABI mode, so no Python extension compilation or Maturin is required. + +```sh +python3 -m pip install . +``` + +To build a source archive and a wheel with the `build` frontend and installed +Setuptools/Wheel tooling: + +```sh +python3 -m build --no-isolation +``` + +Source archives include a Cargo lockfile, and wheel builds use `--locked` so +building from an archive retains the resolved Rust dependency versions. Wheels +are tagged for the build platform and use the Python-independent `py3-none` ABI. +The CLI continues to build directly with Cargo and has no Python dependency. + +For source-tree development, point the CFFI loader at a Cargo-built library: + +```sh +cargo build --release --lib +PHEON_LIBRARY="$PWD/target/release/libpheon.so" \ + python3 -m unittest discover -s tests/python -v +``` + +Use `libpheon.dylib` on macOS or `pheon.dll` on Windows. Without +the explicit override, the loader uses the library bundled inside the package. +After installing a wheel, run its tests outside the source checkout to exercise +the installed package: + +```sh +python3 -I -m unittest discover -s /absolute/path/to/pheon/tests/python -v +``` + +The Python tests cover multidimensional and noncontiguous packing, input +preservation, range errors, and the C declarations for Arrow exports and release +callbacks, input lifetimes and ABI mismatches. Run `cargo test` and +`cargo test --release` for native validation. See [release validation](docs/releases.md) +for wheel builds, platform repair and testing the same artifact on multiple +Python versions. + +## Layout review CLI + +The review CLI is a standalone Rust executable: + +```sh +cargo build --bin pheon +target/debug/pheon dump chip.gds +target/debug/pheon dump chip.gds --records +target/debug/pheon diff old.gds new.gds --hierarchy ascii +target/debug/pheon diff old.gds new.gds --hierarchy mermaid --max-changes 20 +target/debug/pheon hierarchy chip.gds --format mermaid --depth 4 +target/debug/pheon diff old.gds new.gds --top CHIP --layer 1/0 +target/debug/pheon diff old.gds new.gds --show-layer 1/0 +target/debug/pheon diff old.gds new.gds --format json +target/debug/pheon diff old.gds new.gds --format json --hierarchy graph +target/debug/pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 --name 1:0:WG +``` + +The object dump is deterministic and omits source offsets. Record dumps preserve +record order and show offsets, tags, sizes, and values, including records the +object reviewer does not support. They can inspect an invalid metadata declaration +without invoking object-level validation. Zero padding after ENDLIB is accepted. + +The fixed `klamath-object-v2` comparison profile: + +- Matches cells by name and layers by numeric pair. The default scope is the + entire library; explicit selectors are recorded in each report. +- Ignores library/cell timestamps, cell order, element order, and property order. +- Normalizes boundary/box starting vertex and winding, retaining other vertices. +- Preserves duplicate objects and duplicate properties as multisets. +- Compares paths, labels, references, array endpoints/counts, properties, and all + retained flags exactly. Explicit defaults and their omission remain distinct; + SREFs and single-element AREFs remain distinct. REAL8 encodings are retained. +- Reports library fields, including UNITS, separately. It does not rescale files + into a shared grid, infer cell renames, or perform geometric Boolean operations. +- Uses bounding boxes as coordinate envelopes, not evidence of equality. Path + envelopes do not include stroke width; text/reference envelopes are coordinates, + not rendered or recursively transformed bounds. +- Returns an incomplete result for unsupported records, future metadata versions, + unresolved references, or cycles. These cannot produce an equality claim. + +`diff` exits with **0** for equality under this profile, **1** for differences, +and **2** for invalid input or incomplete comparison. `--max-changes` limits the +number of object details displayed, not the comparison. Source offsets are +included in changed-object reports for tracing changes back to record dumps. + +`diff --format json` emits a versioned report with typed before/after evidence, +structured diagnostics, and independent comparison status and completeness. +JSON includes all changes by default. Explicit display filters and `--max-changes` +record omitted evidence without changing totals or exit status. `--hierarchy graph` +adds the complete scoped reference graph, with symbolic array multiplicities; +`--depth` applies to text output only. Timestamp-only changes still return 0. +Input and argument errors also produce JSON when requested. See the +[report contract](docs/diff-reports.md) and +[version 1 JSON Schema](docs/diff-report-v1.schema.json). + +Unmatched objects are paired for explanation only when candidates are unique by +type/layer/bounding box, then by type/layer among the remaining objects. +References additionally match by target. Ambiguous groups stay additions and +removals. Reports explain reference movement, array-count changes, path widths, +and polygons changed within the same bounds. Full before/after records remain +visible, and duplicate counts remain significant. Unchanged layer names appear +once as a legend for changed objects on those layers. + +### Comparison scope and display filters + +`--top NAME` compares that cell and its reachable subtree independently in each +revision. `--cell NAME` compares the exact cell, including its reference records, +without comparing descendant contents. Combine the two to select a cell within +a subtree. A selector absent from both revisions is an error; absence on one +side is an addition/deletion. + +`--layer L/T` selects objects on a numeric layer/type pair, plus references whose +targets use that layer transitively. Cells with no relevant content are omitted. +Unresolved references, cycles, and unsupported records with unknown layer effect +remain incomplete results. Unsupported objects on another known layer can be +excluded. Library fields, including units, remain compared. Unsupported layer-name +version diagnostics are always retained, since those declarations can affect any +layer or cell in the library. Scoped dumps and +hierarchy views accept the same selectors. Scope selection parses the complete +input: malformed record grammar and conflicting metadata are still errors. + +`--show-cell NAME` and `--show-layer L/T` filter object details in a diff report +without changing comparison status, totals, metadata changes, or hierarchy. +References have no numeric layer and are hidden by `--show-layer`. The comparison +scope and any display-only filter are printed explicitly. The Rust API accepts +one combined `Scope`; applying another scope to an already scoped document is +rejected rather than losing reference coverage information. + +Hierarchy output distinguishes directly changed cells, affected ancestors, and +unchanged targets of modified references. It includes edges from both revisions, +so removed branches remain visible. Counts are direct edge multiplicities, +including arrays, rather than expanded placements. Shared definitions are marked +as already shown in text and appear once in Mermaid. `--depth` limits displayed +descendants only. The CLI emits Mermaid source, requiring no diagram renderer. + +Layer names are read from labels in an existing cell. A label with the exact +string `KLAMATH_LAYER_NAMES_V1` identifies an annotation carrier `(layer, texttype)` +pair within that cell. All other labels on that same pair in that same cell must +be `layer:datatype:name` declarations, for example `1:0:WG`. The carrier pair is +not fixed; choose a suitable annotation layer in the design's technology. +Declarations split at the first two colons, allowing colons in names. Numeric +fields are unsigned 16-bit decimal integers. Names are nonempty UTF-8 strings. +Repeated identical declarations are accepted; conflicting declarations fail. +Names are collected across the library. Recognized metadata labels are omitted +from object comparison; mappings are compared separately and printed once per +numeric pair, so renaming a layer does not rewrite every shape description. + +`name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ...` writes +this convention. It consolidates existing recognized declarations into one legend +in the chosen existing cell, retains existing names not explicitly updated, and +overrides the named pairs. The chosen carrier must contain no ordinary design +objects in that cell. Unrelated records, timestamps, unknown records and trailing +padding are copied byte-for-byte. No structures or references are added. Output +must be a new file: the CLI refuses to overwrite an existing destination, including +the input. Invalid declarations, conflicting arguments, oversized labels and +future metadata versions fail before output creation. + +The record-preserving review adapter retains information normalized or omitted +by the Arrow geometry reader; existing Arrow APIs are unchanged. Structured Rust +change records feed the text and hierarchy renderers. LayoutEditor's `LayerNames` +extension, layout extraction/remapping, and geometric Boolean comparison remain +outside the current CLI. + +### Git integration + +Install the executable on PATH with `cargo install --path . --bin pheon`. +The templates in [contrib/git/config](contrib/git/config) and +[contrib/git/attributes](contrib/git/attributes) provide two alternative drivers. +For structural summaries and affected hierarchy, add this attribute: + +```gitattributes +*.gds diff=pheon +``` + +Configure the native external driver in Git config: + +```gitconfig +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false +``` + +The adapter consumes Git's old/new temporary files directly, handles additions, +deletions and renames, and quotes display paths. It returns success after rendering +either equality or differences, as expected by the external-driver protocol. +Invalid or incomplete comparisons retain a nonzero exit and diagnostics. Scope +and display options can precede the final `--` in the configured command. Existing +repositories using `diff=layout` can put the same command under `[diff "layout"]`. + +For ordinary Git line diffs, use canonical textconv instead: + +```gitattributes +*.gds diff=pheon-text +``` + +Configure the driver in Git config: + +```gitconfig +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true +``` + +This is a human review representation, not an applicable GDS patch. Invalidate +Git's textconv cache when changing the dump profile/tool version. For automated +equality checks, use `pheon diff` and its exit status directly. + +`tests/review.rs` contains ten initial fixture pairs, checked-in canonical/text/ +Mermaid goldens, metadata byte-preservation checks, scope and report regressions, +and actual Git integration tests. It also checks near-limit polygon records, +2048-cell hierarchies and arrays representing over a billion placements without +expanding them. Run `cargo test` and `cargo test --release` for debug and optimized +validation. No KLayout installation or Python package is required. diff --git a/contrib/git/attributes b/contrib/git/attributes new file mode 100644 index 0000000..ed817cc --- /dev/null +++ b/contrib/git/attributes @@ -0,0 +1,5 @@ +# Add these lines to the layout repository's .gitattributes. +*.gds diff=pheon + +# For canonical textconv instead, replace the line above with: +# *.gds diff=pheon-text diff --git a/contrib/git/config b/contrib/git/config new file mode 100644 index 0000000..ca84adf --- /dev/null +++ b/contrib/git/config @@ -0,0 +1,10 @@ +# Include this file from Git config after installing pheon on PATH. +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false + +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true diff --git a/docs/diff-report-v1.schema.json b/docs/diff-report-v1.schema.json new file mode 100644 index 0000000..fc85eaa --- /dev/null +++ b/docs/diff-report-v1.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Pheon diff report version 1", + "description": "Object-profile comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", + "type": "object", + "required": ["schema_version", "kind", "producer", "comparison_profile", "inputs", "scope", "coordinates", "status", "result", "exit_code", "summary", "report_selection", "layer_legend", "library_changes", "layer_name_changes", "cell_changes", "hierarchy", "diagnostics"], + "properties": { + "schema_version": {"const": 1}, + "kind": {"const": "diff"}, + "producer": { + "type": "object", "required": ["name", "version"], + "properties": {"name": {"const": "pheon"}, "version": {"type": "string"}} + }, + "comparison_profile": {"type": "string"}, + "inputs": { + "type": "object", "required": ["old", "new"], + "properties": {"old": {"$ref": "#/$defs/input"}, "new": {"$ref": "#/$defs/input"}} + }, + "scope": { + "type": "object", "required": ["top", "cell", "layer"], + "properties": {"top": {"type": ["string", "null"]}, "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}} + }, + "coordinates": { + "type": "object", "required": ["unit", "frame", "bounds"], + "properties": {"unit": {"const": "database_unit"}, "frame": {"const": "cell_local"}, "bounds": {"const": "coordinate_envelope"}} + }, + "status": {"enum": ["complete", "incomplete", "failed"]}, + "result": {"enum": ["equal", "different", "unknown"]}, + "exit_code": {"enum": [0, 1, 2]}, + "summary": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/summary"}]}, + "report_selection": {"$ref": "#/$defs/selection"}, + "layer_legend": { + "type": "array", "items": { + "type": "object", "required": ["layer", "name"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "name": {"type": "string"}} + } + }, + "library_changes": {"type": "array", "items": {"$ref": "#/$defs/libraryChange"}}, + "layer_name_changes": { + "type": "array", "items": { + "type": "object", "required": ["layer", "before", "after"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "before": {"type": ["string", "null"]}, "after": {"type": ["string", "null"]}} + } + }, + "cell_changes": {"type": "array", "items": {"$ref": "#/$defs/cellChange"}}, + "hierarchy": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/graph"}]}, + "diagnostics": {"type": "array", "items": {"$ref": "#/$defs/diagnostic"}} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "complete"}}}, + "then": { + "properties": {"result": {"enum": ["equal", "different"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "exact"}}}, "diagnostics": {"maxItems": 0}}, + "if": {"properties": {"result": {"const": "equal"}}}, + "then": {"properties": {"exit_code": {"const": 0}}}, + "else": {"properties": {"exit_code": {"const": 1}}} + }, + "else": {"properties": {"exit_code": {"const": 2}, "diagnostics": {"minItems": 1}}} + }, + { + "if": {"properties": {"status": {"const": "incomplete"}}}, + "then": {"properties": {"result": {"enum": ["different", "unknown"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "observed"}}}}} + }, + { + "if": {"properties": {"status": {"const": "failed"}}}, + "then": {"properties": {"result": {"const": "unknown"}, "summary": {"type": "null"}, "library_changes": {"maxItems": 0}, "layer_name_changes": {"maxItems": 0}, "cell_changes": {"maxItems": 0}, "hierarchy": {"type": "null"}}} + } + ], + "$defs": { + "decimal": {"type": "string", "pattern": "^(0|[1-9][0-9]*)$"}, + "hex": {"type": "string", "pattern": "^0x([0-9a-f]{2})*$"}, + "uint16": {"type": "integer", "minimum": 0, "maximum": 65535}, + "int32": {"type": "integer", "minimum": -2147483648, "maximum": 2147483647}, + "layer": { + "type": "object", "required": ["number", "purpose"], + "properties": {"number": {"$ref": "#/$defs/uint16"}, "purpose": {"$ref": "#/$defs/uint16"}} + }, + "nullableLayer": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/layer"}]}, + "point": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"$ref": "#/$defs/int32"}}, + "fields": {"type": "array", "items": {"$ref": "#/$defs/field"}}, + "field": { + "type": "object", "required": ["tag", "name", "encoding", "value", "payload_hex"], + "properties": { + "tag": {"$ref": "#/$defs/uint16"}, "name": {"type": "string"}, + "encoding": {"enum": ["xy", "uint16", "int16", "int32", "bits", "real8", "ascii", "none", "bytes"]}, + "payload_hex": {"$ref": "#/$defs/hex"}, + "value": {"type": ["array", "string", "null"]} + }, + "allOf": [ + {"if": {"properties": {"encoding": {"const": "xy"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/point"}}}}}, + {"if": {"properties": {"encoding": {"enum": ["uint16", "bits"]}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/uint16"}}}}}, + {"if": {"properties": {"encoding": {"const": "int16"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": "integer", "minimum": -32768, "maximum": 32767}}}}}, + {"if": {"properties": {"encoding": {"const": "int32"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/int32"}}}}}, + {"if": {"properties": {"encoding": {"const": "real8"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": ["number", "null"]}}}}}, + {"if": {"properties": {"encoding": {"const": "ascii"}}}, "then": {"properties": {"value": {"type": ["string", "null"]}}}}, + {"if": {"properties": {"encoding": {"enum": ["none", "bytes"]}}}, "then": {"properties": {"value": {"type": "null"}}}} + ] + }, + "input": { + "type": "object", "required": ["path", "path_encoded_hex", "present", "units"], + "properties": { + "path": {"type": ["string", "null"]}, + "path_encoded_hex": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/hex"}]}, + "present": {"type": ["boolean", "null"]}, + "units": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]} + } + }, + "summary": { + "type": "object", "required": ["counts", "changed_cells", "removed_objects", "added_objects", "layer_name_changes", "library_field_changes"], + "properties": { + "counts": {"enum": ["exact", "observed"]}, + "changed_cells": {"$ref": "#/$defs/decimal"}, "removed_objects": {"$ref": "#/$defs/decimal"}, "added_objects": {"$ref": "#/$defs/decimal"}, + "layer_name_changes": {"$ref": "#/$defs/decimal"}, "library_field_changes": {"$ref": "#/$defs/decimal"} + } + }, + "selection": { + "type": "object", "required": ["cell", "layer", "max_objects", "truncated", "filtered_cells", "filtered_objects", "limited_objects", "emitted_objects"], + "properties": { + "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}, + "max_objects": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "truncated": {"type": "boolean"}, + "filtered_cells": {"$ref": "#/$defs/decimal"}, "filtered_objects": {"$ref": "#/$defs/decimal"}, + "limited_objects": {"$ref": "#/$defs/decimal"}, "emitted_objects": {"$ref": "#/$defs/decimal"} + } + }, + "libraryChange": { + "oneOf": [ + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"const": "presence"}, "tag": {"type": "null"}, "before": {"type": "boolean"}, "after": {"type": "boolean"}} + }, + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "before": {"$ref": "#/$defs/fields"}, "after": {"$ref": "#/$defs/fields"}} + } + ] + }, + "source": { + "type": "object", "required": ["side", "byte_offset", "end_offset"], + "properties": {"side": {"enum": ["old", "new"]}, "byte_offset": {"$ref": "#/$defs/decimal"}, "end_offset": {"$ref": "#/$defs/decimal"}} + }, + "property": { + "type": "object", "required": ["attribute", "value", "payload_hex"], + "properties": {"attribute": {"$ref": "#/$defs/uint16"}, "value": {"type": ["string", "null"]}, "payload_hex": {"$ref": "#/$defs/hex"}} + }, + "object": { + "type": "object", "required": ["id", "kind", "tag", "source", "layer", "bounds", "target", "origin", "width", "counts", "fields", "properties"], + "properties": { + "id": {"type": "string"}, "kind": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "source": {"$ref": "#/$defs/source"}, + "layer": {"$ref": "#/$defs/nullableLayer"}, + "bounds": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 4, "maxItems": 4, "items": {"$ref": "#/$defs/int32"}}]}, + "target": {"type": ["string", "null"]}, "origin": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/point"}]}, + "width": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/int32"}]}, + "counts": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "integer", "minimum": 1, "maximum": 32767}}]}, + "fields": {"$ref": "#/$defs/fields"}, "properties": {"type": "array", "items": {"$ref": "#/$defs/property"}} + } + }, + "replacement": { + "type": "object", "required": ["removed", "added", "match_rule", "explanation", "field_changes", "properties_changed"], + "properties": { + "removed": {"type": "string"}, "added": {"type": "string"}, "match_rule": {"type": "string"}, "explanation": {"type": "string"}, "properties_changed": {"type": "boolean"}, + "field_changes": {"type": "array", "items": { + "type": "object", "required": ["tag", "before", "after"], + "properties": {"tag": {"$ref": "#/$defs/uint16"}, "before": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}, "after": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}} + }} + } + }, + "cellSide": { + "oneOf": [{"type": "null"}, { + "type": "object", "required": ["byte_offset", "attributes"], + "properties": {"byte_offset": {"$ref": "#/$defs/decimal"}, "attributes": {"$ref": "#/$defs/fields"}} + }] + }, + "cellChange": { + "type": "object", "required": ["name", "before", "after", "attributes_changed", "removed", "added", "replacements"], + "properties": { + "name": {"type": "string"}, "before": {"$ref": "#/$defs/cellSide"}, "after": {"$ref": "#/$defs/cellSide"}, "attributes_changed": {"type": "boolean"}, + "removed": {"type": "array", "items": {"$ref": "#/$defs/object"}}, "added": {"type": "array", "items": {"$ref": "#/$defs/object"}}, + "replacements": {"type": "array", "items": {"$ref": "#/$defs/replacement"}} + } + }, + "diagnostic": { + "type": "object", "required": ["code", "severity", "message", "side", "byte_offset", "cell", "layer", "target"], + "properties": { + "code": {"type": "string"}, "severity": {"const": "error"}, "message": {"type": "string"}, "side": {"enum": ["old", "new", null]}, + "byte_offset": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "cell": {"type": ["string", "null"]}, + "layer": {"$ref": "#/$defs/nullableLayer"}, "target": {"type": ["string", "null"]} + } + }, + "graph": { + "type": "object", "required": ["nodes", "edges"], + "properties": { + "nodes": {"type": "array", "items": { + "type": "object", "required": ["cell", "before", "after", "change"], + "properties": { + "cell": {"type": "string"}, "before": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, "after": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, + "change": {"enum": ["direct", "affected", "unchanged"]} + } + }}, + "edges": {"type": "array", "items": { + "type": "object", "required": ["parent", "target", "before_count", "after_count", "reference_changed"], + "properties": {"parent": {"type": "string"}, "target": {"type": "string"}, "before_count": {"$ref": "#/$defs/decimal"}, "after_count": {"$ref": "#/$defs/decimal"}, "reference_changed": {"type": "boolean"}} + }} + } + } + } +} diff --git a/docs/diff-reports.md b/docs/diff-reports.md new file mode 100644 index 0000000..24dcbf3 --- /dev/null +++ b/docs/diff-reports.md @@ -0,0 +1,189 @@ +# Machine-readable diff reports + +```sh +pheon diff before.gds after.gds --format json > report.json +pheon diff before.gds after.gds --format json --hierarchy graph +pheon diff before.gds after.gds --format json --top CHIP --layer 1/0 +``` + +The command writes one UTF-8 JSON document followed by a newline to stdout. +Comparison diagnostics belong to that document. Output write failures are +reported on stderr and return 2; interrupted output is not a valid report. +JSON output currently belongs to `diff`; `dump`, `hierarchy`, and the Git +external driver retain their existing text interfaces. No external process is +used to produce a report. + +The [version 1 JSON Schema](diff-report-v1.schema.json) defines the structure. +[Golden reports](../tests/golden/report-geometry.json) illustrate complete +evidence; other `report-*.json` goldens cover equality, metadata, incomplete +comparison, and failure. Their producer version is replaced with `test` so +package version changes do not alter the fixtures. + +## Versions and determinism + +Pheon emits schema version 1 with producer name `pheon`. This schema has not +had a public release. The comparison profile remains `klamath-object-v2`, and +the `KLAMATH_LAYER_NAMES_V1` GDS marker retains its existing meaning. + +`schema_version` describes the report structure. `comparison_profile` names the +equality rules, currently `klamath-object-v2`. `producer.version` identifies the +implementation. These versions are independent. Consumers interpreting equality +must understand the comparison profile as well as the schema. + +Compatible additions may introduce optional fields. Consumers should ignore +unfamiliar additional fields and handle unfamiliar diagnostic codes and matching +rules without treating them as equality. Removed fields or changed meanings +require a new schema version. The schema intentionally permits additional fields. + +For the same inputs, paths, options, and producer version, serialization is +deterministic. Cells, layer pairs, fields, properties, and graph edges are sorted. +Duplicate objects and properties remain separate occurrences. Reports contain +no generation timestamp or required content fingerprint. + +## Result and exit status + +| `status` | `result` | Exit code | Interpretation | +| --- | --- | --- | --- | +| `complete` | `equal` | 0 | Equal under the recorded profile and scope. | +| `complete` | `different` | 1 | Differences found. | +| `incomplete` | `different` | 2 | Definite differences found, with comparison limitations. | +| `incomplete` | `unknown` | 2 | Equality cannot be established. | +| `failed` | `unknown` | 2 | No comparison result is available. | + +Unsupported records, future metadata versions, unresolved references, and +hierarchy cycles make the comparison incomplete. Malformed input, read errors, +invalid scopes, and invalid command arguments produce failed reports. CI must +check completeness as well as differences. An exit code of 1 is a normal +comparison outcome, including when stdout is redirected to a report file. + +Library and cell timestamps are excluded from the comparison profile. A valid, +complete timestamp-only comparison returns 0. Raw record dumps remain available +for examining those timestamps. + +`summary` counts the entire scoped comparison, before display selection. Its +`counts` is `exact` for a complete comparison and `observed` for an incomplete +one. Failed reports have a null summary and empty change arrays. Replacement +suggestions and affected ancestors never add to change counts. + +## Scope, selection, and numbers + +`scope` records the actual `top`, `cell`, and numeric `layer` selectors. Null +means unrestricted. The complete file is parsed before applying scope, so an +invalid record outside the selected subtree still fails the command. + +`report_selection` records display-only `cell`, `layer`, and `max_objects`. +JSON has no default object limit. Explicit `--max-changes N` limits the total +number of emitted object occurrences across both sides. Removals precede +additions within each cell. `filtered_cells`, `filtered_objects`, +`limited_objects`, and `emitted_objects` explain evidence selection. `truncated` +is true if filtering or a limit omits any changed cell or object. Library and +layer-name changes, diagnostics, summaries, and the optional graph are retained. +A replacement is emitted only when both linked objects are included. + +All aggregate counts, limits, and byte offsets are unsigned decimal **strings**. +This avoids loss of precision in JSON consumers using binary64 numbers. +Coordinates, numeric tags, layer/type pairs, widths, and array dimensions are +bounded integers represented as JSON numbers. Null means absent or unavailable; +zero and empty arrays do not imply absence. + +`inputs.old` and `inputs.new` contain input paths, presence, and UNITS fields. +Presence is distinct from an empty library. The Rust API also supports absent +Git revisions using `Document::empty_like`; no units are invented for absent +inputs. Failed reports may have null presence and units. Argument failures can +lack input/selection context because the command could not be parsed. + +Paths are UTF-8 when representable. Otherwise `path` is null and +`path_encoded_hex` preserves Rust's platform-native encoded path bytes, prefixed +with `0x`. This is platform-dependent evidence rather than a portable path name. + +## Change evidence + +`library_changes` records changed header fields as typed before/after arrays. +Library presence changes use the special field `presence`, a null tag, and +boolean before/after values. Units are not rescaled to a common grid. + +`layer_name_changes` records numeric pairs with before/after names. A layer +pair is `{ "number": 1, "purpose": 0 }`, where purpose is DATATYPE, TEXTTYPE, +BOXTYPE, or NODETYPE according to the object kind. `layer_legend` contains +unchanged names available in the scope. Embedded annotation labels are excluded +from ordinary object changes. A metadata rename appears once. + +Each `cell_changes` entry has its name, nullable before/after cell records, +attribute evidence, removed and added objects, and replacement suggestions. +A null cell side means the cell is absent in that scope. An existing empty +cell has a byte offset and an empty attribute array. + +Each object retains: + +- Its kind and numeric tag, numeric layer pair, target cell, and coordinate envelope. +- Its original source side and byte range `[byte_offset, end_offset)`. +- Typed field records and the sorted property multiset, including duplicates. +- Convenience fields for text/reference origin, path/text width, and reference + array dimensions. SREF dimensions are `[1, 1]`; the kind and retained fields + distinguish an SREF from a one-element AREF and explicit fields from omission. + +Coordinates are cell-local integer database units. `bounds` contains +`[xmin, ymin, xmax, ymax]` of stored coordinates. It excludes path stroke width, +rendered text dimensions, and recursively transformed target geometry. +Equal envelopes do not establish geometric equality. Unit changes must be +considered before deriving physical movements or dimensions. + +A retained field has a numeric `tag`, convenience `name`, `encoding`, decoded +`value`, and `payload_hex`. The numeric tag is authoritative; names are display +aids. XY values are arrays of `[x, y]` pairs. Other numeric fields are arrays +of their declared type. Unsupported encodings retain bytes with a null value. +String and property values are null if their bytes are not valid UTF-8; +`payload_hex` always preserves them without replacement characters. + +Payloads follow the comparison profile: boundary/box rings use canonical start +and winding, string padding is removed, and fields/properties are sorted. They +are comparison evidence, not a verbatim serialization of the source records. +REAL8 payload encodings are retained exactly alongside their decoded values. +Explicit defaults and omitted fields remain distinct. Source ranges allow raw +record inspection when original encoding or order is needed. + +Replacement suggestions link report-local removed/added IDs, with a `match_rule`, +typed `field_changes`, and `properties_changed`. Their explanation is optional +for consumers to display and must not be parsed. Current matching rules are +`unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. +These are explanatory matches, not persistent identity or shape equivalence. +Ambiguous candidates stay independent removals and additions. IDs are unique +within a report and have no meaning across reports. + +## Diagnostics and hierarchy + +Diagnostics have `code`, `severity`, `message`, input `side`, original byte +offset, cell, layer, and target context. Unknown context is null. Messages are +for display; codes and context support automation. Current limitation codes are +`unsupported_record`, `unsupported_metadata_version`, `unresolved_reference`, +`hierarchy_cycle`, and `scope_mismatch`. Failure codes are `invalid_input`, +`io_error`, `invalid_scope`, and `command_error`. Fatal parser errors currently +retain their location in the human message; their structured byte offset is +null. Consumers must not depend on message wording. + +`--hierarchy graph` includes the full scoped union graph of both revisions. +Otherwise `hierarchy` is null. Nodes are keyed by cell name, with per-side +states `included`, `outside_scope`, `unresolved`, or `absent`. Their `change` +is `direct`, `affected` through a descendant, or `unchanged`. + +Edges record parent, target, before/after direct placement counts, and whether +the underlying reference records changed. AREF columns times rows contribute +symbolically; arrays are never expanded. Counts are direct edge multiplicities, +not recursively accumulated placements. Shared cells occur once. Removed edges +remain visible, and ancestors are computed over the union of both revisions. +Graph content is independent of display filters and object limits. Text depth +limits and ASCII/Mermaid choices do not apply to JSON graphs. + +## Development + +`cargo test --test review` checks report semantics, CLI behavior, and versioned +goldens. To intentionally regenerate report goldens after reviewing a contract +change, run: + +```sh +PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens +``` + +Schema validators can validate the goldens or actual CLI output against +`diff-report-v1.schema.json`. Schema validation does not replace semantic checks +such as replacement links, summary totals, or status/exit-code agreement. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..5836f98 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,51 @@ +# Building and validating a release + +The source archive, Python wheels and standalone CLI are separate artifacts. +Python wheels contain `pheon` and its shared library; Cargo installation contains +the CLI. Neither installation searches for the other. + +Build the source archive and then the wheel from that archive: + +```sh +python -m pip install build +python -m build +``` + +The default `build` invocation builds its wheel from the source archive. Both +carry the Cargo version, license and locked dependency resolution. A source build +requires Rust 1.88 or newer. The checkout selects current stable through +`rust-toolchain.toml`; `RUSTUP_TOOLCHAIN=1.88.0` selects the minimum compiler +explicitly for source-package builds. An installed wheel has no Rust runtime +requirement. + +The package workflow builds the archive once, then uses cibuildwheel to build +from it and repair wheels for Linux x86-64/ARM64, macOS x86-64/ARM64 and Windows +x86-64. Repair uses the platform tools supplied by cibuildwheel. Wheels must keep +the `py3-none` interpreter/ABI tags and a platform tag, including an appropriate +manylinux tag for Linux releases. Local unrepaired `linux_*` wheels are development +artifacts, not portable Linux releases. + +Each repaired wheel is tested on Python 3.11 through 3.14 without rebuilding it. +Tests run in isolated mode against the installed package and bundled library, +including an import that verifies integer packing does not import PyArrow. To +repeat that check locally after installing a wheel and PyArrow: + +```sh +python -I /path/to/pheon/tools/check_installed.py +``` + +Unset `PHEON_LIBRARY` for artifact tests. It exists to select an explicit external +library during development, and version checks apply equally to that library. +The minimum Rust compiler and current stable compiler run native tests in both +debug and release modes. CLI artifacts contain the executable, license, README +and Git integration templates. They are tied to their build OS and architecture. + +Workflows upload build artifacts only. They do not publish to PyPI, crates.io, +or a hosted release. The repository URL and checkout directory can be renamed +to `pheon` separately; no remote repository settings are changed by this work. + +The project rename provides no Python import shim or old CLI alias. Companion +releases must migrate their imports and dependencies together. The Python C API +version and Arrow layout version are currently 1. CLI reports use schema version +1 and retain the `klamath-object-v2` comparison profile and existing GDS layer-name +metadata marker. diff --git a/pheon/__init__.py b/pheon/__init__.py new file mode 100644 index 0000000..09bb1d6 --- /dev/null +++ b/pheon/__init__.py @@ -0,0 +1,8 @@ +"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers.""" + +from ._native import PheonError as PheonError +from .basic import pack_int2 as pack_int2 +from .basic import pack_int4 as pack_int4 + + +__version__ = "0.3.0" diff --git a/pheon/_ffi.h b/pheon/_ffi.h new file mode 100644 index 0000000..6bca31f --- /dev/null +++ b/pheon/_ffi.h @@ -0,0 +1,53 @@ +#ifndef PHEON_H +#define PHEON_H +#include +#include + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +uint32_t pheon_abi_version(void); +uint32_t pheon_arrow_schema_version(void); +int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema); +const char *last_error_message(void); + +double f64_to_i16(double *data, size_t size); +double f64_to_i32(double *data, size_t size); +float f32_to_i16(float *data, size_t size); +float f32_to_i32(float *data, size_t size); +int64_t i64_to_i16(int64_t *data, size_t size); +int64_t i64_to_i32(int64_t *data, size_t size); +int32_t i32_to_i16(int32_t *data, size_t size); +int32_t i32_to_i32(int32_t *data, size_t size); +int16_t i16_to_i16(int16_t *data, size_t size); +uint64_t u64_to_i16(uint64_t *data, size_t size); +uint64_t u64_to_i32(uint64_t *data, size_t size); +uint32_t u32_to_i16(uint32_t *data, size_t size); +uint32_t u32_to_i32(uint32_t *data, size_t size); +uint16_t u16_to_i16(uint16_t *data, size_t size); +#endif diff --git a/pheon/_native.py b/pheon/_native.py new file mode 100644 index 0000000..fc9dbd7 --- /dev/null +++ b/pheon/_native.py @@ -0,0 +1,55 @@ +"""Load a bundled or explicitly selected library through its stable C ABI.""" + +import os +from pathlib import Path +import sys + +from cffi import FFI + + +ffi = FFI() +header = Path(__file__).with_name("_ffi.h").read_text() +ffi.cdef( + "\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")) +) + +if sys.platform == "win32": + filename = "pheon.dll" +elif sys.platform == "darwin": + filename = "libpheon.dylib" +else: + filename = "libpheon.so" + +# An explicit override supports source-tree development without copying binaries +# into the package. Installed wheels load only their bundled native library. +library = ( + Path(os.environ["PHEON_LIBRARY"]).expanduser() + if "PHEON_LIBRARY" in os.environ + else Path(__file__).with_name(filename) +) +try: + lib = ffi.dlopen(str(library.resolve())) +except OSError as error: + raise ImportError( + f"Could not load Pheon library {library}: {error}. " + "Install a Pheon wheel or set PHEON_LIBRARY to a Cargo-built shared library." + ) from error + +ABI_VERSION = 1 +ARROW_SCHEMA_VERSION = 1 +try: + native_abi = lib.pheon_abi_version() + native_schema = lib.pheon_arrow_schema_version() +except AttributeError as error: + raise ImportError( + "The selected library does not expose the Pheon version API" + ) from error +if (native_abi, native_schema) != (ABI_VERSION, ARROW_SCHEMA_VERSION): + raise ImportError( + f"Incompatible Pheon library: C ABI {native_abi}, Arrow schema {native_schema}; " + f"expected C ABI {ABI_VERSION}, Arrow schema {ARROW_SCHEMA_VERSION}" + ) + + +class PheonError(ValueError): + """A native GDSII parse or selection error.""" diff --git a/pheon/arrow.py b/pheon/arrow.py new file mode 100644 index 0000000..5d90631 --- /dev/null +++ b/pheon/arrow.py @@ -0,0 +1,105 @@ +"""GDSII readers returning a one-row Arrow StructArray for the library. + +The returned arrays own their buffers independently of the input. This module +requires the optional ``arrow`` extra; integer packing does not import PyArrow. +""" + +from __future__ import annotations + +from collections.abc import Sequence +import gzip +import os +from typing import Any, Callable + +import numpy +import pyarrow + +from ._native import PheonError, ffi, lib + + +def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray: + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + if call(array, schema) != 0: + error = lib.last_error_message() + message = ( + ffi.string(error).decode("utf-8", errors="replace") + if error != ffi.NULL + else "GDSII read failed" + ) + raise PheonError(message) + # PyArrow consumes the structures, setting their release callbacks to + # null. The finally block also covers failure during import. + return pyarrow.Array._import_from_c( + int(ffi.cast("uintptr_t", array)), int(ffi.cast("uintptr_t", schema)) + ) + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + +def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: + """Read a GDSII file, recognizing gzip by its magic bytes.""" + filename = os.path.abspath(os.path.expanduser(os.fspath(path))) + with open(filename, "rb") as stream: + compressed = stream.read(2) == b"\x1f\x8b" + if compressed: + stream.seek(0) + with gzip.GzipFile(fileobj=stream) as source: + return read_bytes(source.read()) + # The native Windows path API accepts UTF-8; Unix accepts filesystem bytes. + encoded = filename.encode("utf-8") if os.name == "nt" else os.fsencode(filename) + return _export(lambda array, schema: lib.read_path(encoded, array, schema)) + + +def _read_buffer( + buffer: Any, call: Callable[[Any, int, Any, Any], int] +) -> pyarrow.StructArray: + with memoryview(buffer) as view: + if not view.c_contiguous: + raise ValueError("GDSII input must be a contiguous buffer") + with ffi.from_buffer("const uint8_t[]", view) as data: + return _export(lambda array, schema: call(data, view.nbytes, array, schema)) + + +def read_bytes(buffer: Any) -> pyarrow.StructArray: + """Read a complete GDSII library from a contiguous borrowed byte buffer.""" + return _read_buffer(buffer, lib.read_bytes) + + +def scan_bytes(buffer: Any) -> pyarrow.StructArray: + """Scan metadata, cell byte ranges and references without decoding geometry.""" + return _read_buffer(buffer, lib.scan_bytes) + + +def read_cells_bytes( + buffer: Any, ranges: Sequence[Sequence[int]] | numpy.ndarray +) -> pyarrow.StructArray: + """Read complete cells in the requested order from half-open byte ranges. + + ``ranges`` has shape (N, 2) and unsigned 64-bit integer offsets. Signed + integer inputs are accepted after checking that every offset is nonnegative. + An empty sequence requests metadata with no cells. + """ + values = numpy.asarray(ranges) + if values.shape == (0,): + values = numpy.empty((0, 2), dtype=numpy.uint64) + if values.ndim != 2 or values.shape[1] != 2: + raise ValueError("Cell ranges must have shape (N, 2)") + if values.dtype.kind not in "iu": + raise TypeError("Cell offsets must be integers") + if values.dtype.kind == "i" and numpy.any(values < 0): + raise ValueError("Cell offsets must be nonnegative") + values = numpy.require( + values, dtype=numpy.uint64, requirements=("C_CONTIGUOUS", "ALIGNED") + ) + with ffi.from_buffer("const uint64_t[]", values) as offsets: + return _read_buffer( + buffer, + lambda data, size, array, schema: lib.read_cells_bytes( + data, size, offsets, len(values), array, schema + ), + ) diff --git a/pheon/basic.py b/pheon/basic.py new file mode 100644 index 0000000..9ca4745 --- /dev/null +++ b/pheon/basic.py @@ -0,0 +1,72 @@ +from collections.abc import Sequence +from typing import Any + +import numpy +from numpy.typing import NDArray + +from ._native import lib, ffi + + +CONV_TABLE_i16 = { + numpy.float64: lib.f64_to_i16, + numpy.float32: lib.f32_to_i16, + numpy.int64: lib.i64_to_i16, + numpy.int32: lib.i32_to_i16, + numpy.int16: lib.i16_to_i16, + numpy.uint64: lib.u64_to_i16, + numpy.uint32: lib.u32_to_i16, + numpy.uint16: lib.u16_to_i16, +} + +CONV_TABLE_i32 = { + numpy.float64: lib.f64_to_i32, + numpy.float32: lib.f32_to_i32, + numpy.int64: lib.i64_to_i32, + numpy.int32: lib.i32_to_i32, + numpy.uint64: lib.u64_to_i32, + numpy.uint32: lib.u32_to_i32, +} + + +def _pack_int( + data: NDArray[numpy.integer] | Sequence[int] | int, + width: int, + converters: dict[type[numpy.generic], Any], +) -> bytes: + arr = numpy.atleast_1d(data) + target_dtype = numpy.dtype(f">i{width}") + + for dtype, convert in converters.items(): + if arr.dtype != dtype: + continue + + arr = numpy.require( + arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") + ) + if arr is data: + arr = numpy.array(arr, copy=True) + + with ffi.from_buffer( + ffi.typeof(convert).args[0], arr, require_writable=True + ) as buf: + result = convert(buf, arr.size) + if result != 0: + raise ValueError(f"Invalid value for conversion to Int{width}: {result}") + + return arr.reshape(-1).view(target_dtype)[:: arr.itemsize // width].tobytes() + + if arr.dtype == target_dtype: + return arr.tobytes() + + limit = 1 << (width * 8 - 1) + if (arr > limit - 1).any() or (arr < -limit).any(): + raise Exception(f"int{width} data out of range: {arr}") + return arr.astype(target_dtype).tobytes() + + +def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 2, CONV_TABLE_i16) + + +def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 4, CONV_TABLE_i32) diff --git a/pheon/py.typed b/pheon/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..835880a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pheon" +description = "Native GDSII parsing, Arrow interchange, and structural layout review" +readme = "README.md" +license = "GPL-3.0-only" +license-files = ["LICENSE.md"] +authors = [ + { name="Jan Petykiewicz", email="jan@mpxd.net" }, + ] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", +# "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Manufacturing", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", + ] +requires-python = ">=3.11" +dynamic = ["version"] +dependencies = [ + "cffi", + "numpy", + ] + +[project.optional-dependencies] +arrow = ["pyarrow>=14"] + +[project.urls] +Homepage = "https://mpxd.net/code/jan/pheon" +Repository = "https://mpxd.net/code/jan/pheon" + +[tool.setuptools] +packages = ["pheon"] + +[tool.setuptools.package-data] +pheon = ["py.typed", "_ffi.h"] + diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..292fe49 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ca4fd70 --- /dev/null +++ b/setup.py @@ -0,0 +1,76 @@ +"""Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tomllib + +from setuptools import Distribution, setup +from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist +from wheel.bdist_wheel import bdist_wheel + + +ROOT = Path(__file__).resolve().parent +VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] + + +def ensure_lockfile(): + if not (ROOT / "Cargo.lock").is_file(): + subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True) + + +class CargoBuild(build_py): + def run(self): + ensure_lockfile() + super().run() + result = subprocess.run( + ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], + cwd=ROOT, capture_output=True, text=True, + ) + sys.stderr.write(result.stderr) + library = None + for line in result.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + print(line, file=sys.stderr) + continue + if message.get("reason") == "compiler-message": + sys.stderr.write(message["message"].get("rendered") or "") + if (message.get("reason") == "compiler-artifact" + and message["target"]["name"] == "pheon" + and "cdylib" in message["target"]["crate_types"]): + for filename in message["filenames"]: + if Path(filename).suffix in (".so", ".dylib", ".dll"): + library = Path(filename) + result.check_returncode() + if library is None: + raise RuntimeError("Cargo did not produce the pheon shared library") + shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) + + +class CargoSdist(sdist): + def run(self): + ensure_lockfile() + super().run() + + +class NativeDistribution(Distribution): + def has_ext_modules(self): + return True + + +class NativeWheel(bdist_wheel): + def get_tag(self): + # CFFI ABI mode needs the host platform, but no CPython extension ABI. + _, _, platform = super().get_tag() + return "py3", "none", platform + + +setup( + version=VERSION, + distclass=NativeDistribution, + cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, +) diff --git a/src/basic.rs b/src/basic.rs new file mode 100644 index 0000000..f089ff3 --- /dev/null +++ b/src/basic.rs @@ -0,0 +1,353 @@ +/// +/// Functionality for parsing and writing basic data types +/// +use std::io; +use std::fmt; + +pub type OResult = Result; + +pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>; + +#[derive(Debug)] +pub enum ErrType { + Incomplete(Option), + Failed(String), +} + +pub fn fail(input: &[u8], msg: String) -> IResult<'_, O> { + Err((input, ErrType::Failed(msg))) +} + +pub fn incomplete(input: &[u8], size: Option) -> IResult<'_, O> { + Err((input, ErrType::Incomplete(size))) +} + +pub fn take_bytes>(input: &[u8], count: CC) -> IResult<'_, &[u8]> { + let cc = count.into(); + if input.len() < cc { + incomplete(input, Some(cc)) + } else { + let (taken, input) = input.split_at(cc); + Ok((input, taken)) + } +} + + +/* + * Parse functions + */ +fn take_array(input: &[u8]) -> IResult<'_, [u8; N]> { + match input.split_first_chunk::() { + Some((bytes, remaining)) => Ok((remaining, *bytes)), + None => incomplete(input, Some(N)), + } +} + +pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> { + let (input, bytes) = take_array(input)?; + Ok((input, u16::from_be_bytes(bytes))) +} + +pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> { + let (input, bytes) = take_array(input)?; + Ok((input, i16::from_be_bytes(bytes))) +} + +pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> { + let (input, bytes) = take_array(input)?; + Ok((input, i32::from_be_bytes(bytes))) +} + +/// Convert GDS REAL8 to IEEE float64 +pub fn decode_real8(int: u64) -> f64 { + let neg = int & 0x8000_0000_0000_0000; + let exp = (int >> 56) & 0x7f; + let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64; + if neg != 0 { + mant *= -1.0 + } + let exp2 = 4 * (exp as i32 - 64) - 56; + mant * 2_f64.powi(exp2) +} + +pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { + let (input, bytes) = take_array(input)?; + let data = u64::from_be_bytes(bytes); + Ok((input, decode_real8(data))) +} + + +pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { + let mut buf = [0_i16; 6]; + let mut input = input; + for bb in &mut buf { + (input, *bb) = parse_int2(input)?; + } + buf[0] += 1900; // Year is from 1900 + Ok((input, buf)) +} + + +pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { + let mut bits = [false; 16]; + let (input, val) = parse_int2(input)?; + for (ii, bit) in bits.iter_mut().enumerate() { + *bit = ((val >> (16 - 1 - ii)) & 0x01) == 1; + } + Ok((input, bits)) +} + + +pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec> { + let length = length as usize; + let (input, data) = take_bytes(input, length)?; + let true_length = length - usize::from(data.last() == Some(&0)); + let vec = data[..true_length].to_vec(); + Ok((input, vec)) +} + + +/* + * Pack functions + */ + +pub fn bitarray2int(bits: &[bool; 16]) -> u16 { + let mut int: u16 = 0; + for (ii, bit) in bits.iter().enumerate() { + int |= (*bit as u16) << (16 - 1 - ii); + } + int +} + +pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { + buf[..2].copy_from_slice(&bitarray2int(bits).to_be_bytes()); +} + + +pub fn pack_int2(buf: &mut [u8], int: i16) { + buf[..2].copy_from_slice(&int.to_be_bytes()); +} + +pub fn pack_int4(buf: &mut [u8], int: i32) { + buf[..4].copy_from_slice(&int.to_be_bytes()); +} + +pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { + let bytes = encode_real8(fnum)?.to_be_bytes(); + buf[..8].copy_from_slice(&bytes); + Ok(()) +} + +pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { + let len = data.len(); + buf[..len].copy_from_slice(data); + if len % 2 == 1 { + buf[len] = 0; + len + 1 + } else { + len + } +} + + +pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) { + assert!(buf.len() >= 6 * 2); + let year = date[0] - 1900; + pack_int2(buf, year); + for ii in 1..6 { + pack_int2(&mut buf[(2 * ii)..], date[ii]); + } +} + + +#[derive(Debug, Clone)] +pub struct FloatTooBigError { + float_value: f64, +} +impl fmt::Display for FloatTooBigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Float {0} is too large for Real8", self.float_value) + } +} + +/// Convert float64 to a normalized GDS REAL8 representation. +/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0. +/// Nonfinite values and magnitudes at or above 2^252 return an error. +pub fn encode_real8(fnum: f64) -> Result { + let ieee = fnum.to_bits(); + let ieee_exp = ((ieee >> 52) & 0x7ff) as i32; + if ieee_exp == 0x7ff { + return Err(FloatTooBigError { float_value: fnum }); + } + + let exp2 = ieee_exp - 1023; + if exp2 < -260 { + // This also handles both signed zeros and every IEEE subnormal. + return Ok(0); + } + + let gds_exp = exp2.div_euclid(4) + 65; + if gds_exp > 0x7f { + return Err(FloatTooBigError { float_value: fnum }); + } + + // The 53-bit IEEE significand needs only 0..3 left shifts to become + // a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8. + let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52); + let mantissa = significand << exp2.rem_euclid(4); + let sign = ieee & 0x8000_0000_0000_0000; + Ok(sign | ((gds_exp as u64) << 56) | mantissa) +} + + +#[cfg(test)] +mod tests { + #[test] + fn test_parse_bitarray() { + use crate::basic::parse_bitarray; + + //assert!(parse_bitarray(b"59") == 13625); + assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); + assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); + + let arr_0001 = parse_bitarray(b"\x00\x01").unwrap().1; + for (ii, &vv) in arr_0001.iter().enumerate() { + assert_eq!(ii == 15, vv); + } + + let arr_8000 = parse_bitarray(b"\x80\x00").unwrap().1; + for (ii, &vv) in arr_8000.iter().enumerate() { + assert_eq!(ii == 0, vv); + } + } + + #[test] + fn test_parse_int2() { + use crate::basic::parse_int2; + assert_eq!(parse_int2(b"59").unwrap().1, 13625); + assert_eq!(parse_int2(b"\0\0").unwrap().1, 0); + assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1); + } + + #[test] + fn test_parse_int4() { + use crate::basic::parse_int4; + assert_eq!(parse_int4(b"4321").unwrap().1, 875770417); + } + + #[test] + fn test_decode_real8() { + use crate::basic::decode_real8; + + // zeroes + assert_eq!(decode_real8(0x0), 0.0); + assert_eq!(decode_real8(1<<63), 0.0); // negative + assert_eq!(decode_real8(0xff << 56), 0.0); // denormalized + + assert_eq!(decode_real8(0x4110 << 48), 1.0); + assert_eq!(decode_real8(0xC120 << 48), -2.0); + } + + #[test] + #[should_panic] + fn test_encode_real8_panic() { + use crate::basic::encode_real8; + encode_real8(1e80).unwrap(); + } + + + #[test] + fn test_parse_real8() { + use crate::basic:: parse_real8; + + assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1); + assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1); + assert_eq!(-2.0, parse_real8(&[0xC1, 0x20, 0, 0, 0, 0, 0, 0]).unwrap().1); + } + + #[test] + fn test_parse_ascii() { + use crate::basic::parse_ascii; + + assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345"); + assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte + assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456"); + } + + #[test] + fn test_pack_bitarray() { + use crate::basic::pack_bitarray; + let mut buf = [10; 3]; + let mut bools = [false; 16]; + bools[1] = true; + bools[2] = true; + bools[11] = true; + + pack_bitarray(&mut buf, &bools); + assert_eq!(buf[0], 0b0110_0000); + assert_eq!(buf[1], 0b0001_0000); + assert_eq!(buf[2], 10); + } + + #[test] + fn test_pack_int2() { + use crate::basic::pack_int2; + let mut buf = [10; 3 * 2]; + pack_int2(&mut buf, -3); + pack_int2(&mut buf[2..], 2); + pack_int2(&mut buf[4..], -1); + assert_eq!(buf[0..2], [0xFF, 0xFD]); + assert_eq!(buf[2..4], [0x00, 0x02]); + assert_eq!(buf[4..6], [0xFF, 0xFF]); + } + + #[test] + fn test_pack_int4() { + use crate::basic::pack_int4; + let mut buf = [10; 3 * 4]; + pack_int4(&mut buf, -3); + pack_int4(&mut buf[4..], 2); + pack_int4(&mut buf[8..], -1); + assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]); + assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]); + assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]); + } + + #[test] + fn test_encode_real8() { + use crate::basic::{encode_real8, decode_real8}; + const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; + for vv in REALS { + print!("{vv}\n"); + assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON); + } + } + + #[test] + fn test_pack_real8() { + use crate::basic::{pack_real8, parse_real8}; + const COUNT: usize = 7; + const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; + let mut buf = [10; 8 * COUNT]; + for (ii, &vv) in REALS.iter().enumerate() { + pack_real8(&mut buf[ii * 8..], vv).unwrap(); + } + for (ii, &vv) in REALS.iter().enumerate() { + print!("{vv}\n"); + let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1; + assert!((parsed_val - vv).abs() < f64::EPSILON); + } + } + + #[test] + fn test_pack_ascii() { + use crate::basic::pack_ascii; + let mut buf = [10; 12]; + pack_ascii(&mut buf[0..], "4321".as_bytes()); + pack_ascii(&mut buf[6..], "321".as_bytes()); + assert_eq!(&buf[0..4], "4321".as_bytes()); + assert_eq!(&buf[4..6], [10, 10]); + assert_eq!(&buf[6..9], "321".as_bytes()); + assert_eq!(&buf[9..], [0, 10, 10]); + } +} diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs new file mode 100644 index 0000000..646ec6e --- /dev/null +++ b/src/bin/pheon.rs @@ -0,0 +1,379 @@ +use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; +use pheon::review::{ + Comparison, DisplayFilter, Document, Scope, dump_records, hierarchy, parse_layer, + parse_layer_name, write_layer_names, +}; +use std::{ + collections::BTreeMap, + ffi::OsString, + fs, + io::{self, Write}, + path::PathBuf, + process::ExitCode, +}; + +const HELP: &str = "Usage: + pheon dump FILE [--records] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T]\n [--show-cell NAME] [--show-layer L/T] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] + pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE + pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... + +Object profile: ignore timestamps and element order, normalize polygon start/winding, +preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. +"; + +fn value(args: &[OsString], index: &mut usize) -> Result { + *index += 1; + args.get(*index) + .and_then(|s| s.to_str()) + .map(str::to_owned) + .ok_or_else(|| "missing UTF-8 option value".to_string()) +} + +fn name_layers(args: &[OsString]) -> Result { + if args.len() < 2 { + return Err(HELP.to_string()); + } + let (mut cell, mut carrier) = (None, None); + let mut names = BTreeMap::new(); + let mut i = 2; + while i < args.len() { + match args[i].to_str() { + Some("--cell") => cell = Some(value(args, &mut i)?), + Some("--carrier") => carrier = Some(parse_layer(&value(args, &mut i)?)?), + Some("--name") => { + let (pair, name) = parse_layer_name(&value(args, &mut i)?)?; + if names.get(&pair).is_some_and(|old| old != &name) { + return Err("conflicting --name arguments".to_string()); + } + names.insert(pair, name); + } + _ => { + return Err( + "name-layers requires --cell, --carrier and --name arguments".to_string(), + ); + } + } + i += 1; + } + if names.is_empty() { + return Err("provide at least one --name".to_string()); + } + let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; + let output = write_layer_names( + &input, + &cell.ok_or("provide --cell")?, + carrier.ok_or("provide --carrier")?, + &names, + )?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[1]) + .map_err(|e| format!("{:?}: {e}", args[1]))?; + if let Err(e) = file.write_all(&output).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(&args[1]); + return Err(e.to_string()); + } + Ok(0) +} + +fn stdout(output: &str) -> Result<(), String> { + io::stdout() + .lock() + .write_all(output.as_bytes()) + .map_err(|e| e.to_string()) +} + +fn json_stdout(report: &serde_json::Value) -> Result<(), String> { + let mut output = io::stdout().lock(); + serde_json::to_writer_pretty(&mut output, report).map_err(|e| e.to_string())?; + output.write_all(b"\n").map_err(|e| e.to_string()) +} + +fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { + let inputs = ReportInputs { + old: Some(PathBuf::from(&paths[0])), + new: Some(PathBuf::from(&paths[1])), + }; + let comparison = || { + let read = |index, side| { + fs::read(&paths[index]) + .map_err(|e| ("io_error", format!("{:?}: {e}", paths[index]), Some(side))) + }; + let old_bytes = read(0, "old")?; + let new_bytes = read(1, "new")?; + let old = Document::parse(&old_bytes).map_err(|e| ("invalid_input", e, Some("old")))?; + let new = Document::parse(&new_bytes).map_err(|e| ("invalid_input", e, Some("new")))?; + scope + .validate_pair(&old, &new) + .map_err(|e| ("invalid_scope", e, None))?; + let old = old + .selected(scope, true) + .map_err(|e| ("invalid_scope", e, Some("old")))?; + let new = new + .selected(scope, true) + .map_err(|e| ("invalid_scope", e, Some("new")))?; + let diff = Comparison::between(&old, &new); + Ok::<_, (&str, String, Option<&str>)>(( + diff.json_report(&old, &new, &inputs, options), + diff.exit_code(), + )) + }; + let (report, status) = match comparison() { + Ok(result) => result, + Err((code, message, side)) => ( + failure_report(&inputs, scope, options, code, &message, side), + 2, + ), + }; + if let Err(error) = json_stdout(&report) { + // Once output starts, never append a second report on a write failure. + eprintln!("pheon: {error}"); + return Ok(2); + } + Ok(status) +} + +fn run(args: &[OsString]) -> Result { + if args.len() == 1 && matches!(args[0].to_str(), Some("--version" | "-V")) { + stdout(&format!("pheon {}\n", env!("CARGO_PKG_VERSION")))?; + return Ok(0); + } + if args.is_empty() || args[0] == "--help" || args[0] == "-h" { + stdout(HELP)?; + return Ok(0); + } + let command = args[0].to_str().ok_or("invalid command")?; + if command == "name-layers" { + return name_layers(&args[1..]); + } + let expected = match command { + "dump" | "hierarchy" => 1, + "diff" => 2, + "git-diff" => 7, + _ => return Err(HELP.to_string()), + }; + let compare = command == "diff" || command == "git-diff"; + let mut scope = Scope::default(); + let mut show = DisplayFilter::default(); + let mut raw = false; + let mut format = if compare { + "none".to_string() + } else { + "ascii".to_string() + }; + let mut max_changes = 100; + let mut explicit_max = None; + let mut output_format = "text".to_string(); + let mut depth = 8; + let mut explicit_depth = false; + let mut paths = Vec::new(); + let mut literal = false; + let mut i = 1; + while i < args.len() { + if literal { + paths.push(args[i].clone()); + i += 1; + continue; + } + match args[i].to_str() { + Some("--") => literal = true, + Some("--help" | "-h") => { + stdout(HELP)?; + return Ok(0); + } + Some("--records") if command == "dump" => raw = true, + Some("--top") => scope.top = Some(value(args, &mut i)?), + Some("--cell") => scope.cell = Some(value(args, &mut i)?), + Some("--layer") => scope.layer = Some(parse_layer(&value(args, &mut i)?)?), + Some("--show-cell") if compare => show.cell = Some(value(args, &mut i)?), + Some("--show-layer") if compare => { + show.layer = Some(parse_layer(&value(args, &mut i)?)?) + } + Some("--hierarchy") if compare => format = value(args, &mut i)?, + Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, + Some("--format") if command == "diff" => output_format = value(args, &mut i)?, + Some("--max-changes") if compare => { + max_changes = value(args, &mut i)? + .parse() + .map_err(|_| "invalid change limit")?; + explicit_max = Some(max_changes); + } + Some("--depth") if command != "dump" => { + depth = value(args, &mut i)?.parse().map_err(|_| "invalid depth")?; + explicit_depth = true; + } + Some(s) if s.starts_with('-') => return Err(format!("unknown option {s:?}")), + _ => paths.push(args[i].clone()), + } + i += 1; + } + if paths.len() != expected && !(command == "git-diff" && paths.len() == 9) { + return Err(HELP.to_string()); + } + if !matches!(output_format.as_str(), "text" | "json") { + return Err("diff format must be text or json".to_string()); + } + if output_format == "json" { + if !matches!(format.as_str(), "none" | "graph") || explicit_depth { + return Err( + "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), + ); + } + return json_diff( + &paths, + &scope, + &ReportOptions { + filter: show, + max_objects: explicit_max, + hierarchy: format == "graph", + }, + ); + } + if !matches!(format.as_str(), "ascii" | "mermaid" | "none") + || (command == "hierarchy" && format == "none") + { + return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); + } + let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); + if raw { + if scope != Scope::default() { + return Err("record dumps do not accept object scope selectors".to_string()); + } + stdout(&dump_records(&read(&paths[0])?)?)?; + return Ok(0); + } + if !compare { + let bytes = read(&paths[0])?; + let doc = Document::parse(&bytes)?.selected(&scope, false)?; + let output = if command == "dump" { + doc.canonical_dump() + } else { + hierarchy(&doc, &doc, None, format == "mermaid", depth) + }; + stdout(&output)?; + if command == "hierarchy" { + for issue in &doc.issues { + eprintln!("INCOMPLETE: {issue}"); + } + } + return Ok(if doc.issues.is_empty() { 0 } else { 2 }); + } + let git = command == "git-diff"; + let old_missing = git && (paths[3] == "000000" || paths[3] == "."); + let new_missing = git && (paths[6] == "000000" || paths[6] == "."); + if old_missing && new_missing { + return Err("both Git revisions are absent".to_string()); + } + let old_bytes = if old_missing { + None + } else { + Some(read(&paths[if git { 1 } else { 0 }])?) + }; + let new_bytes = if new_missing { + None + } else { + Some(read(&paths[if git { 4 } else { 1 }])?) + }; + let old = old_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; + let new = new_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; + let old = old.unwrap_or_else(|| new.as_ref().unwrap().empty_like()); + let new = new.unwrap_or_else(|| old.empty_like()); + scope.validate_pair(&old, &new)?; + let old = old.selected(&scope, true)?; + let new = new.selected(&scope, true)?; + let diff = Comparison::between(&old, &new); + let mut output = String::new(); + if git { + if paths.len() == 9 { + output.push_str(&format!("layout {:?} -> {:?}\n", paths[0], paths[7])); + } else { + output.push_str(&format!("layout {:?}\n", paths[0])); + } + if old_missing { + output.push_str("layout added\n"); + } else if new_missing { + output.push_str("layout deleted\n"); + } + if paths[3] != paths[6] { + output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); + } + } + output.push_str(&diff.text_filtered(max_changes, &show)); + if format != "none" { + if format == "mermaid" { + output.push_str("```mermaid\n"); + } + output.push_str(&hierarchy( + &old, + &new, + Some(&diff), + format == "mermaid", + depth, + )); + if format == "mermaid" { + output.push_str("```\n"); + } + } + stdout(&output)?; + // Git's external driver expects successful rendering for either equality + // or differences. Actual comparison/input failures remain nonzero. + Ok(if git && diff.exit_code() == 1 { + 0 + } else { + diff.exit_code() + }) +} + +fn wants_json(args: &[OsString]) -> bool { + if args.first().is_none_or(|arg| arg != "diff") { + return false; + } + let mut format = None; + let mut i = 1; + while i < args.len() { + match args[i].to_str() { + Some("--") => break, + Some("--format") => { + format = args.get(i + 1).and_then(|v| v.to_str()); + i += 1; + } + Some( + "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" + | "--max-changes" | "--depth", + ) => i += 1, + _ => (), + } + i += 1; + } + format == Some("json") +} + +fn main() -> ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let json = wants_json(&args); + match run(&args) { + Ok(status) => ExitCode::from(status), + Err(message) => { + if json { + let report = failure_report( + &ReportInputs::default(), + &Scope::default(), + &ReportOptions::default(), + "command_error", + &message, + None, + ); + if let Err(error) = json_stdout(&report) { + eprintln!("pheon: {error}"); + } + } else { + eprintln!("pheon: {message}"); + } + ExitCode::from(2) + } + } +} diff --git a/src/elements.rs b/src/elements.rs new file mode 100644 index 0000000..f043ccc --- /dev/null +++ b/src/elements.rs @@ -0,0 +1,1004 @@ +/// +/// Functionality for reading/writing elements (geometry, text labels, +/// structure references) and associated properties. +/// + +use crate::records::{ + //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; + +use crate::records; +use crate::record::{RecordHeader, Record}; +use crate::basic::{IResult, fail, take_bytes}; //OResult + +use std::string::String; +use std::collections::HashMap; +//use std::io::Write; + +use arrow_array::builder::{ + StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, + Int32Builder, Int16Builder, UInt64Builder, UInt32Builder, UInt8Builder, Float64Builder, + }; + + +pub type DListBuilder = ListBuilder>; +pub type FListBuilder = FixedSizeListBuilder>; + +struct PropertyRecord { + key: i16, + value: String, +} + +struct BoundaryRecord { + layer_id: u32, + vertices: Vec, + properties: Vec, +} + +struct BoundaryBatchRecord { + layer_id: u32, + vertices: Vec, + vertex_offsets: Vec, +} + +struct RectBatchRecord { + layer_id: u32, + rects: Vec, +} + +struct SRefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + properties: Vec, +} + +struct ARefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + xy0: u64, + xy1: u64, + counts: u32, + properties: Vec, +} + +#[derive(Clone)] +pub struct ScanRefRecord { + pub target_id: u32, + pub invert_y: bool, + pub scale: f64, + pub angle_rad: f64, + pub xy: u64, + pub xy0: u64, + pub xy1: u64, + pub counts: u32, +} + +enum RefRecord { + SRef(SRefRecord), + ARef(ARefRecord), +} + + +const UNIT_COUNTS: u32 = (1_u32 << 16) | 1_u32; +const ZERO_XY: u64 = 0; + + +fn layer_id_for( + layer: i16, + dtype: i16, + layers: &mut HashMap, + ) -> u32 { + let layer32 = ((layer as u16 as u32) << 16) | (dtype as u16 as u32); + let next_id = layers.len(); + *layers.entry(layer32).or_insert(next_id.try_into().unwrap()) +} + + +fn insert_layer( + layer: i16, + dtype: i16, + layers: &mut HashMap, + struct_builder: &mut StructBuilder, + field_index: usize, + ) { + let id = layer_id_for(layer, dtype, layers); + let layer_builder = struct_builder.field_builder::(field_index).unwrap(); + layer_builder.append_value(id); +} + + +pub fn read_elements<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + names: &mut HashMap, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let mut input = input; + let mut boundaries = Vec::::new(); + let mut srefs = Vec::::new(); + let mut arefs = Vec::::new(); + let (_, mut header) = RecordHeader::read(input)?; // don't consume tag + while header.tag != records::RTAG_ENDSTR { + (input, _) = match header.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, header.tag, names)?; + match ref_record { + RefRecord::SRef(sref) => srefs.push(sref), + RefRecord::ARef(aref) => arefs.push(aref), + } + (next_input, ()) + }, + records::RTAG_BOUNDARY => { + let (next_input, boundary) = read_boundary(input, layers)?; + boundaries.push(boundary); + (next_input, ()) + }, + records::RTAG_PATH => read_path(input, cell_builder, layers)?, + records::RTAG_NODE => read_boxnode(input, cell_builder, header.tag, layers)?, + records::RTAG_BOX => read_boxnode(input, cell_builder, header.tag, layers)?, + records::RTAG_TEXT => read_text(input, cell_builder, layers)?, + _ => { + // don't care, skip + let result = take_bytes(input, header.data_size + 4)?; + (result.0, ()) + } + }; + (_, header) = RecordHeader::read(input)?; // don't consume tag + } + (input, _) = take_bytes(input, 4_usize)?; // consume endstr tag + + let mut sref_plain = Vec::::with_capacity(srefs.len() + arefs.len() / 4); + let mut sref_props = Vec::::with_capacity(srefs.len() / 16 + arefs.len() / 64); + for sref in srefs { + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } + + let mut aref_plain = Vec::::with_capacity(arefs.len()); + let mut aref_props = Vec::::with_capacity(arefs.len() / 16); + for aref in arefs { + if aref.counts == UNIT_COUNTS { + let sref = aref_to_sref(aref); + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } else if aref.properties.is_empty() { + aref_plain.push(aref); + } else { + aref_props.push(aref); + } + } + sref_plain.sort_by_key(|sref| sref.target_id); + aref_plain.sort_by_key(|aref| aref.target_id); + + let mut boundary_plain = Vec::::with_capacity(boundaries.len()); + let mut boundary_props = Vec::::with_capacity(boundaries.len() / 16); + for boundary in boundaries { + if boundary.properties.is_empty() { + boundary_plain.push(boundary); + } else { + boundary_props.push(boundary); + } + } + boundary_plain.sort_by_key(|boundary| boundary.layer_id); + + let mut rect_batches = Vec::::new(); + let mut boundary_batches = Vec::::new(); + let mut ii = 0; + while ii < boundary_plain.len() { + let start = ii; + let layer_id = boundary_plain[ii].layer_id; + while ii < boundary_plain.len() && boundary_plain[ii].layer_id == layer_id { + ii += 1; + } + + let group = &boundary_plain[start..ii]; + let rect_capacity = group.len() * 4; + let vertex_capacity = group.iter().map(|boundary| boundary.vertices.len()).sum(); + let mut rect_batch = RectBatchRecord { + layer_id, + rects: Vec::with_capacity(rect_capacity), + }; + let mut boundary_batch = BoundaryBatchRecord { + layer_id, + vertices: Vec::with_capacity(vertex_capacity), + vertex_offsets: Vec::with_capacity(group.len()), + }; + + let mut next_offset: u32 = 0; + for boundary in group { + if let Some(rect) = boundary_to_rect(&boundary.vertices) { + rect_batch.rects.extend_from_slice(&rect); + } else { + boundary_batch.vertex_offsets.push(next_offset); + boundary_batch.vertices.extend_from_slice(&boundary.vertices); + next_offset += (boundary.vertices.len() / 2) as u32; + } + } + if !rect_batch.rects.is_empty() { + rect_batches.push(rect_batch); + } + if !boundary_batch.vertex_offsets.is_empty() { + boundary_batches.push(boundary_batch); + } + } + + let sref_builder = cell_builder.field_builder::(2).unwrap(); + for sref in sref_plain { + append_sref(sref_builder, sref); + } + sref_builder.append(true); + + let aref_builder = cell_builder.field_builder::(3).unwrap(); + for aref in aref_plain { + append_aref(aref_builder, aref); + } + aref_builder.append(true); + + let sref_props_builder = cell_builder.field_builder::(4).unwrap(); + for sref in sref_props { + append_sref_prop(sref_props_builder, sref); + } + sref_props_builder.append(true); + + let aref_props_builder = cell_builder.field_builder::(5).unwrap(); + for aref in aref_props { + append_aref_prop(aref_props_builder, aref); + } + aref_props_builder.append(true); + + let rect_batches_builder = cell_builder.field_builder::(6).unwrap(); + for rect_batch in rect_batches { + append_rect_batch(rect_batches_builder, rect_batch); + } + rect_batches_builder.append(true); + + let boundary_batches_builder = cell_builder.field_builder::(7).unwrap(); + for boundary_batch in boundary_batches { + append_boundary_batch(boundary_batches_builder, boundary_batch); + } + boundary_batches_builder.append(true); + + let boundary_props_builder = cell_builder.field_builder::(8).unwrap(); + for boundary in boundary_props { + append_boundary_prop(boundary_props_builder, boundary); + } + boundary_props_builder.append(true); + + let paths_builder = cell_builder.field_builder::(9).unwrap(); + paths_builder.append(true); + let nodes_builder = cell_builder.field_builder::(10).unwrap(); + nodes_builder.append(true); + let boxes_builder = cell_builder.field_builder::(11).unwrap(); + boxes_builder.append(true); + let texts_builder = cell_builder.field_builder::(12).unwrap(); + texts_builder.append(true); + + Ok((input, ())) +} + + +pub fn scan_elements<'a>( + input: &'a [u8], + names: &mut HashMap, + ) -> IResult<'a, Vec> { + let mut input = input; + let mut refs = Vec::::new(); + let (_, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDSTR { + (input, _) = match header.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, header.tag, names)?; + refs.push(scan_ref_from_record(ref_record)); + (next_input, ()) + }, + records::RTAG_BOUNDARY + | records::RTAG_PATH + | records::RTAG_NODE + | records::RTAG_BOX + | records::RTAG_TEXT => skip_element(input)?, + _ => { + let result = take_bytes(input, header.data_size + 4)?; + (result.0, ()) + } + }; + (_, header) = RecordHeader::read(input)?; + } + refs.sort_by_key(|scan_ref| scan_ref.target_id); + (input, _) = take_bytes(input, 4_usize)?; + Ok((input, refs)) +} + + +fn append_sref( + sref_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_struct_builder = sref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + sref_struct_builder.append(true); +} + + +fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { + match ref_record { + RefRecord::SRef(sref) => ScanRefRecord { + target_id: sref.target_id, + invert_y: sref.invert_y, + scale: sref.scale, + angle_rad: sref.angle_rad, + xy: sref.xy, + xy0: ZERO_XY, + xy1: ZERO_XY, + counts: UNIT_COUNTS, + }, + RefRecord::ARef(aref) => ScanRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + xy0: aref.xy0, + xy1: aref.xy1, + counts: aref.counts, + }, + } +} + + +fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { + let mut input = input; + loop { + let (next_input, header) = RecordHeader::read(input)?; + let (next_input, _) = take_bytes(next_input, header.data_size)?; + input = next_input; + if header.tag == ENDEL::tag() { + return Ok((input, ())); + } + } +} + + +fn append_aref( + aref_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_struct_builder = aref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_struct_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_struct_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_struct_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + aref_struct_builder.append(true); +} + + +fn append_property(props_builder: &mut DListBuilder, property: PropertyRecord) { + let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); + let key_builder = prop_builder.field_builder::(0).unwrap(); + key_builder.append_value(property.key); + + let val_builder = prop_builder.field_builder::(1).unwrap(); + val_builder.append_value(property.value); + + prop_builder.append(true); +} + + +fn append_properties( + props_builder: &mut DListBuilder, + properties: Vec, + ) { + for prop in properties { + append_property(props_builder, prop); + } + props_builder.append(true); +} + + +fn append_sref_prop( + sref_props_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_prop_builder = sref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + let props_builder = sref_prop_builder.field_builder::(5).unwrap(); + append_properties(props_builder, sref.properties); + + sref_prop_builder.append(true); +} + + +fn append_aref_prop( + aref_props_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_prop_builder = aref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_prop_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_prop_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_prop_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + let props_builder = aref_prop_builder.field_builder::(8).unwrap(); + append_properties(props_builder, aref.properties); + + aref_prop_builder.append(true); +} + + +fn aref_to_sref(aref: ARefRecord) -> SRefRecord { + SRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + properties: aref.properties, + } +} + + +fn read_boundary<'a>( + input: &'a [u8], + layers: &mut HashMap, + ) -> IResult<'a, BoundaryRecord> { + let (input, _) = records::BOUNDARY::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + let layer_id = layer_id_for(layer, dtype, layers); + + let (input, xy_iter) = XY::read(input)?; + let mut vertices: Vec = xy_iter.collect(); + vertices.truncate(vertices.len().saturating_sub(2)); + + let (input, properties) = read_properties_vec(input)?; + + Ok((input, BoundaryRecord { layer_id, vertices, properties })) +} + + +fn append_boundary_batch( + boundary_batches_builder: &mut DListBuilder, + boundary_batch: BoundaryBatchRecord, + ) { + let boundary_batch_builder = boundary_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary_batch.layer_id); + + let vertices_builder = boundary_batch_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary_batch.vertices); + vertices_builder.append(true); + + let offsets_builder = boundary_batch_builder.field_builder::(2).unwrap(); + let offset_builder = offsets_builder.values().as_any_mut().downcast_mut::().unwrap(); + offset_builder.append_slice(&boundary_batch.vertex_offsets); + offsets_builder.append(true); + + boundary_batch_builder.append(true); +} + + +fn append_rect_batch( + rect_batches_builder: &mut DListBuilder, + rect_batch: RectBatchRecord, + ) { + let rect_batch_builder = rect_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = rect_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(rect_batch.layer_id); + + let rects_builder = rect_batch_builder.field_builder::(1).unwrap(); + let rect_builder = rects_builder.values().as_any_mut().downcast_mut::().unwrap(); + rect_builder.append_slice(&rect_batch.rects); + rects_builder.append(true); + + rect_batch_builder.append(true); +} + + +fn append_boundary_prop( + boundary_props_builder: &mut DListBuilder, + boundary: BoundaryRecord, + ) { + let boundary_prop_builder = boundary_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_prop_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary.layer_id); + + let vertices_builder = boundary_prop_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary.vertices); + vertices_builder.append(true); + + let props_builder = boundary_prop_builder.field_builder::(2).unwrap(); + append_properties(props_builder, boundary.properties); + + boundary_prop_builder.append(true); +} + + +pub fn read_path<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let paths_builder = cell_builder.field_builder::(9).unwrap(); + let path_builder = paths_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = records::PATH::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + insert_layer(layer, dtype, layers, path_builder, 0); + + let mut path_type = None; + let mut width = None; + let mut bgn_ext = None; + let mut end_ext = None; + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PATHTYPE => { + let _path_type; + (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; + path_type = Some(_path_type); + }, + records::RTAG_WIDTH => { + let _width; + (input, _width) = WIDTH::read_data(input, header.data_size)?; + width = Some(_width); + }, + records::RTAG_BGNEXTN => { + let _bgn_ext; + (input, _bgn_ext) = BGNEXTN::read_data(input, header.data_size)?; + bgn_ext = Some(_bgn_ext); + }, + records::RTAG_ENDEXTN => { + let _end_ext; + (input, _end_ext) = ENDEXTN::read_data(input, header.data_size)?; + end_ext = Some(_end_ext); + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + }; + (input, header) = RecordHeader::read(input)?; + } + let path_type_builder = path_builder.field_builder::(1).unwrap(); + path_type_builder.append_option(path_type); + let ext0_builder = path_builder.field_builder::(2).unwrap(); + ext0_builder.append_option(bgn_ext); + let ext1_builder = path_builder.field_builder::(3).unwrap(); + ext1_builder.append_option(end_ext); + let width_builder = path_builder.field_builder::(4).unwrap(); + width_builder.append_option(width); + + let xys_builder = path_builder.field_builder::(5).unwrap(); + let (input, xy_iter) = XY::read_data(input, header.data_size)?; + for xy in xy_iter { + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = path_builder.field_builder::(6).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + path_builder.append(true); + Ok((input, ())) +} + +pub fn read_boxnode<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + tag: u16, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let field_num = match tag { + records::RTAG_NODE => 10, + records::RTAG_BOX => 11, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let boxnodes_builder = cell_builder.field_builder::(field_num).unwrap(); + let boxnode_builder = boxnodes_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = match tag { + records::RTAG_NODE => records::NODE::read(input)?, + records::RTAG_BOX => records::BOX::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = match tag { + records::RTAG_NODE => NODETYPE::read(input)?, + records::RTAG_BOX => BOXTYPE::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + insert_layer(layer, dtype, layers, boxnode_builder, 0); + + let xys_builder = boxnode_builder.field_builder::(1).unwrap(); + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + let (input, xy_iter) = XY::read(input)?; + for xy in xy_iter { + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = boxnode_builder.field_builder::(2).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + boxnode_builder.append(true); + Ok((input, ())) +} + +pub fn read_text<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let texts_builder = cell_builder.field_builder::(12).unwrap(); + let text_builder = texts_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let mut path_type = None; + let mut pres_hori = None; + let mut pres_vert = None; + let mut pres_font = None; + let mut invert_y = None; + let mut width = None; + let mut mag = None; + let mut angle_deg = None; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = TEXTTYPE::read(input)?; + insert_layer(layer, dtype, layers, text_builder, 0); + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + // TODO warn if repeat tags? + records::RTAG_PRESENTATION => { + let _presentation; + (input, _presentation) = PRESENTATION::read_data(input, header.data_size)?; + pres_hori = Some(_presentation[14] as u8 * 2 + _presentation[15] as u8); + pres_vert = Some(_presentation[12] as u8 * 2 + _presentation[13] as u8); + pres_font = Some(_presentation[10] as u8 * 2 + _presentation[11] as u8); + }, + records::RTAG_PATHTYPE => { + let _path_type; + (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; + path_type = Some(_path_type); + }, + records::RTAG_WIDTH => { + let _width; + (input, _width) = WIDTH::read_data(input, header.data_size)?; + width = Some(_width); + }, + records::RTAG_STRANS => { + let strans; + (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = Some(strans[0]); + }, + records::RTAG_MAG => { + let _mag; + (input, _mag) = MAG::read_data(input, header.data_size)?; + mag = Some(_mag); + }, + records::RTAG_ANGLE => { + let _angle_deg; + (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; + angle_deg = Some(_angle_deg); + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + } + (input, header) = RecordHeader::read(input)?; + } + + let pres_hori_builder = text_builder.field_builder::(1).unwrap(); + pres_hori_builder.append_option(pres_hori); + let pres_vert_builder = text_builder.field_builder::(2).unwrap(); + pres_vert_builder.append_option(pres_vert); + let pres_font_builder = text_builder.field_builder::(3).unwrap(); + pres_font_builder.append_option(pres_font); + let path_type_builder = text_builder.field_builder::(4).unwrap(); + path_type_builder.append_option(path_type); + let width_builder = text_builder.field_builder::(5).unwrap(); + width_builder.append_option(width); + let inv_builder = text_builder.field_builder::(6).unwrap(); + inv_builder.append_option(invert_y); + let mag_builder = text_builder.field_builder::(7).unwrap(); + mag_builder.append_option(mag); + let angle_builder = text_builder.field_builder::(8).unwrap(); + angle_builder.append_option(angle_deg); + + let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; + let xx = xy_iter.next().unwrap(); + let yy = xy_iter.next().unwrap(); + let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let xy_builder = text_builder.field_builder::(9).unwrap(); + xy_builder.append_value(xy); + + let (input, string_bytes) = STRING::read(input)?; + let string = String::from_utf8(string_bytes).unwrap(); + let string_builder = text_builder.field_builder::(10).unwrap(); + string_builder.append_value(string); + + let props_builder = text_builder.field_builder::(11).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + text_builder.append(true); + Ok((input, ())) +} + + +fn boundary_to_rect(vertices: &[i32]) -> Option<[i32; 4]> { + if vertices.len() != 8 { + return None; + } + + let xs = [vertices[0], vertices[2], vertices[4], vertices[6]]; + let ys = [vertices[1], vertices[3], vertices[5], vertices[7]]; + + let min_x = *xs.iter().min().unwrap(); + let max_x = *xs.iter().max().unwrap(); + let min_y = *ys.iter().min().unwrap(); + let max_y = *ys.iter().max().unwrap(); + if min_x >= max_x || min_y >= max_y { + return None; + } + + let mut seen_corners: u8 = 0; + for ii in 0..4 { + let jj = (ii + 1) % 4; + let same_x = xs[jj] == xs[ii]; + let same_y = ys[jj] == ys[ii]; + if same_x == same_y { + return None; + } + + let x = xs[ii]; + let y = ys[ii]; + if (x != min_x && x != max_x) || (y != min_y && y != max_y) { + return None; + } + let code = ((x == max_x) as u8) << 1 | ((y == max_y) as u8); + seen_corners |= 1 << code; + } + + if seen_corners != 0b1111 { + return None; + } + Some([min_x, min_y, max_x, max_y]) +} + + + +fn read_ref<'a>( + input: &'a [u8], + tag: u16, + names: &mut HashMap, + ) -> IResult<'a, RefRecord> { + let (input, _) = match tag { + records::RTAG_SREF => records::SREF::read(input)?, + records::RTAG_AREF => records::AREF::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let is_aref = tag == records::RTAG_AREF; + let mut invert_y = false; + let mut scale = 1.0; + let mut angle_rad = 0.0; + let mut colrow = None; + + let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; + let struct_name = String::from_utf8(struct_name_bytes).unwrap(); + let next_id = names.len(); + let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_STRANS => { + let strans; + (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = strans[0]; + }, + records::RTAG_MAG => { + let _mag; + (input, _mag) = MAG::read_data(input, header.data_size)?; + scale = _mag; + }, + records::RTAG_ANGLE => { + let _angle_deg; + (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; + angle_rad = _angle_deg.to_radians(); + }, + records::RTAG_COLROW => { + if header.data_size != 4 { + return fail(input, "COLROW must contain two counts".to_string()); + } + let mut _colrow; + (input, _colrow) = COLROW::read_data(input, header.data_size)?; + colrow = Some((_colrow.next().unwrap(), _colrow.next().unwrap())); + if !is_aref { + return fail(input, "Got a COLROW record inside an SREF".to_string()); + } + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + }; + (input, header) = RecordHeader::read(input)?; + } + + let expected_xy_size = if is_aref { 24 } else { 8 }; + if header.data_size != expected_xy_size { + return fail(input, format!("Expected {expected_xy_size} bytes of reference coordinates, got {}", header.data_size)); + } + let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; + let xx = xy_iter.next().unwrap(); + let yy = xy_iter.next().unwrap(); + let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let (input, properties) = read_properties_vec(input)?; + + if is_aref { + if colrow.is_none() { + return fail(input, "AREF without COLROW before XY".to_string()) + } + let (count0, count1) = colrow.unwrap(); + if count0 <= 0 || count1 <= 0 { + return fail(input, "AREF column and row counts must be positive".to_string()); + } + + let pitch = |end: i32, origin: i32, count: i16| { + let delta = i64::from(end) - i64::from(origin); + let divisor = i64::from(count); + if delta % divisor != 0 { + return Err((input, crate::basic::ErrType::Failed( + "AREF pitch is not an integer database-unit increment; the Arrow schema cannot represent it exactly".to_string() + ))); + } + let value = delta / divisor; + i32::try_from(value).map_err(|_| (input, crate::basic::ErrType::Failed( + "AREF pitch exceeds signed 32-bit coordinate range".to_string() + ))) + }; + let x0a = pitch(xy_iter.next().unwrap(), xx, count0)?; + let y0a = pitch(xy_iter.next().unwrap(), yy, count0)?; + let x1a = pitch(xy_iter.next().unwrap(), xx, count1)?; + let y1a = pitch(xy_iter.next().unwrap(), yy, count1)?; + + let xy0a = ((x0a as u32 as u64) << 32) | (y0a as u32 as u64); + let xy1a = ((x1a as u32 as u64) << 32) | (y1a as u32 as u64); + let counts = ((count0 as u16 as u32) << 16) | (count1 as u16 as u32); + Ok((input, RefRecord::ARef(ARefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy, + xy0: xy0a, + xy1: xy1a, + counts, + properties, + }))) + } else { + Ok((input, RefRecord::SRef(SRefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy, + properties, + }))) + } +} + + +/// +/// Read element properties. +/// +/// Preserves record order and duplicate property attributes. +/// Stops reading after consuming ENDEL record. +/// +pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { + let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; + props_builder.append(true); + Ok((input, ())) +} + + +fn walk_properties<'a>( + input: &'a [u8], + mut on_property: impl FnMut(PropertyRecord), + ) -> IResult<'a, ()> { + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != ENDEL::tag() { + if header.tag == PROPATTR::tag() { + let key; + let value_bytes; + (input, key) = PROPATTR::read_data(input, header.data_size)?; + (input, value_bytes) = PROPVALUE::read(input)?; + + on_property(PropertyRecord { + key, + value: String::from_utf8(value_bytes).unwrap(), + }); + } + (input, header) = RecordHeader::read(input)?; + } + Ok((input, ())) +} + + +fn read_properties_vec<'a>(input: &'a [u8]) -> IResult<'a, Vec> { + let mut properties = Vec::new(); + let (input, ()) = walk_properties(input, |property| properties.push(property))?; + Ok((input, properties)) +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..0abcca5 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,307 @@ +pub mod basic; +pub mod record; +pub mod records; +pub mod elements; +pub mod library; +pub mod review; + +use crate::library::{read_library, read_selected_library, scan_library}; + +use std::any::Any; +use std::cell::RefCell; +use std::mem::size_of; +use std::ffi::{CStr, CString, c_char}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::ptr; +#[cfg(unix)] +use std::os::unix::ffi::OsStringExt; +use std::str; +use std::fs; +use std::slice; + +use arrow_array::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, StructArray}; + + +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} + +/// Stable C API version. Increment for incompatible function/ownership changes. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_abi_version() -> u32 { + 1 +} + +/// Layout of the full-read and hierarchy-scan Arrow payloads. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_arrow_schema_version() -> u32 { + 1 +} + + +fn clear_last_error() { + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = None); +} + + +fn set_last_error(message: String) -> i32 { + let sanitized = message.replace('\0', "\\0"); + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = Some(CString::new(sanitized).unwrap())); + -1 +} + + +fn describe_panic(payload: Box) -> String { + if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unexpected panic".to_string() + } +} + + +fn describe_parse_error(input: &[u8], (remaining, reason): (&[u8], crate::basic::ErrType)) -> String { + // Selected-cell errors may contain a bounded range rather than a file suffix. + let offset = remaining.as_ptr() as usize - input.as_ptr() as usize; + let message = match reason { + crate::basic::ErrType::Failed(message) => message, + crate::basic::ErrType::Incomplete(Some(size)) => format!("Unexpected end of input; need {size} bytes"), + crate::basic::ErrType::Incomplete(None) => "Unexpected end of input".to_string(), + }; + format!("at byte {offset}: {message}") +} + + +fn export_struct_array( + op_name: &str, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + build: impl FnOnce() -> Result, + ) -> i32 { + clear_last_error(); + if arr.is_null() || schema.is_null() { + return set_last_error(format!("{op_name}: Arrow output pointer was null")); + } + match catch_unwind(AssertUnwindSafe(build)) { + Ok(Ok(struct_arr)) => match to_ffi(&struct_arr.to_data()) { + Ok((ffi_arr, ffi_schema)) => { + unsafe { + *arr = ffi_arr; + *schema = ffi_schema; + } + 0 + }, + Err(err) => set_last_error(format!("{op_name}: failed to export Arrow data: {err}")), + }, + Ok(Err(message)) => set_last_error(format!("{op_name}: {message}")), + Err(payload) => set_last_error(format!("{op_name}: {}", describe_panic(payload))), + } +} + + +fn pathbuf_from_c(cpath: *const c_char) -> Result { + if cpath.is_null() { + return Err("path pointer was null".to_string()); + } + + let cstr = unsafe { CStr::from_ptr(cpath) }; + #[cfg(unix)] + { + return Ok(PathBuf::from(std::ffi::OsString::from_vec(cstr.to_bytes().to_vec()))); + } + #[cfg(windows)] + { + let ustr = str::from_utf8(cstr.to_bytes()) + .map_err(|_| "Non-UTF8 paths are not supported".to_string())?; + return Ok(PathBuf::from(ustr)); + } + #[allow(unreachable_code)] + Err("Unsupported OS".to_string()) +} + + +/// Return this thread's last parser error, or null if its last call succeeded. +/// The returned string remains valid until this thread's next parser call or exit. +#[unsafe(no_mangle)] +pub extern "C" fn last_error_message() -> *const c_char { + LAST_ERROR.with(|last_error| last_error.borrow() + .as_ref() + .map_or(ptr::null(), |message| message.as_ptr())) +} + + +/// Read a GDSII file and export its Arrow data. Return 0 on success or -1 on error. +/// +/// # Safety +/// `cpath` must point to a null-terminated path. Non-null output pointers must +/// point to initialized, writable Arrow FFI structures. Null pointers are rejected. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_path( + cpath: *const c_char, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_path", arr, schema, || { + let path = pathbuf_from_c(cpath)?; + let input = fs::read(Path::new(&path)) + .map_err(|err| format!("file read failed for {}: {err}", path.display()))?; + let (_input, struct_arr) = read_library(&input) + .map_err(|err| format!("read failed {}", describe_parse_error(&input, err)))?; + Ok(struct_arr) + }) +} + + +unsafe fn input_slice<'a, T>(data: *const T, size: usize, name: &str) -> Result<&'a [T], String> { + if size == 0 { + return Ok(&[]); + } + if data.is_null() { + return Err(format!("{name} pointer was null")); + } + if !data.is_aligned() { + return Err(format!("{name} pointer was misaligned")); + } + if size.checked_mul(size_of::()).is_none_or(|bytes| bytes > isize::MAX as usize) { + return Err(format!("{name} length is too large")); + } + Ok(unsafe { slice::from_raw_parts(data, size) }) +} + +/// Read a borrowed GDSII buffer, exporting Arrow data that owns its storage. +/// +/// # Safety +/// `data` must cover `size` readable bytes (null is allowed for zero bytes). +/// Outputs must be initialized Arrow FFI structures, as for `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, +) -> i32 { + export_struct_array("read_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = read_library(input) + .map_err(|err| format!("read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Scan cell ranges and hierarchy references. Return 0 on success or -1 on error. +/// +/// # Safety +/// Non-null `data` must point to `size` readable bytes. Null is allowed when +/// `size` is zero. Output pointers have the same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scan_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("scan_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = scan_library(input) + .map_err(|err| format!("scan failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Read the cells in half-open byte ranges. Return 0 on success or -1 on error. +/// +/// # Safety +/// `data` has the same requirements as `scan_bytes`. Non-null `ranges` must +/// point to `range_count * 2` readable, aligned u64 values, alternating start +/// and end offsets. Null is allowed for zero ranges. Output pointers have the +/// same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_cells_bytes( + data: *const u8, + size: usize, + ranges: *const u64, + range_count: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_cells_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let range_size = range_count.checked_mul(2) + .ok_or_else(|| "range count is too large".to_string())?; + let flat_ranges = unsafe { input_slice(ranges, range_size, "ranges") }?; + let cell_ranges: Vec<(u64, u64)> = flat_ranges + .as_chunks::<2>() + .0 + .iter() + .map(|&[start, end]| (start, end)) + .collect(); + let (_input, struct_arr) = read_selected_library(input, &cell_ranges) + .map_err(|err| format!("selected-cell read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +macro_rules! impl_intbe { + ($source:ty, $target:ty, $arr:ident, $size:ident) => {{ + let values = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; + for value in values { + // All source types represent the target bounds exactly in f64. + let number = *value as f64; + if !number.is_finite() + || number < <$target>::MIN as f64 + || number > <$target>::MAX as f64 + { + return *value; + } + let mut slot = [0; size_of::<$source>()]; + slot[..size_of::<$target>()].copy_from_slice(&(*value as $target).to_be_bytes()); + *value = <$source>::from_ne_bytes(slot); + } + 0 as $source + }}; +} + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i32, arr, size) } + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_intbe!(u16, i16, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_intbe!(i16, i16, arr, size) } diff --git a/src/library.rs b/src/library.rs new file mode 100644 index 0000000..1ea4c6b --- /dev/null +++ b/src/library.rs @@ -0,0 +1,588 @@ +/// +/// File-level read/write functionality. +/// + +use std::io::Write; +//use std::collections::HashMap; + +pub use crate::record; +pub use crate::record::{RecordHeader, Record}; +pub use crate::records; +pub use crate::elements; +pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +pub use crate::basic::{IResult, OResult, take_bytes, fail}; + +use std::string::String; +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field, Fields}; +use arrow_array::{ + StructArray, + builder::{ + BooleanBuilder, Float64Builder, Int16Builder, StringBuilder, StructBuilder, + UInt32Builder, UInt64Builder, + }, +}; + + +/// +/// Representation of the GDS file header. +/// +/// File header records: HEADER BGNLIB LIBNAME UNITS +/// Optional records are ignored if present and never written. +/// +/// Version is written as `600`. +/// +#[derive(Debug, Clone)] +pub struct FileHeader { + /// Number of user units in one database unit + user_units_per_db_unit: f64, + /// Number of meters in one database unit + meters_per_db_unit: f64, + /// Last-modified time [y, m, d, h, m, s] + mod_time: [i16; 6], + /// Last-accessed time [y, m, d, h, m, s] + acc_time: [i16; 6], + /// Library name + name: Vec, +} + +impl FileHeader { + pub fn new(name: &[u8], meters_per_db_unit: f64, user_units_per_db_unit: f64) -> Self { + FileHeader{ + mod_time: [0, 1, 1, 0, 0, 0], + acc_time: [0, 1, 1, 0, 0, 0], + name: name.to_owned(), + user_units_per_db_unit, + meters_per_db_unit, + } + } + + /// Read and construct a header from the provided input. + /// + /// Args: + /// input: Seekable input to read from + /// + /// Returns: + /// FileHeader object + /// + pub fn read(input: &[u8]) -> IResult<'_, Self> { + let (input, _version) = records::HEADER::read(input)?; + let (input, [mod_time, acc_time]) = records::BGNLIB::read(input)?; + let (input, name) = records::LIBNAME::skip_and_read(input)?; + let (input, (uu, dbu)) = records::UNITS::skip_and_read(input)?; + + Ok((input, FileHeader{ + mod_time, + acc_time, + name, + user_units_per_db_unit: uu, + meters_per_db_unit: dbu, + })) + } + + /// Write the header to a input + /// + /// Args: + /// input: input to write to + /// + /// Returns: + /// number of bytes written + /// + pub fn write(&self, ww: &mut W) -> OResult { + let mut size = 0; + size += records::HEADER::write(ww, &600)?; + size += records::BGNLIB::write(ww, &[self.mod_time, self.acc_time])?; + size += records::LIBNAME::write(ww, &self.name)?; + size += records::UNITS::write(ww, &(self.user_units_per_db_unit, self.meters_per_db_unit))?; + Ok(size) + } +} + + +fn make_name_list_type() -> DataType { + DataType::List(Arc::new( + Field::new_list_field(DataType::Utf8, false) + )) +} + + +fn make_time_type() -> DataType { + DataType::FixedSizeList(Arc::new( + Field::new_list_field(DataType::Int16, false), + ), + 6, + ) +} + + +fn make_top_level_builder(cells_field: Field, extra_fields: Vec) -> StructBuilder { + let mut fields = vec![ + Field::new("meters_per_db_unit", DataType::Float64, false), + Field::new("user_units_per_db_unit", DataType::Float64, false), + Field::new("lib_name", DataType::Utf8, false), + Field::new("mod_time", make_time_type(), false), + Field::new("acc_time", make_time_type(), false), + cells_field, + Field::new("cell_names", make_name_list_type(), false), + ]; + fields.extend(extra_fields); + StructBuilder::from_fields(fields, 0) +} + + +fn reference_type(is_array: bool, properties: Option<&DataType>) -> DataType { + let mut fields = vec![ + Field::new("target", DataType::UInt32, false), + Field::new("invert_y", DataType::Boolean, false), + Field::new("scale", DataType::Float64, false), + Field::new("angle_rad", DataType::Float64, false), + Field::new("xy", DataType::UInt64, false), + ]; + if is_array { + fields.extend([ + Field::new("xy0", DataType::UInt64, false), + Field::new("xy1", DataType::UInt64, false), + Field::new("counts", DataType::UInt32, false), + ]); + } + if let Some(properties) = properties { + fields.push(Field::new("properties", properties.clone(), true)); + } + DataType::Struct(Fields::from(fields)) +} + + +fn make_library_builder() -> StructBuilder { + let property_t = DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Int16, false), + Field::new("value", DataType::Utf8, false), + ])); + + let property_list_t = DataType::List(Arc::new( + Field::new_list_field(property_t, false) + )); + + let sref_struct_t = reference_type(false, None); + let aref_struct_t = reference_type(true, None); + let sref_prop_struct_t = reference_type(false, Some(&property_list_t)); + let aref_prop_struct_t = reference_type(true, Some(&property_list_t)); + + let text_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("presentation_horiz", DataType::UInt8, true), + Field::new("presentation_vert", DataType::UInt8, true), + Field::new("presentation_font", DataType::UInt8, true), + Field::new("path_type", DataType::Int16, true), + Field::new("width", DataType::Int32, true), + Field::new("invert_y", DataType::Boolean, true), + Field::new("mag", DataType::Float64, true), + Field::new("angle_deg", DataType::Float64, true), + Field::new("xy", DataType::UInt64, false), + Field::new("string", DataType::Utf8, false), + Field::new("properties", property_list_t.clone(), true), + ])); + + + let coords_t = DataType::List(Arc::new( + Field::new_list_field(DataType::Int32, false) + )); + + let boundary_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new( + "vertex_offsets", + DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false))), + false, + ), + ])); + + let rect_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("rects", coords_t.clone(), false), + ])); + + let boundary_prop_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let path_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("path_type", DataType::Int16, true), + Field::new("extension_start", DataType::Int32, true), + Field::new("extension_end", DataType::Int32, true), + Field::new("width", DataType::Int32, true), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let boxnode_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let sref_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_struct_t, false) + )); + + let aref_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_struct_t, false) + )); + + let sref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_prop_struct_t, false) + )); + + let aref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_prop_struct_t, false) + )); + + let text_list_t = DataType::List(Arc::new( + Field::new_list_field(text_struct_t, false) + )); + + let boundary_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_batch_struct_t, false) + )); + + let rect_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(rect_batch_struct_t, false) + )); + + let boundary_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_prop_struct_t, false) + )); + + let path_list_t = DataType::List(Arc::new( + Field::new_list_field(path_struct_t, false) + )); + + let boxnode_list_t = DataType::List(Arc::new( + Field::new_list_field(boxnode_struct_t, false) + )); + + let layer_list_t = DataType::List(Arc::new( + Field::new_list_field(DataType::UInt32, false) + )); + + let cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("file_offset", DataType::UInt64, false), + Field::new("srefs", sref_list_t, false), + Field::new("arefs", aref_list_t, false), + Field::new("sref_props", sref_prop_list_t, false), + Field::new("aref_props", aref_prop_list_t, false), + Field::new("rect_batches", rect_batch_list_t, false), + Field::new("boundary_batches", boundary_batch_list_t, false), + Field::new("boundary_props", boundary_prop_list_t, false), + Field::new("paths", path_list_t, false), + Field::new("nodes", boxnode_list_t.clone(), true), + Field::new("boxes", boxnode_list_t.clone(), true), + Field::new("texts", text_list_t, false), + ])); + + let cells_list_t = DataType::List(Arc::new( + Field::new_list_field(cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", cells_list_t, false), + vec![ + Field::new("layers", layer_list_t, false), + ], + ) +} + + +fn make_scan_builder() -> StructBuilder { + let scan_ref_struct_t = reference_type(true, None); + + let scan_ref_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_ref_struct_t, false) + )); + + let scan_cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("struct_start_offset", DataType::UInt64, false), + Field::new("struct_end_offset", DataType::UInt64, false), + Field::new("refs", scan_ref_list_t, false), + ])); + + let scan_cells_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", scan_cells_list_t, false), + Vec::new(), + ) +} + + +fn append_file_header( + lib_builder: &mut StructBuilder, + header: &FileHeader, + ) { + let dbu_builder = lib_builder.field_builder::(0).unwrap(); + dbu_builder.append_value(header.meters_per_db_unit); + let uu_builder = lib_builder.field_builder::(1).unwrap(); + uu_builder.append_value(header.user_units_per_db_unit); + let libname_builder = lib_builder.field_builder::(2).unwrap(); + libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); + let mtl_builder = lib_builder.field_builder::(3).unwrap(); + let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); + mt_builder.append_values(&header.mod_time, &[true; 6]); + mtl_builder.append(true); + let atl_builder = lib_builder.field_builder::(4).unwrap(); + let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); + at_builder.append_values(&header.acc_time, &[true; 6]); + atl_builder.append(true); +} + + +fn append_names( + lib_builder: &mut StructBuilder, + field_index: usize, + names: HashMap, + ) { + // Interned IDs are consecutive, so a vector directly represents ID order. + let mut ordered_names = vec![String::new(); names.len()]; + for (name, id) in names { + ordered_names[id as usize] = name; + } + let names_builder = lib_builder.field_builder::(field_index).unwrap(); + let name_builder = names_builder.values().as_any_mut().downcast_mut::().unwrap(); + for name in ordered_names { + name_builder.append_value(name); + } + names_builder.append(true); +} + + +fn append_layers( + lib_builder: &mut StructBuilder, + layers: HashMap, + ) { + let mut ordered_layers = vec![0; layers.len()]; + for (layer, id) in layers { + ordered_layers[id as usize] = layer; + } + let layers_builder = lib_builder.field_builder::(7).unwrap(); + let layer_builder = layers_builder.values().as_any_mut().downcast_mut::().unwrap(); + layer_builder.append_slice(&ordered_layers); + layers_builder.append(true); +} + + +fn append_scan_ref( + refs_builder: &mut DListBuilder, + scan_ref: ScanRefRecord, + ) { + let ref_builder = refs_builder.values().as_any_mut().downcast_mut::().unwrap(); + let target_builder = ref_builder.field_builder::(0).unwrap(); + target_builder.append_value(scan_ref.target_id); + let invert_builder = ref_builder.field_builder::(1).unwrap(); + invert_builder.append_value(scan_ref.invert_y); + let scale_builder = ref_builder.field_builder::(2).unwrap(); + scale_builder.append_value(scan_ref.scale); + let angle_builder = ref_builder.field_builder::(3).unwrap(); + angle_builder.append_value(scan_ref.angle_rad); + let xy_builder = ref_builder.field_builder::(4).unwrap(); + xy_builder.append_value(scan_ref.xy); + let xy0_builder = ref_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(scan_ref.xy0); + let xy1_builder = ref_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(scan_ref.xy1); + let counts_builder = ref_builder.field_builder::(7).unwrap(); + counts_builder.append_value(scan_ref.counts); + ref_builder.append(true); +} + + +fn intern_name(names: &mut HashMap, name: String) -> u32 { + let next_id = names.len(); + *names.entry(name).or_insert(next_id.try_into().unwrap()) +} + + +fn append_full_cell<'a>( + cells_builder: &mut DListBuilder, + names: &mut HashMap, + layers: &mut HashMap, + struct_input: &'a [u8], + id: u32, + file_offset: u64, + ) -> IResult<'a, ()> { + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let offset_builder = cell_builder.field_builder::(1).unwrap(); + offset_builder.append_value(file_offset); + + let (next_input, _) = read_elements(struct_input, cell_builder, names, layers)?; + cell_builder.append(true); + Ok((next_input, ())) +} + + +fn walk_structs<'a, F>( + input: &'a [u8], + base_offset: usize, + names: &mut HashMap, + mut on_struct: F, + ) -> IResult<'a, ()> +where + F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, +{ + let input_size = input.len(); + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDLIB { + let record_start = base_offset + input_size - input.len() - 4; + (input, _) = take_bytes(input, header.data_size)?; + if header.tag == records::RTAG_BGNSTR { + let name_bytes; + (input, name_bytes) = records::STRNAME::read(input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let id = intern_name(names, name); + let (next_input, ()) = on_struct(input, record_start, id, names)?; + input = next_input; + } + (input, header) = RecordHeader::read(input)?; + } + Ok((input, ())) +} + + +pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_library_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, _record_start, id, names| { + let position = base_offset + walk_input_size - struct_input.len(); + append_full_cell( + cells_builder, + names, + &mut layers, + struct_input, + id, + position.try_into().unwrap(), + ) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Scan cell byte ranges and references without decoding geometry. +pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_scan_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, record_start, id, names| { + let (after_cell, refs) = scan_elements(struct_input, names)?; + let struct_end = base_offset + walk_input_size - after_cell.len(); + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let start_builder = cell_builder.field_builder::(1).unwrap(); + start_builder.append_value(record_start.try_into().unwrap()); + + let end_builder = cell_builder.field_builder::(2).unwrap(); + end_builder.append_value(struct_end.try_into().unwrap()); + + let refs_builder = cell_builder.field_builder::(3).unwrap(); + for scan_ref in refs { + append_scan_ref(refs_builder, scan_ref); + } + refs_builder.append(true); + cell_builder.append(true); + Ok((after_cell, ())) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Read exact half-open cell ranges returned by `scan_library`, in caller order. +/// Each range includes BGNSTR through ENDSTR. Cell and layer IDs are local to +/// this result. This random-access operation returns the original input slice +/// as its remainder; it does not advance a sequential parsing cursor. +pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { + let mut lib_builder = make_library_builder(); + let (_, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + for (start_u64, end_u64) in ranges { + if start_u64 >= end_u64 || *end_u64 > input.len() as u64 { + return fail(input, format!("Cell byte range out of bounds: [{start_u64}, {end_u64})")); + } + let start = usize::try_from(*start_u64).unwrap(); + let end = usize::try_from(*end_u64).unwrap(); + + let struct_input = &input[start..end]; + let (struct_input, _) = records::BGNSTR::read(struct_input)?; + let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let file_offset = end - struct_input.len(); + + let id = intern_name(&mut names, name); + let (remaining_struct, _) = append_full_cell( + cells_builder, + &mut names, + &mut layers, + struct_input, + id, + file_offset.try_into().unwrap(), + )?; + if !remaining_struct.is_empty() { + return fail(remaining_struct, "Cell byte range extends past ENDSTR".to_string()); + } + } + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} diff --git a/src/misc.py b/src/misc.py new file mode 100644 index 0000000..1e507c7 --- /dev/null +++ b/src/misc.py @@ -0,0 +1,39 @@ +''' +https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py +''' +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes +import pyarrow as pa +from pyarrow.cffi import ffi + + c_schema = ffi.new('struct ArrowSchema*') + ptr_schema = int(ffi.cast('uintptr_t', c_schema)) + c_array = ffi.new('struct ArrowArray*') + ptr_array = int(ffi.cast('uintptr_t', c_array)) + + # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) + + # import gc + # gc.collect() # Make sure no Arrow data dangles in a ref cycle + # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) + + arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) + + diff --git a/src/record.rs b/src/record.rs new file mode 100644 index 0000000..cb64ea2 --- /dev/null +++ b/src/record.rs @@ -0,0 +1,430 @@ +/// +/// Generic record-level read/write functionality. +/// +use std::io::{self, Write}; + + +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::records; + + +//#[no_mangle] +//pub extern "C" fn write_record_header( + + +#[repr(C)] +#[derive(Debug, Clone)] +pub struct RecordHeader { + pub tag: u16, + pub data_size: u16, +} + +impl RecordHeader { + pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { + let (input, size) = parse_u16(input)?; + let (input, tag) = parse_u16(input)?; + let Some(data_size) = size.checked_sub(4) else { + return fail(input, format!("Record size {size} is smaller than its header")); + }; + Ok((input, RecordHeader{tag, data_size})) + } + + pub fn pack_into(&self) -> [u8; 4] { + let size = self.data_size.checked_add(4).expect("Record too big!"); + let [size_hi, size_lo] = size.to_be_bytes(); + let [tag_hi, tag_lo] = self.tag.to_be_bytes(); + [size_hi, size_lo, tag_hi, tag_lo] + } + + pub fn write(&self, ww: &mut W) -> OResult { + let bytes = self.pack_into(); + ww.write_all(&bytes)?; + Ok(bytes.len()) + } +} + + +pub trait RecordData { + type BareData<'a>; + type InData : ?Sized; + type ByteData : AsRef<[u8]>; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; + fn pack_into(buf: &mut [u8], data: &Self::InData); + //fn size(data: &Self::BareData<'_>) -> u16; + fn pack(data: &Self::InData) -> Self::ByteData; +} + + +pub trait Record { + fn tag() -> u16; + fn expected_size() -> Option; + + fn check_size(actual_size: u16) -> Result<(), String> { + match Self::expected_size() { + Some(size) => if size == actual_size { + Ok(()) + } else { + Err(format!("Expected record size {}, got {}", size, actual_size)) + }, + None => Ok(()), + } + } + + fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { + RecordHeader::read(input) + } + + fn write_header(ww: &mut W, data_size: u16) -> OResult { + RecordHeader{tag: Self::tag(), data_size}.write(ww) + } + + fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + RData::read(input, size) + } + + fn pack_data(buf: &mut [u8], data: &RData::InData) { + RData::pack_into(buf, data) + } + + /// + /// Skip to the end of the next occurence of this record. + /// + /// Return: + /// True if the record was encountered and skipped. + /// False if the end of the library was reached. + /// + fn skip_past(input: &[u8]) -> IResult<'_, bool> { + let original_input = input; + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != Self::tag() { + (input, _) = take_bytes(input, header.data_size)?; + if header.tag == records::RTAG_ENDLIB { + return Ok((original_input, false)) + } + (input, header) = RecordHeader::read(input)?; + } + (input, _) = take_bytes(input, header.data_size)?; + Ok((input, true)) + } + + fn skip_and_read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != Self::tag() { + (input, _) = take_bytes(input, header.data_size)?; + (input, header) = RecordHeader::read(input)?; + } + let (input, data) = Self::read_data(input, header.data_size)?; + Ok((input, data)) + } + + fn expect_header(input: &[u8]) -> IResult<'_, u16> { + let (input, header) = RecordHeader::read(input)?; + if header.tag != Self::tag() { + fail(input, format!("Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", header.tag, Self::tag())) + } else { + Ok((input, header.data_size)) + } + } + + fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let (input, size) = Self::expect_header(input)?; + Self::check_size(size).unwrap(); + let (input, data) = Self::read_data(input, size)?; + Ok((input, data)) + } + + fn write(ww: &mut W, data: &RData::InData) -> OResult { + let packed_data = RData::pack(data); + let data_bytes = packed_data.as_ref(); + let len = u16::try_from(data_bytes.len()).ok() + .filter(|len| *len <= u16::MAX - 4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + Self::write_header(ww, len)?; + ww.write_all(data_bytes)?; + Ok(usize::from(len) + 4) + } +} + +pub struct BitArray; +impl RecordData for BitArray { + type BareData<'a> = [bool; 16]; + type InData = [bool; 16]; + type ByteData = [u8; 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2); + parse_bitarray(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_bitarray(buf, data); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct Int2; +impl RecordData for Int2 { + type BareData<'a> = i16; + type InData = i16; + type ByteData = [u8; 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2); + parse_int2(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_int2(buf, *data) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int4; +impl RecordData for Int4 { + type BareData<'a> = i32; + type InData = i32; + type ByteData = [u8; 4]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 4); + parse_int4(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_int4(buf, *data) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 4]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct Int2Array; +impl RecordData for Int2Array { + type BareData<'a> = Int2ArrayReader<'a>; + type InData = [i16]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers"); + //let mut input = input; + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int2ArrayReader{bytes})) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + assert_eq!(buf.len(), data.len() * 2); + for (bytes, value) in buf.as_chunks_mut::<2>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() * 2]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int2ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int2ArrayReader<'_> { + type Item = i16; + + fn next(&mut self) -> Option { + if self.bytes.len() < 2 { + None + } else { + let (remaining, val) = parse_int2(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + +pub struct Int4Array; +impl RecordData for Int4Array { + type BareData<'a> = Int4ArrayReader<'a>; + type InData = [i32]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers"); + //let mut input = input; + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int4ArrayReader{bytes})) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + assert_eq!(buf.len(), data.len() * 4); + for (bytes, value) in buf.as_chunks_mut::<4>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() * 4]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int4ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int4ArrayReader<'_> { + type Item = i32; + + fn next(&mut self) -> Option { + if self.bytes.len() < 4 { + None + } else { + let (remaining, val) = parse_int4(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + +pub struct Real8; +impl RecordData for Real8 { + type BareData<'a> = f64; + type InData = f64; + type ByteData = [u8; 8]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 8); + parse_real8(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_real8(buf, *data).unwrap_or_else(|_| panic!("Float {0} too big for Real8", data)) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 8]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Real8Pair; +impl RecordData for Real8Pair { + type BareData<'a> = (f64, f64); + type InData = (f64, f64); + type ByteData = [u8; 2 * 8]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2 * 8); + let (input, data0) = parse_real8(input)?; + let (input, data1) = parse_real8(input)?; + Ok((input, (data0, data1))) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_real8(buf, data.0).unwrap_or_else(|_| panic!("Float.0 {0} too big for Real8", data.0)); + pack_real8(&mut buf[8 * 1..], data.1).unwrap_or_else(|_| panic!("Float.1 {0} too big for Real8", data.1)); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2 * 8]; + Self::pack_into(&mut buf, data); + buf + } + + //fn write(ww: &mut W, data: &Self::BareData) -> OResult { + // let mut buf = [u8; 2 * 6 * 2]; + // Self::pack_into(&mut buf, data) + //} +} + +pub struct ASCII; +impl RecordData for ASCII { + type BareData<'a> = Vec; + type InData = [u8]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + parse_ascii(input, size) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_ascii(buf, data); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() + data.len() % 2]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct DateTimePair; +impl RecordData for DateTimePair { + type BareData<'a> = [[i16; 6]; 2]; + type InData = [[i16; 6]; 2]; + type ByteData = [u8; 2 * 6 * 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2 * 6 * 2); + let (input, data0) = parse_datetime(input)?; + let (input, data1) = parse_datetime(input)?; + Ok((input, [data0, data1])) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_datetime(buf, &data[0]); + pack_datetime(&mut buf[6 * 2 * 1..], &data[1]); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2 * 6 * 2]; + Self::pack_into(&mut buf, data); + buf + } + + //fn write(ww: &mut W, data: &Self::BareData) -> OResult { + // let mut buf = [u8; 2 * 6 * 2]; + // Self::pack_into(&mut buf, data) + //} +} + +pub struct Empty; +impl RecordData for Empty { + type BareData<'a> = (); + type InData = (); + type ByteData = [u8; 0]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 0); + Ok((input, ())) + } + + fn pack_into(_buf: &mut [u8], _data: &Self::InData) { + } + + fn pack(_data: &Self::InData) -> Self::ByteData { + [] + } + //fn write(ww: &mut W, data: &Self::BareData) { + //} +} diff --git a/src/records.rs b/src/records.rs new file mode 100644 index 0000000..4e86404 --- /dev/null +++ b/src/records.rs @@ -0,0 +1,567 @@ +/// +/// Record type and tag definitions +/// + +use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; + +//use std::io::Write; + +// record tags +pub const RTAG_HEADER: u16 = 0x0002; +pub const RTAG_BGNLIB: u16 = 0x0102; +pub const RTAG_LIBNAME: u16 = 0x0206; +pub const RTAG_UNITS: u16 = 0x0305; // (user_units_per_db_unit, db_units_per_meter) +pub const RTAG_ENDLIB: u16 = 0x0400; +pub const RTAG_BGNSTR: u16 = 0x0502; +pub const RTAG_STRNAME: u16 = 0x0606; +pub const RTAG_ENDSTR: u16 = 0x0700; +pub const RTAG_BOUNDARY: u16 = 0x0800; +pub const RTAG_PATH: u16 = 0x0900; +pub const RTAG_SREF: u16 = 0x0a00; +pub const RTAG_AREF: u16 = 0x0b00; +pub const RTAG_TEXT: u16 = 0x0c00; +pub const RTAG_LAYER: u16 = 0x0d02; +pub const RTAG_DATATYPE: u16 = 0x0e02; +pub const RTAG_WIDTH: u16 = 0x0f03; +pub const RTAG_XY: u16 = 0x1003; +pub const RTAG_ENDEL: u16 = 0x1100; +pub const RTAG_SNAME: u16 = 0x1206; +pub const RTAG_COLROW: u16 = 0x1302; +pub const RTAG_NODE: u16 = 0x1500; +pub const RTAG_TEXTTYPE: u16 = 0x1602; +pub const RTAG_PRESENTATION: u16 = 0x1701; +pub const RTAG_SPACING: u16 = 0x1802; // unused; not sure about 02 +pub const RTAG_STRING: u16 = 0x1906; +pub const RTAG_STRANS: u16 = 0x1a01; +pub const RTAG_MAG: u16 = 0x1b05; +pub const RTAG_ANGLE: u16 = 0x1c05; +pub const RTAG_UINTEGER: u16 = 0x1d02; // unused; not sure about 02 +pub const RTAG_USTRING: u16 = 0x1e06; // unused; not sure about 06 +pub const RTAG_REFLIBS: u16 = 0x1f06; +pub const RTAG_FONTS: u16 = 0x2006; +pub const RTAG_PATHTYPE: u16 = 0x2102; +pub const RTAG_GENERATIONS: u16 = 0x2202; +pub const RTAG_ATTRTABLE: u16 = 0x2306; +pub const RTAG_STYPTABLE: u16 = 0x2406; // unused; not sure about 06 +pub const RTAG_STRTYPE: u16 = 0x2502; // unused +pub const RTAG_ELFLAGS: u16 = 0x2601; +pub const RTAG_ELKEY: u16 = 0x2703; // unused +pub const RTAG_LINKTYPE: u16 = 0x2803; // unused +pub const RTAG_LINKKEYS: u16 = 0x2903; // unused +pub const RTAG_NODETYPE: u16 = 0x2a02; +pub const RTAG_PROPATTR: u16 = 0x2b02; +pub const RTAG_PROPVALUE: u16 = 0x2c06; +pub const RTAG_BOX: u16 = 0x2d00; +pub const RTAG_BOXTYPE: u16 = 0x2e02; +pub const RTAG_PLEX: u16 = 0x2f03; +pub const RTAG_BGNEXTN: u16 = 0x3003; +pub const RTAG_ENDEXTN: u16 = 0x3103; +pub const RTAG_TAPENUM: u16 = 0x3202; +pub const RTAG_TAPECODE: u16 = 0x3302; +pub const RTAG_STRCLASS: u16 = 0x3401; +pub const RTAG_RESERVED: u16 = 0x3503; +pub const RTAG_FORMAT: u16 = 0x3602; +pub const RTAG_MASK: u16 = 0x3706; // list of Layers and dtypes +pub const RTAG_ENDMASKS: u16 = 0x3800; // end of MASKS records +pub const RTAG_LIBDIRSIZE: u16 = 0x3902; +pub const RTAG_SRFNAME: u16 = 0x3a06; +pub const RTAG_LIBSECUR: u16 = 0x3b02; +pub const RTAG_BORDER: u16 = 0x3c00; +pub const RTAG_SOFTFENCE: u16 = 0x3d00; +pub const RTAG_HARDFENCE: u16 = 0x3f00; +pub const RTAG_SOFTWIRE: u16 = 0x3f00; +pub const RTAG_HARDWIRE: u16 = 0x4000; +pub const RTAG_PATHPORT: u16 = 0x4100; +pub const RTAG_NODEPORT: u16 = 0x4200; +pub const RTAG_USERCONSTRAINT: u16 = 0x4300; +pub const RTAG_SPACERERROR: u16 = 0x4400; +pub const RTAG_CONTACT: u16 = 0x4500; + +/* +// data types +pub const DATA_TYPE_NONE: u16 = 0x00; +pub const DATA_TYPE_BIT: u16 = 0x01; +pub const DATA_TYPE_INT16: u16 = 0x02; +pub const DATA_TYPE_INT32: u16 = 0x03; +pub const DATA_TYPE_REAL32: u16 = 0x04; +pub const DATA_TYPE_REAL64: u16 = 0x05; +pub const DATA_TYPE_STR: u16 = 0x06; + +pub const MAX_DATA_SIZE: usize = 8; + +/// Returns the size of the given data type in bytes. +pub fn data_size(t: u16) -> Option { + match t { + x if x == DATA_TYPE_NONE => 0, + x if x == DATA_TYPE_BIT => 2, + x if x == DATA_TYPE_INT16 => 2, + x if x == DATA_TYPE_INT32 => 4, + x if x == DATA_TYPE_REAL32 => 4, + x if x == DATA_TYPE_REAL64 => 8, + _ => 0 + } +*/ + +pub struct HEADER; +impl Record for HEADER { + fn tag() -> u16 { RTAG_HEADER } + fn expected_size() -> Option { Some(2) } +} +//impl Record for HEADER; + +pub struct BGNLIB; +impl Record for BGNLIB { + fn tag() -> u16 { RTAG_BGNLIB } + fn expected_size() -> Option { Some(2 * 2 * 6) } +} + +pub struct LIBNAME; +impl Record for LIBNAME { + fn tag() -> u16 { RTAG_LIBNAME } + fn expected_size() -> Option { None } +} + +pub struct UNITS; +impl Record for UNITS { + // (user_units_per_db_unit, db_units_per_meter) + fn tag() -> u16 { RTAG_UNITS } + fn expected_size() -> Option { Some(2 * 8) } +} + +pub struct ENDLIB; +impl Record for ENDLIB { + fn tag() -> u16 { RTAG_ENDLIB } + fn expected_size() -> Option { Some(0) } +} + +pub struct BGNSTR; +impl Record for BGNSTR { + fn tag() -> u16 { RTAG_BGNSTR } + fn expected_size() -> Option { Some(2 * 6 * 2) } +} + +pub struct STRNAME; +impl Record for STRNAME { + fn tag() -> u16 { RTAG_STRNAME } + fn expected_size() -> Option { None } +} + +pub struct ENDSTR; +impl Record for ENDSTR { + fn tag() -> u16 { RTAG_ENDSTR } + fn expected_size() -> Option { Some(0) } +} + +pub struct BOUNDARY; +impl Record for BOUNDARY { + fn tag() -> u16 { RTAG_BOUNDARY } + fn expected_size() -> Option { Some(0) } +} + +pub struct PATH; +impl Record for PATH { + fn tag() -> u16 { RTAG_PATH } + fn expected_size() -> Option { Some(0) } +} + +pub struct SREF; +impl Record for SREF { + fn tag() -> u16 { RTAG_SREF } + fn expected_size() -> Option { Some(0) } +} + +pub struct AREF; +impl Record for AREF { + fn tag() -> u16 { RTAG_AREF } + fn expected_size() -> Option { Some(0) } +} + +pub struct TEXT; +impl Record for TEXT { + fn tag() -> u16 { RTAG_TEXT } + fn expected_size() -> Option { Some(0) } +} + +pub struct LAYER; +impl Record for LAYER { + fn tag() -> u16 { RTAG_LAYER } + fn expected_size() -> Option { Some(2) } +} + +pub struct DATATYPE; +impl Record for DATATYPE { + fn tag() -> u16 { RTAG_DATATYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct WIDTH; +impl Record for WIDTH { + fn tag() -> u16 { RTAG_WIDTH } + fn expected_size() -> Option { Some(4) } +} + +pub struct XY; +impl Record for XY { + fn tag() -> u16 { RTAG_XY } + fn expected_size() -> Option { None } +} + +pub struct ENDEL; +impl Record for ENDEL { + fn tag() -> u16 { RTAG_ENDEL } + fn expected_size() -> Option { Some(0) } +} + +pub struct SNAME; +impl Record for SNAME { + fn tag() -> u16 { RTAG_SNAME } + fn expected_size() -> Option { None } +} + +pub struct COLROW; +impl Record for COLROW { + fn tag() -> u16 { RTAG_COLROW } + fn expected_size() -> Option { Some(4) } +} + +pub struct NODE; +impl Record for NODE { + fn tag() -> u16 { RTAG_NODE } + fn expected_size() -> Option { Some(0) } +} + +pub struct TEXTTYPE; +impl Record for TEXTTYPE { + fn tag() -> u16 { RTAG_TEXTTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PRESENTATION; +impl Record for PRESENTATION { + fn tag() -> u16 { RTAG_PRESENTATION } + fn expected_size() -> Option { Some(2) } +} + +pub struct SPACING; +impl Record for SPACING { + fn tag() -> u16 { RTAG_SPACING } + fn expected_size() -> Option { Some(2) } +} + +pub struct STRING; +impl Record for STRING { + fn tag() -> u16 { RTAG_STRING } + fn expected_size() -> Option { None } +} + +pub struct STRANS; +impl Record for STRANS { + fn tag() -> u16 { RTAG_STRANS } + fn expected_size() -> Option { Some(2) } +} + +pub struct MAG; +impl Record for MAG { + fn tag() -> u16 { RTAG_MAG } + fn expected_size() -> Option { Some(8) } +} + +pub struct ANGLE; +impl Record for ANGLE { + fn tag() -> u16 { RTAG_ANGLE } + fn expected_size() -> Option { Some(8) } +} + +pub struct UINTEGER; +impl Record for UINTEGER { + fn tag() -> u16 { RTAG_UINTEGER } + fn expected_size() -> Option { Some(2) } +} + +pub struct USTRING; +impl Record for USTRING { + fn tag() -> u16 { RTAG_USTRING } + fn expected_size() -> Option { None } +} + +pub struct REFLIBS; +impl Record for REFLIBS { + fn tag() -> u16 { RTAG_REFLIBS } + fn expected_size() -> Option { None } +} +impl REFLIBS { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct FONTS; +impl Record for FONTS { + fn tag() -> u16 { RTAG_FONTS } + fn expected_size() -> Option { None } +} +impl FONTS { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct PATHTYPE; +impl Record for PATHTYPE { + fn tag() -> u16 { RTAG_PATHTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct GENERATIONS; +impl Record for GENERATIONS { + fn tag() -> u16 { RTAG_GENERATIONS } + fn expected_size() -> Option { Some(2) } +} + +pub struct ATTRTABLE; +impl Record for ATTRTABLE { + fn tag() -> u16 { RTAG_ATTRTABLE } + fn expected_size() -> Option { None } +} +impl ATTRTABLE { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct STYPTABLE; +impl Record for STYPTABLE { + fn tag() -> u16 { RTAG_STYPTABLE } + fn expected_size() -> Option { None } +} + +pub struct STRTYPE; +impl Record for STRTYPE { + fn tag() -> u16 { RTAG_STRTYPE } + fn expected_size() -> Option { None } +} + +pub struct ELFLAGS; +impl Record for ELFLAGS { + fn tag() -> u16 { RTAG_ELFLAGS } + fn expected_size() -> Option { Some(2) } +} + +pub struct ELKEY; +impl Record for ELKEY { + fn tag() -> u16 { RTAG_ELKEY } + fn expected_size() -> Option { Some(2) } +} + +pub struct LINKTYPE; +impl Record for LINKTYPE { + fn tag() -> u16 { RTAG_LINKTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct LINKKEYS; +impl Record for LINKKEYS { + fn tag() -> u16 { RTAG_LINKKEYS } + fn expected_size() -> Option { Some(2) } +} + +pub struct NODETYPE; +impl Record for NODETYPE { + fn tag() -> u16 { RTAG_NODETYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PROPATTR; +impl Record for PROPATTR { + fn tag() -> u16 { RTAG_PROPATTR } + fn expected_size() -> Option { Some(2) } +} + +pub struct PROPVALUE; +impl Record for PROPVALUE { + fn tag() -> u16 { RTAG_PROPVALUE } + fn expected_size() -> Option { None } +} + +pub struct BOX; +impl Record for BOX { + fn tag() -> u16 { RTAG_BOX } + fn expected_size() -> Option { Some(0) } +} + +pub struct BOXTYPE; +impl Record for BOXTYPE { + fn tag() -> u16 { RTAG_BOXTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PLEX; +impl Record for PLEX { + fn tag() -> u16 { RTAG_PLEX } + fn expected_size() -> Option { Some(4) } +} + +pub struct BGNEXTN; +impl Record for BGNEXTN { + fn tag() -> u16 { RTAG_BGNEXTN } + fn expected_size() -> Option { Some(4) } +} + +pub struct ENDEXTN; +impl Record for ENDEXTN { + fn tag() -> u16 { RTAG_ENDEXTN } + fn expected_size() -> Option { Some(4) } +} + +pub struct TAPENUM; +impl Record for TAPENUM { + fn tag() -> u16 { RTAG_TAPENUM } + fn expected_size() -> Option { Some(2) } +} + +pub struct TAPECODE; +impl Record for TAPECODE { + fn tag() -> u16 { RTAG_TAPECODE } + fn expected_size() -> Option { Some(2 * 6) } +} + +pub struct STRCLASS; +impl Record for STRCLASS { + fn tag() -> u16 { RTAG_STRCLASS } + fn expected_size() -> Option { Some(2) } +} + +pub struct RESERVED; +impl Record for RESERVED { + fn tag() -> u16 { RTAG_RESERVED } + fn expected_size() -> Option { Some(2) } +} + +pub struct FORMAT; +impl Record for FORMAT { + fn tag() -> u16 { RTAG_FORMAT } + fn expected_size() -> Option { Some(2) } +} + +pub struct MASK; +impl Record for MASK { + fn tag() -> u16 { RTAG_MASK } + fn expected_size() -> Option { None } +} + +/// End of MASKS records +pub struct ENDMASKS; +impl Record for ENDMASKS { + fn tag() -> u16 { RTAG_ENDMASKS } + fn expected_size() -> Option { Some(0) } +} + +pub struct LIBDIRSIZE; +impl Record for LIBDIRSIZE { + fn tag() -> u16 { RTAG_LIBDIRSIZE } + fn expected_size() -> Option { Some(2) } +} + +pub struct SRFNAME; +impl Record for SRFNAME { + fn tag() -> u16 { RTAG_SRFNAME } + fn expected_size() -> Option { None } +} + +pub struct LIBSECUR; +impl Record for LIBSECUR { + fn tag() -> u16 { RTAG_LIBSECUR } + fn expected_size() -> Option { Some(2) } +} + +pub struct BORDER; +impl Record for BORDER { + fn tag() -> u16 { RTAG_BORDER } + fn expected_size() -> Option { Some(0) } +} + +pub struct SOFTFENCE; +impl Record for SOFTFENCE { + fn tag() -> u16 { RTAG_SOFTFENCE } + fn expected_size() -> Option { Some(0) } +} + +pub struct HARDFENCE; +impl Record for HARDFENCE { + fn tag() -> u16 { RTAG_HARDFENCE } + fn expected_size() -> Option { Some(0) } +} + +pub struct SOFTWIRE; +impl Record for SOFTWIRE { + fn tag() -> u16 { RTAG_SOFTWIRE } + fn expected_size() -> Option { Some(0) } +} + +pub struct HARDWIRE; +impl Record for HARDWIRE { + fn tag() -> u16 { RTAG_HARDWIRE } + fn expected_size() -> Option { Some(0) } +} + +pub struct PATHPORT; +impl Record for PATHPORT { + fn tag() -> u16 { RTAG_PATHPORT } + fn expected_size() -> Option { Some(0) } +} + +pub struct NODEPORT; +impl Record for NODEPORT { + fn tag() -> u16 { RTAG_NODEPORT } + fn expected_size() -> Option { Some(0) } +} + +pub struct USERCONSTRAINT; +impl Record for USERCONSTRAINT { + fn tag() -> u16 { RTAG_USERCONSTRAINT } + fn expected_size() -> Option { Some(0) } +} + +pub struct SPACERERROR; +impl Record for SPACERERROR { + fn tag() -> u16 { RTAG_SPACERERROR } + fn expected_size() -> Option { Some(0) } +} + +pub struct CONTACT; +impl Record for CONTACT { + fn tag() -> u16 { RTAG_CONTACT } + fn expected_size() -> Option { Some(0) } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bgnstr_reads_both_timestamps() { + let mut bytes = vec![0, 28, 0x05, 0x02]; + for value in [126_i16, 11, 14, 12, 30, 0, 125, 10, 13, 9, 15, 1] { + bytes.extend_from_slice(&value.to_be_bytes()); + } + bytes.extend_from_slice(&[0, 4, 0x07, 0]); + + let (remaining, timestamps) = BGNSTR::read(&bytes).unwrap(); + + assert_eq!(timestamps, [[2026, 11, 14, 12, 30, 0], [2025, 10, 13, 9, 15, 1]]); + assert_eq!(remaining, &[0, 4, 0x07, 0]); + } +} diff --git a/src/review.rs b/src/review.rs new file mode 100644 index 0000000..918ddeb --- /dev/null +++ b/src/review.rs @@ -0,0 +1,1492 @@ +//! Deterministic GDS object review without the normalization performed by the +//! Arrow reader. Raw payloads borrow the file; only polygon keys are copied. + +use crate::{basic::decode_real8, record::RecordHeader, records::*}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet}, + fmt::Write, +}; + +mod names; +pub use names::{parse_layer, parse_layer_name, write_layer_names}; +mod selection; +pub use selection::{DisplayFilter, Scope}; +mod changes; +pub use changes::Replacement; +pub mod report; + +/// A comparison limitation with source context independent of its wording. +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub code: &'static str, + pub message: String, + pub offset: Option, + pub cell: Option, + pub layer: Option, + pub target: Option, +} + +impl Diagnostic { + fn at(code: &'static str, offset: usize, message: String) -> Self { + Self { + code, + message: error(offset, message), + offset: Some(offset), + cell: None, + layer: None, + target: None, + } + } +} + +pub const PROFILE: &str = "klamath-object-v2"; +pub const LAYER_SENTINEL: &str = "KLAMATH_LAYER_NAMES_V1"; +pub type Layer = (u16, u16); +pub type Bounds = [i32; 4]; + +#[derive(Clone, Debug)] +pub struct Record<'a> { + pub offset: usize, + pub tag: u16, + pub data: &'a [u8], +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct Key<'a> { + kind: u16, + fields: Vec<(u16, Cow<'a, [u8]>)>, + properties: Vec<(u16, &'a [u8])>, +} + +#[derive(Debug)] +pub struct Object<'a> { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub multiplicity: u64, + key: Key<'a>, + metadata: bool, + issues: Vec, +} + +impl Object<'_> { + fn field(&self, tag: u16) -> Option<&[u8]> { + self.key + .fields + .iter() + .find(|(t, _)| *t == tag) + .map(|(_, v)| v.as_ref()) + } + + pub fn description(&self) -> String { + let mut out = kind_name(self.kind).to_string(); + for (tag, value) in &self.key.fields { + write!(out, " {}={}", tag_name(*tag), format_data(*tag, value)).unwrap(); + } + for (attribute, value) in &self.key.properties { + write!(out, " property[{attribute}]={}", quoted(value)).unwrap(); + } + out + } +} + +#[derive(Debug)] +pub struct Cell<'a> { + pub offset: usize, + /// Start of ENDSTR, for inserting records without rewriting the cell. + pub end_offset: usize, + pub objects: Vec>, + attributes: Vec<(u16, &'a [u8])>, + issues: Vec, +} + +#[derive(Debug)] +pub struct Document<'a> { + pub records: Vec>, + pub cells: BTreeMap>, + pub layer_names: BTreeMap, + /// Unsupported features or invalid reference graphs prevent an equality claim. + pub issues: Vec, + pub diagnostics: Vec, + header: Vec<(u16, &'a [u8])>, + global_issues: Vec, + known_cells: BTreeSet, + pub scope: Scope, + present: bool, +} + +fn error(offset: usize, message: impl std::fmt::Display) -> String { + format!("at byte {offset}: {message}") +} + +fn string(data: &[u8], offset: usize) -> Result { + decoded_string(unpad(data), offset) +} + +fn decoded_string(data: &[u8], offset: usize) -> Result { + if data.contains(&0) { + return Err(error(offset, "name or metadata contains an embedded NUL")); + } + std::str::from_utf8(data) + .map(str::to_owned) + .map_err(|_| error(offset, "name or metadata is not UTF-8")) +} + +fn unpad(data: &[u8]) -> &[u8] { + if data.last() == Some(&0) { + &data[..data.len() - 1] + } else { + data + } +} + +fn integer(data: &[u8]) -> u16 { + u16::from_be_bytes(data.try_into().unwrap()) +} + +fn points(data: &[u8]) -> Vec<[i32; 2]> { + data.as_chunks::<8>() + .0 + .iter() + .map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + }) + .collect() +} + +// Booth's least-rotation algorithm avoids quadratic work on repeated vertices. +fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { + let n = points.len(); + let (mut i, mut j, mut k) = (0, 1, 0); + while i < n && j < n && k < n { + match points[(i + k) % n].cmp(&points[(j + k) % n]) { + std::cmp::Ordering::Equal => k += 1, + std::cmp::Ordering::Greater => { + i += k + 1; + if i <= j { + i = j + 1; + } + k = 0; + } + std::cmp::Ordering::Less => { + j += k + 1; + if j <= i { + j = i + 1; + } + k = 0; + } + } + } + let start = i.min(j); + (0..n).map(|k| points[(start + k) % n]).collect() +} + +fn canonical_ring(data: &[u8]) -> Vec { + let mut ring = points(data); + ring.pop(); // The caller has checked closure. + let forward = least_rotation(&ring); + ring.reverse(); + let backward = least_rotation(&ring); + let mut ring = forward.min(backward); + ring.push(ring[0]); + ring.into_iter() + .flatten() + .flat_map(i32::to_be_bytes) + .collect() +} + +fn validate_payload(record: &Record<'_>) -> Result<(), String> { + let size = record.data.len(); + let valid = match record.tag { + RTAG_BGNLIB | RTAG_BGNSTR => size == 24, + RTAG_UNITS => size == 16, + RTAG_HEADER | RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE + | RTAG_PROPATTR | RTAG_PATHTYPE | RTAG_STRANS | RTAG_PRESENTATION | RTAG_ELFLAGS + | RTAG_STRCLASS | RTAG_GENERATIONS => size == 2, + RTAG_WIDTH | RTAG_BGNEXTN | RTAG_ENDEXTN | RTAG_PLEX | RTAG_COLROW => size == 4, + RTAG_MAG | RTAG_ANGLE => size == 8, + RTAG_XY => size.is_multiple_of(8), + RTAG_ENDLIB | RTAG_ENDSTR | RTAG_ENDEL | RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF + | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX | RTAG_ENDMASKS => size == 0, + _ => true, + }; + if !valid { + return Err(error( + record.offset, + format!("invalid payload size {size} for {}", tag_name(record.tag)), + )); + } + if record.tag == RTAG_UNITS + && record.data.as_chunks::<8>().0.iter().any(|&bytes| { + let value = decode_real8(u64::from_be_bytes(bytes)); + !value.is_finite() || value <= 0.0 + }) + { + return Err(error(record.offset, "UNITS must be positive finite values")); + } + Ok(()) +} + +fn is_element(tag: u16) -> bool { + matches!( + tag, + RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX + ) +} + +fn allowed_field(kind: u16, tag: u16) -> bool { + if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) { + return true; + } + match kind { + RTAG_BOUNDARY => matches!(tag, RTAG_LAYER | RTAG_DATATYPE), + RTAG_BOX => matches!(tag, RTAG_LAYER | RTAG_BOXTYPE), + RTAG_NODE => matches!(tag, RTAG_LAYER | RTAG_NODETYPE), + RTAG_PATH => matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_WIDTH | RTAG_PATHTYPE | RTAG_BGNEXTN | RTAG_ENDEXTN + ), + RTAG_SREF => matches!(tag, RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE), + RTAG_AREF => matches!( + tag, + RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE | RTAG_COLROW + ), + RTAG_TEXT => matches!( + tag, + RTAG_LAYER + | RTAG_TEXTTYPE + | RTAG_STRING + | RTAG_STRANS + | RTAG_MAG + | RTAG_ANGLE + | RTAG_PRESENTATION + | RTAG_PATHTYPE + | RTAG_WIDTH + ), + _ => false, + } +} + +fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result, String> { + let first = &records[0]; + let mut fields = Vec::new(); + let mut properties = Vec::new(); + let mut seen = BTreeSet::new(); + let mut i = 1; + while i + 1 < records.len() { + let record = &records[i]; + if record.tag == RTAG_PROPATTR { + let value = records + .get(i + 1) + .filter(|r| r.tag == RTAG_PROPVALUE) + .ok_or_else(|| error(record.offset, "PROPATTR without PROPVALUE"))?; + properties.push((integer(record.data), unpad(value.data))); + i += 2; + continue; + } + if !allowed_field(first.tag, record.tag) { + issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!( + "unsupported record 0x{:04x} in {}", + record.tag, + kind_name(first.tag) + ), + )); + } + if !seen.insert(record.tag) { + return Err(error(record.offset, "duplicate element field")); + } + let mut data = Cow::Borrowed(record.data); + if record.tag & 0xff == 6 { + data = Cow::Borrowed(unpad(record.data)); + } + if record.tag == RTAG_XY && matches!(first.tag, RTAG_BOUNDARY | RTAG_BOX) { + let p = points(record.data); + if p.len() < 4 || p.first() != p.last() { + return Err(error( + record.offset, + "polygon must have at least three vertices and be closed", + )); + } + data = Cow::Owned(canonical_ring(record.data)); + } + fields.push((record.tag, data)); + i += 1; + } + fields.sort(); + properties.sort(); + let mut object = Object { + offset: first.offset, + end_offset: records.last().unwrap().offset + 4, + kind: first.tag, + layer: None, + bounds: None, + target: None, + multiplicity: 1, + key: Key { + kind: first.tag, + fields, + properties, + }, + metadata: false, + issues: Vec::new(), + }; + let xy = object + .field(RTAG_XY) + .ok_or_else(|| error(first.offset, "element has no XY record"))?; + let p = points(xy); + let valid_count = match first.tag { + RTAG_TEXT | RTAG_SREF => p.len() == 1, + RTAG_AREF => p.len() == 3, + RTAG_PATH => p.len() >= 2, + RTAG_NODE => !p.is_empty(), + RTAG_BOX => p.len() == 5, + _ => true, + }; + if !valid_count { + return Err(error(first.offset, "invalid coordinate count")); + } + object.bounds = Some([ + p.iter().map(|p| p[0]).min().unwrap(), + p.iter().map(|p| p[1]).min().unwrap(), + p.iter().map(|p| p[0]).max().unwrap(), + p.iter().map(|p| p[1]).max().unwrap(), + ]); + if matches!(first.tag, RTAG_SREF | RTAG_AREF) { + object.target = Some(decoded_string( + object + .field(RTAG_SNAME) + .ok_or_else(|| error(first.offset, "reference has no SNAME"))?, + first.offset, + )?); + if first.tag == RTAG_AREF { + let counts = object + .field(RTAG_COLROW) + .ok_or_else(|| error(first.offset, "array has no COLROW"))?; + let a = i16::from_be_bytes(counts[..2].try_into().unwrap()); + let b = i16::from_be_bytes(counts[2..].try_into().unwrap()); + if a <= 0 || b <= 0 { + return Err(error(first.offset, "array counts must be positive")); + } + object.multiplicity = a as u64 * b as u64; + } + } else { + let dtype = match first.tag { + RTAG_TEXT => RTAG_TEXTTYPE, + RTAG_BOX => RTAG_BOXTYPE, + RTAG_NODE => RTAG_NODETYPE, + _ => RTAG_DATATYPE, + }; + object.layer = Some(( + integer( + object + .field(RTAG_LAYER) + .ok_or_else(|| error(first.offset, "element has no LAYER"))?, + ), + integer( + object + .field(dtype) + .ok_or_else(|| error(first.offset, "element has no datatype/texttype"))?, + ), + )); + if first.tag == RTAG_TEXT && object.field(RTAG_STRING).is_none() { + return Err(error(first.offset, "text has no STRING")); + } + } + Ok(object) +} + +fn read_records(input: &[u8]) -> Result>, String> { + let mut records = Vec::new(); + let mut offset = 0; + while offset < input.len() { + let (payload, header) = RecordHeader::read(&input[offset..]) + .map_err(|(_, e)| error(offset, format!("invalid record header: {e:?}")))?; + let size = header.data_size as usize; + if !(size + 4).is_multiple_of(2) { + return Err(error(offset, "odd record length")); + } + let data = payload + .get(..size) + .ok_or_else(|| error(offset, "truncated record"))?; + let record = Record { + offset, + tag: header.tag, + data, + }; + offset += size + 4; + let ended = record.tag == RTAG_ENDLIB; + records.push(record); + if ended { + if input[offset..].iter().any(|b| *b != 0) { + return Err(error(offset, "nonzero data after ENDLIB")); + } + break; + } + } + Ok(records) +} + +impl<'a> Document<'a> { + pub fn parse(input: &'a [u8]) -> Result { + let records = read_records(input)?; + for record in &records { + validate_payload(record)?; + } + if records.first().map(|r| r.tag) != Some(RTAG_HEADER) + || records.get(1).map(|r| r.tag) != Some(RTAG_BGNLIB) + || records.last().map(|r| r.tag) != Some(RTAG_ENDLIB) + { + return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string()); + } + let mut doc = Self { + records, + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + issues: Vec::new(), + diagnostics: Vec::new(), + header: Vec::new(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: Scope::default(), + present: true, + }; + let mut i = 2; + let mut global_seen = BTreeSet::new(); + let mut cells_started = false; + doc.header.push((RTAG_HEADER, doc.records[0].data)); + while i + 1 < doc.records.len() { + let record = &doc.records[i]; + if record.tag != RTAG_BGNSTR { + if cells_started { + return Err(error( + record.offset, + "library field outside the library header", + )); + } + if record.tag != RTAG_MASK && !global_seen.insert(record.tag) { + return Err(error(record.offset, "duplicate library field")); + } + if !matches!( + record.tag, + RTAG_LIBNAME + | RTAG_UNITS + | RTAG_REFLIBS + | RTAG_FONTS + | RTAG_GENERATIONS + | RTAG_ATTRTABLE + | RTAG_FORMAT + | RTAG_MASK + | RTAG_ENDMASKS + | RTAG_LIBDIRSIZE + | RTAG_SRFNAME + | RTAG_LIBSECUR + ) { + doc.global_issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported library record 0x{:04x}", record.tag), + )); + } + doc.header.push(( + record.tag, + if record.tag & 0xff == 6 { + unpad(record.data) + } else { + record.data + }, + )); + i += 1; + continue; + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err(error(record.offset, "LIBNAME and UNITS must precede cells")); + } + cells_started = true; + let offset = record.offset; + i += 1; + let name_record = doc + .records + .get(i) + .filter(|r| r.tag == RTAG_STRNAME) + .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; + let name = string(name_record.data, name_record.offset)?; + if name.is_empty() { + return Err(error(name_record.offset, "empty cell name")); + } + i += 1; + let mut cell = Cell { + offset, + end_offset: 0, + objects: Vec::new(), + attributes: Vec::new(), + issues: Vec::new(), + }; + loop { + let record = doc + .records + .get(i) + .ok_or_else(|| error(offset, "missing ENDSTR"))?; + if record.tag == RTAG_ENDSTR { + cell.end_offset = record.offset; + i += 1; + break; + } + if is_element(record.tag) { + let start = i; + i += 1; + while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { + if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) + || is_element(doc.records[i].tag) + { + return Err(error(doc.records[i].offset, "missing ENDEL")); + } + i += 1; + } + if i == doc.records.len() { + return Err(error(offset, "missing ENDEL")); + } + let mut issues = Vec::new(); + let mut parsed = object(&doc.records[start..=i], &mut issues)?; + parsed.issues = issues; + cell.objects.push(parsed); + i += 1; + } else { + if matches!(record.tag, RTAG_ENDLIB | RTAG_BGNSTR) { + return Err(error(record.offset, "missing ENDSTR")); + } + if record.tag != RTAG_STRCLASS { + cell.issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported cell record 0x{:04x}", record.tag), + )); + } else if !cell.objects.is_empty() + || cell.attributes.iter().any(|(tag, _)| *tag == RTAG_STRCLASS) + { + return Err(error( + record.offset, + "STRCLASS must occur once before elements", + )); + } + cell.attributes.push((record.tag, record.data)); + i += 1; + } + } + cell.attributes.sort(); + cell.objects.sort_by(|a, b| a.key.cmp(&b.key)); + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err("library must contain LIBNAME and UNITS".to_string()); + } + doc.header.sort(); + doc.read_metadata()?; + doc.known_cells = doc.cells.keys().cloned().collect(); + doc.refresh_issues(); + Ok(doc) + } + + fn read_metadata(&mut self) -> Result<(), String> { + for (cell_name, cell) in &mut self.cells { + for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { + let text = object.field(RTAG_STRING).unwrap(); + if text.starts_with(b"KLAMATH_LAYER_NAMES_") && text != LAYER_SENTINEL.as_bytes() { + // Layer names apply across the library. An unknown version + // may describe layers used outside this carrier or cell, so + // geometry scope selection cannot establish its relevance. + let mut diagnostic = Diagnostic::at( + "unsupported_metadata_version", + object.offset, + "unsupported layer-name metadata version".to_string(), + ); + diagnostic.cell = Some(cell_name.clone()); + diagnostic.layer = object.layer; + diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); + self.global_issues.push(diagnostic); + } + } + let carriers: BTreeSet<_> = cell + .objects + .iter() + .filter(|o| { + o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) + }) + .filter_map(|o| o.layer) + .collect(); + for object in &mut cell.objects { + if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) + { + continue; + } + let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; + if text != LAYER_SENTINEL { + let ((layer, dtype), name) = + parse_layer_name(&text).map_err(|message| error(object.offset, message))?; + if self + .layer_names + .get(&(layer, dtype)) + .is_some_and(|old| old != &name) + { + return Err(error( + object.offset, + format!("conflicting names for {layer}/{dtype}"), + )); + } + self.layer_names.insert((layer, dtype), name.to_string()); + } + object.metadata = true; + } + } + Ok(()) + } + + fn check_graph(&mut self) { + let mut unresolved = BTreeMap::new(); + for (parent, cell) in &self.cells { + for object in &cell.objects { + if let Some(target) = &object.target + && !self.known_cells.contains(target) + { + unresolved.entry((parent, target)).or_insert(object.offset); + } + } + } + for ((parent, target), offset) in unresolved { + self.diagnostics.push(Diagnostic { + code: "unresolved_reference", + message: format!("cell {parent:?}: unresolved reference {target:?}"), + offset: Some(offset), + cell: Some(parent.clone()), + layer: None, + target: Some(target.clone()), + }); + } + let mut state = BTreeMap::<&str, u8>::new(); + for root in self.cells.keys() { + let mut stack = vec![(root.as_str(), false)]; + while let Some((name, leaving)) = stack.pop() { + if leaving { + state.insert(name, 2); + continue; + } + match state.get(name) { + Some(2) => continue, + Some(1) => { + self.diagnostics.push(Diagnostic { + code: "hierarchy_cycle", + message: format!("cyclic hierarchy involving {name:?}"), + offset: self.cells.get(name).map(|c| c.offset), + cell: Some(name.to_string()), + layer: None, + target: None, + }); + continue; + } + _ => (), + } + state.insert(name, 1); + stack.push((name, true)); + if let Some(cell) = self.cells.get(name) { + for object in &cell.objects { + if let Some(target) = &object.target { + stack.push((target, false)); + } + } + } + } + } + } + + pub fn edges(&self) -> BTreeMap<(String, String), u64> { + let mut edges = BTreeMap::new(); + for (name, cell) in &self.cells { + for object in &cell.objects { + if let Some(target) = &object.target { + let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64); + *count = count.saturating_add(object.multiplicity); + } + } + } + edges + } + + pub fn canonical_dump(&self) -> String { + let mut out = format!( + "profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n", + self.scope.description() + ); + for (tag, data) in &self.header { + writeln!(out, "{}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + } + for ((layer, dtype), name) in &self.layer_names { + writeln!(out, "layer {layer}/{dtype} name={name:?}").unwrap(); + } + for (name, cell) in &self.cells { + writeln!(out, "cell {name:?}").unwrap(); + for (tag, data) in &cell.attributes { + writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + } + for object in cell.objects.iter().filter(|o| !o.metadata) { + writeln!(out, " {}", object.description()).unwrap(); + } + } + for issue in &self.issues { + writeln!(out, "INCOMPLETE: {issue}").unwrap(); + } + out + } + + pub fn record_dump(&self) -> String { + format_records(&self.records) + } +} + +/// Inspect record boundaries even when object grammar or metadata is invalid. +pub fn dump_records(input: &[u8]) -> Result { + Ok(format_records(&read_records(input)?)) +} + +fn format_records(records: &[Record<'_>]) -> String { + let mut out = String::new(); + for record in records { + writeln!( + out, + "{:010} {:04x} {} size={} {}", + record.offset, + record.tag, + tag_name(record.tag), + record.data.len() + 4, + format_data( + record.tag, + if record.tag & 0xff == 6 { + unpad(record.data) + } else { + record.data + } + ) + ) + .unwrap(); + } + out +} + +/// A retained comparison field. Polygon coordinates and string padding follow +/// the comparison profile; REAL encodings and unsupported payloads are exact. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FieldValue { + pub tag: u16, + pub data: Vec, +} + +fn owned_fields<'a>(fields: impl IntoIterator) -> Vec { + fields + .into_iter() + .map(|(tag, data)| FieldValue { + tag, + data: data.to_vec(), + }) + .collect() +} + +#[derive(Debug)] +pub struct ObjectChange { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub origin: Option<[i32; 2]>, + pub width: Option, + pub counts: Option<[u16; 2]>, + pub description: String, + pub fields: Vec, + pub properties: Vec<(u16, Vec)>, +} + +impl From<&Object<'_>> for ObjectChange { + fn from(object: &Object<'_>) -> Self { + Self { + offset: object.offset, + end_offset: object.end_offset, + kind: object.kind, + layer: object.layer, + bounds: object.bounds, + target: object.target.clone(), + origin: object.field(RTAG_XY).map(|data| { + [ + i32::from_be_bytes(data[..4].try_into().unwrap()), + i32::from_be_bytes(data[4..8].try_into().unwrap()), + ] + }), + width: object + .field(RTAG_WIDTH) + .map(|data| i32::from_be_bytes(data.try_into().unwrap())), + counts: object.target.as_ref().map(|_| { + if object.kind == RTAG_AREF { + let data = object.field(RTAG_COLROW).unwrap(); + [integer(&data[..2]), integer(&data[2..])] + } else { + [1, 1] + } + }), + description: object.description(), + fields: owned_fields( + object + .key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + ), + properties: object + .key + .properties + .iter() + .map(|(a, v)| (*a, v.to_vec())) + .collect(), + } + } +} + +#[derive(Debug)] +pub struct CellChange { + pub name: String, + pub old_offset: Option, + pub new_offset: Option, + pub attributes_changed: bool, + pub old_attributes: Vec, + pub new_attributes: Vec, + pub removed: Vec, + pub added: Vec, + pub replacements: Vec, +} + +#[derive(Debug)] +pub struct LayerNameChange { + pub layer: Layer, + pub old: Option, + pub new: Option, +} + +#[derive(Debug)] +pub struct LibraryFieldChange { + pub field: String, + pub tag: Option, + pub old_fields: Vec, + pub new_fields: Vec, + pub old: Vec, + pub new: Vec, +} + +#[derive(Debug)] +pub struct Comparison { + pub scope: Scope, + /// Unchanged names are displayed once when objects on that layer change. + pub layer_legend: BTreeMap, + pub library_fields: Vec, + pub layer_names: Vec, + pub cells: Vec, + pub issues: Vec, +} + +impl Comparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + let mut result = Self { + scope: old.scope.clone(), + layer_legend: old + .layer_names + .iter() + .filter(|(pair, name)| new.layer_names.get(pair) == Some(*name)) + .map(|(pair, name)| (*pair, name.clone())) + .collect(), + library_fields: Vec::new(), + layer_names: Vec::new(), + cells: Vec::new(), + issues: Vec::new(), + }; + if old.scope != new.scope { + result.issues.push("comparison scopes differ".to_string()); + } + if old.present != new.present { + result.library_fields.push(LibraryFieldChange { + field: "presence".to_string(), + tag: None, + old_fields: Vec::new(), + new_fields: Vec::new(), + old: vec![if old.present { "present" } else { "absent" }.to_string()], + new: vec![if new.present { "present" } else { "absent" }.to_string()], + }); + } + for (side, doc) in [("old", old), ("new", new)] { + result + .issues + .extend(doc.issues.iter().map(|i| format!("{side}: {i}"))); + } + let header_tags: BTreeSet<_> = old + .header + .iter() + .chain(&new.header) + .map(|(tag, _)| *tag) + .collect(); + for tag in header_tags { + let a: Vec<_> = old + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + let b: Vec<_> = new + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + if a != b { + result.library_fields.push(LibraryFieldChange { + field: tag_name(tag), + tag: Some(tag), + old_fields: owned_fields(a.iter().map(|data| (tag, *data))), + new_fields: owned_fields(b.iter().map(|data| (tag, *data))), + old: a.iter().map(|data| format_data(tag, data)).collect(), + new: b.iter().map(|data| format_data(tag, data)).collect(), + }); + } + } + let layers: BTreeSet<_> = old + .layer_names + .keys() + .chain(new.layer_names.keys()) + .copied() + .collect(); + for layer in layers { + if old.layer_names.get(&layer) != new.layer_names.get(&layer) { + result.layer_names.push(LayerNameChange { + layer, + old: old.layer_names.get(&layer).cloned(), + new: new.layer_names.get(&layer).cloned(), + }); + } + } + let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect(); + for name in names { + let a = old.cells.get(name); + let b = new.cells.get(name); + let mut change = CellChange { + name: name.clone(), + old_offset: a.map(|c| c.offset), + new_offset: b.map(|c| c.offset), + attributes_changed: a.map(|c| &c.attributes) != b.map(|c| &c.attributes), + old_attributes: owned_fields( + a.into_iter().flat_map(|c| c.attributes.iter().copied()), + ), + new_attributes: owned_fields( + b.into_iter().flat_map(|c| c.attributes.iter().copied()), + ), + removed: Vec::new(), + added: Vec::new(), + replacements: Vec::new(), + }; + let a: Vec<_> = a + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let b: Vec<_> = b + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let (mut i, mut j) = (0, 0); + while i < a.len() || j < b.len() { + let order = match (a.get(i), b.get(j)) { + (Some(a), Some(b)) => a.key.cmp(&b.key), + (Some(_), None) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Greater, + }; + match order { + std::cmp::Ordering::Equal => { + i += 1; + j += 1; + } + std::cmp::Ordering::Less => { + change.removed.push(a[i].into()); + i += 1; + } + std::cmp::Ordering::Greater => { + change.added.push(b[j].into()); + j += 1; + } + } + } + if change.old_offset.is_none() + || change.new_offset.is_none() + || change.attributes_changed + || !change.removed.is_empty() + || !change.added.is_empty() + { + change.replacements = changes::replacements(&change.removed, &change.added); + result.cells.push(change); + } + } + result + } + + pub fn is_equal(&self) -> bool { + self.issues.is_empty() + && self.library_fields.is_empty() + && self.layer_names.is_empty() + && self.cells.is_empty() + } + + pub fn exit_code(&self) -> u8 { + if !self.issues.is_empty() { + 2 + } else if self.is_equal() { + 0 + } else { + 1 + } + } + + /// Limit presentation only: the complete comparison has already been made. + pub fn text(&self, max_objects: usize) -> String { + self.text_filtered(max_objects, &DisplayFilter::default()) + } + + pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { + let mut out = format!("profile {PROFILE}\n{}\n", self.scope.description()); + if show.cell.is_some() || show.layer.is_some() { + writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap(); + } + for change in &self.library_fields { + writeln!( + out, + "library {}: {:?} -> {:?}", + change.field, change.old, change.new + ) + .unwrap(); + } + if self.library_fields.iter().any(|c| c.field == "UNITS") { + out.push_str("UNITS changed; coordinates remain in each file's database units\n"); + } + for change in &self.layer_names { + writeln!( + out, + "layer {}/{} name: {:?} -> {:?}", + change.layer.0, change.layer.1, change.old, change.new + ) + .unwrap(); + } + let changed_layers: BTreeSet<_> = self + .cells + .iter() + .filter(|cell| show.cell.as_ref().is_none_or(|name| name == &cell.name)) + .flat_map(|cell| cell.removed.iter().chain(&cell.added)) + .filter_map(|object| object.layer) + .filter(|layer| show.layer.is_none_or(|selected| selected == *layer)) + .collect(); + for layer in changed_layers { + if let Some(name) = self.layer_legend.get(&layer) { + writeln!(out, "layer {}/{} name={name:?}", layer.0, layer.1).unwrap(); + } + } + let total: usize = self + .cells + .iter() + .map(|c| c.removed.len() + c.added.len()) + .sum(); + let mut shown: usize = 0; + for cell in &self.cells { + if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + continue; + } + writeln!( + out, + "cell {:?} old@{:?} new@{:?}{}", + cell.name, + cell.old_offset, + cell.new_offset, + if cell.attributes_changed { + " [cell/header changed]" + } else { + "" + } + ) + .unwrap(); + let mut paired_old = BTreeSet::new(); + let mut paired_new = BTreeSet::new(); + for replacement in &cell.replacements { + let a = &cell.removed[replacement.removed]; + let b = &cell.added[replacement.added]; + if shown.saturating_add(2) <= max_objects + && show + .layer + .is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer)) + { + writeln!(out, " ~ {}", replacement.explanation).unwrap(); + writeln!( + out, + " - @{} bbox={:?} {}", + a.offset, a.bounds, a.description + ) + .unwrap(); + writeln!( + out, + " + @{} bbox={:?} {}", + b.offset, b.bounds, b.description + ) + .unwrap(); + paired_old.insert(replacement.removed); + paired_new.insert(replacement.added); + shown += 2; + } + } + for (sign, objects, paired) in [ + ("-", &cell.removed, &paired_old), + ("+", &cell.added, &paired_new), + ] { + for (index, object) in objects.iter().enumerate() { + if paired.contains(&index) { + continue; + } + if show.layer.is_some_and(|layer| object.layer != Some(layer)) { + continue; + } + if shown < max_objects { + writeln!( + out, + " {sign} @{} bbox={:?} {}", + object.offset, object.bounds, object.description + ) + .unwrap(); + shown += 1; + } + } + } + } + writeln!( + out, + "{} changed cells; {shown} of {total} object changes shown; {} layer-name changes", + self.cells.len(), + self.layer_names.len() + ) + .unwrap(); + for issue in &self.issues { + writeln!(out, "INCOMPLETE: {issue}").unwrap(); + } + if self.is_equal() { + out.push_str("equal under this profile\n"); + } + out + } +} + +/// The graph joins both revisions, retaining removed branches and old callers. +pub fn hierarchy( + old: &Document<'_>, + new: &Document<'_>, + diff: Option<&Comparison>, + mermaid: bool, + max_depth: usize, +) -> String { + let old_edges = old.edges(); + let new_edges = new.edges(); + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = diff + .into_iter() + .flat_map(|d| d.cells.iter().map(|c| c.name.clone())) + .collect(); + let changed_refs: BTreeSet<_> = diff + .into_iter() + .flat_map(|d| &d.cells) + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + let visible: BTreeSet<_> = if diff.is_some() { + affected + .iter() + .cloned() + .chain(changed_refs.iter().map(|(_, t)| t.clone())) + .collect() + } else { + old.cells + .keys() + .chain(new.cells.keys()) + .cloned() + .chain(edges.iter().map(|(_, t)| t.clone())) + .collect() + }; + let mut children = BTreeMap::>::new(); + let mut targets = BTreeSet::new(); + for (parent, target) in &edges { + if visible.contains(parent) && visible.contains(target) { + children + .entry(parent.clone()) + .or_default() + .push(target.clone()); + targets.insert(target.clone()); + } + } + let roots: Vec<_> = visible.difference(&targets).cloned().collect(); + // Distinguish rootless cycles from descendants hidden by the depth limit. + let mut reachable = BTreeSet::new(); + let mut pending = roots.clone(); + while let Some(name) = pending.pop() { + if reachable.insert(name.clone()) { + pending.extend(children.get(&name).into_iter().flatten().cloned()); + } + } + let mut included = BTreeSet::new(); + let mut rows = Vec::new(); + let mut stack: Vec<_> = roots + .iter() + .rev() + .map(|n| (n.clone(), 0, None::)) + .collect(); + while let Some((name, depth, parent)) = stack.pop() { + let seen = !included.insert(name.clone()); + let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); + rows.push(( + name.clone(), + depth, + parent, + seen, + depth >= max_depth && !descendants.is_empty(), + )); + if !seen && depth < max_depth { + for child in descendants.iter().rev() { + stack.push((child.clone(), depth + 1, Some(name.clone()))); + } + } + } + for name in visible.difference(&reachable) { + rows.push((name.clone(), 0, None, false, true)); + } + let status = |name: &String| { + if !old.cells.contains_key(name) && !new.cells.contains_key(name) { + if old.known_cells.contains(name) || new.known_cells.contains(name) { + " [outside scope]" + } else { + " [unresolved]" + } + } else if direct.contains(name) { + " [cell changed]" + } else if affected.contains(name) { + " [affected]" + } else if diff.is_some() { + " [cell unchanged]" + } else { + "" + } + }; + let edge_label = |parent: &String, target: &String| { + let key = (parent.clone(), target.clone()); + let a = old_edges.get(&key).copied().unwrap_or(0); + let b = new_edges.get(&key).copied().unwrap_or(0); + let count = if a == b { + format!("x{b}") + } else { + format!("x{a} -> x{b}") + }; + format!( + "{count}{}", + if changed_refs.contains(&key) { + " [reference changed]" + } else { + "" + } + ) + }; + if mermaid { + let ids: BTreeMap<_, _> = rows + .iter() + .map(|r| &r.0) + .collect::>() + .into_iter() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + let mut out = format!("flowchart TD\n %% {}\n", new.scope.description()); + for (name, id) in &ids { + let label = format!("{name}{}", status(name)); + let label = label + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") + .replace(['\r', '\n'], " "); + writeln!(out, " n{id}[\"{label}\"]").unwrap(); + } + for (parent, target) in &edges { + if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { + writeln!(out, " n{a} -->|\"{}\"| n{b}", edge_label(parent, target)).unwrap(); + } + } + if rows.is_empty() { + out.push_str(" empty[\"No changed hierarchy\"]\n"); + } + out + } else { + let mut out = format!("{}\n", new.scope.description()); + for (name, depth, parent, seen, limited) in rows { + let count = parent + .map(|p| edge_label(&p, &name)) + .unwrap_or_else(|| "x1".to_string()); + writeln!( + out, + "{}{name:?} {count}{}{}{}", + " ".repeat(depth), + status(&name), + if seen { " [shown above]" } else { "" }, + if limited { " [children omitted]" } else { "" } + ) + .unwrap(); + } + if visible.is_empty() { + out.push_str("No changed hierarchy\n"); + } + out + } +} + +fn quoted(data: &[u8]) -> String { + let mut out = "\"".to_string(); + for byte in data { + for escaped in std::ascii::escape_default(*byte) { + out.push(escaped as char); + } + } + out.push('"'); + out +} + +fn format_data(tag: u16, data: &[u8]) -> String { + if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_BOXTYPE | RTAG_NODETYPE + ) && data.len() == 2 + { + return integer(data).to_string(); + } + match tag & 0xff { + 0 if data.is_empty() => String::new(), + 2 if data.len().is_multiple_of(2) => format!( + "{:?}", + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + 3 if data.len().is_multiple_of(4) => format!( + "{:?}", + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + 5 if data.len().is_multiple_of(8) => format!( + "{:?} raw={}", + data.as_chunks::<8>() + .0 + .iter() + .map(|&b| decode_real8(u64::from_be_bytes(b))) + .collect::>(), + hex(data) + ), + 6 => quoted(data), + _ => hex(data), + } +} + +fn hex(data: &[u8]) -> String { + let mut out = "0x".to_string(); + for byte in data { + write!(out, "{byte:02x}").unwrap(); + } + out +} + +fn kind_name(tag: u16) -> &'static str { + match tag { + RTAG_BOUNDARY => "BOUNDARY", + RTAG_PATH => "PATH", + RTAG_SREF => "SREF", + RTAG_AREF => "AREF", + RTAG_TEXT => "TEXT", + RTAG_NODE => "NODE", + RTAG_BOX => "BOX", + _ => "UNKNOWN", + } +} + +fn tag_name(tag: u16) -> String { + let name = match tag { + RTAG_HEADER => "HEADER", + RTAG_BGNLIB => "BGNLIB", + RTAG_LIBNAME => "LIBNAME", + RTAG_UNITS => "UNITS", + RTAG_ENDLIB => "ENDLIB", + RTAG_BGNSTR => "BGNSTR", + RTAG_STRNAME => "STRNAME", + RTAG_ENDSTR => "ENDSTR", + RTAG_ENDEL => "ENDEL", + RTAG_LAYER => "layer", + RTAG_DATATYPE => "datatype", + RTAG_TEXTTYPE => "texttype", + RTAG_XY => "xy", + RTAG_SNAME => "target", + RTAG_COLROW => "counts", + RTAG_STRANS => "strans", + RTAG_MAG => "mag", + RTAG_ANGLE => "angle", + RTAG_WIDTH => "width", + RTAG_PATHTYPE => "pathtype", + RTAG_STRING => "string", + RTAG_PROPATTR => "PROPATTR", + RTAG_PROPVALUE => "PROPVALUE", + RTAG_PRESENTATION => "presentation", + RTAG_BGNEXTN => "bgnextn", + RTAG_ENDEXTN => "endextn", + RTAG_BOXTYPE => "boxtype", + RTAG_NODETYPE => "nodetype", + RTAG_ELFLAGS => "elflags", + RTAG_PLEX => "plex", + RTAG_STRCLASS => "STRCLASS", + tag if is_element(tag) => kind_name(tag), + _ => return format!("record_{tag:04x}"), + }; + name.to_string() +} diff --git a/src/review/changes.rs b/src/review/changes.rs new file mode 100644 index 0000000..d4c2f9a --- /dev/null +++ b/src/review/changes.rs @@ -0,0 +1,127 @@ +use super::*; + +#[derive(Debug)] +pub struct Replacement { + pub removed: usize, + pub added: usize, + pub explanation: String, + /// Explanation matching is never object identity or geometric equivalence. + pub match_rule: &'static str, +} + +type Group = (u16, Option, Option, Option); + +fn group(object: &ObjectChange, bounds: bool) -> Group { + ( + object.kind, + object.layer, + object.target.clone(), + if bounds && object.target.is_none() { + object.bounds + } else { + None + }, + ) +} + +/// Pair only unique candidates in the remaining multisets. Ambiguous groups +/// remain additions/removals; no identity or geometric equivalence is inferred. +pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new()); + let mut result = Vec::new(); + for bounds in [true, false] { + let (mut a, mut b) = ( + BTreeMap::>::new(), + BTreeMap::>::new(), + ); + for (i, object) in old + .iter() + .enumerate() + .filter(|(i, _)| !used_old.contains(i)) + { + a.entry(group(object, bounds)).or_default().push(i); + } + for (i, object) in new + .iter() + .enumerate() + .filter(|(i, _)| !used_new.contains(i)) + { + b.entry(group(object, bounds)).or_default().push(i); + } + for (key, indices) in a { + if let (1, Some(others)) = (indices.len(), b.get(&key)) + && others.len() == 1 + { + let (i, j) = (indices[0], others[0]); + result.push(Replacement { + removed: i, + added: j, + explanation: explain(&old[i], &new[j]), + match_rule: if old[i].target.is_some() { + "unique_kind_target" + } else if bounds { + "unique_kind_layer_bounds" + } else { + "unique_kind_layer" + }, + }); + used_old.insert(i); + used_new.insert(j); + } + } + } + result.sort_by_key(|r| (r.removed, r.added)); + result +} + +fn explain(old: &ObjectChange, new: &ObjectChange) -> String { + let mut parts = vec![format!( + "{} modified{}", + kind_name(old.kind), + old.target + .as_ref() + .map(|t| format!(" target={t:?}")) + .unwrap_or_default() + )]; + if old.target.is_some() + && old.origin != new.origin + && let (Some(a), Some(b)) = (old.origin, new.origin) + { + parts.push(format!( + "origin {a:?} -> {b:?}, delta=({}, {}) dbu", + i64::from(b[0]) - i64::from(a[0]), + i64::from(b[1]) - i64::from(a[1]) + )); + } + if old.counts != new.counts { + let counts = |c: Option<[u16; 2]>| { + c.map(|[a, b]| format!("{a} x {b}")) + .unwrap_or_else(|| "absent".to_string()) + }; + parts.push(format!( + "columns/rows {} -> {}", + counts(old.counts), + counts(new.counts) + )); + } + if old.width != new.width { + let width = |n: Option| { + n.map(|n| n.to_string()) + .unwrap_or_else(|| "absent".to_string()) + }; + parts.push(format!( + "width {} -> {} dbu", + width(old.width), + width(new.width) + )); + } + if old.bounds == new.bounds { + parts.push("coordinate bounds unchanged".to_string()); + } else { + parts.push(format!( + "coordinate bounds {:?} -> {:?}", + old.bounds, new.bounds + )); + } + parts.join("; ") +} diff --git a/src/review/names.rs b/src/review/names.rs new file mode 100644 index 0000000..a3040d5 --- /dev/null +++ b/src/review/names.rs @@ -0,0 +1,120 @@ +use super::*; + +pub fn parse_layer(value: &str) -> Result { + let (a, b) = value + .split_once('/') + .ok_or("expected layer/type, for example 900/7")?; + Ok((decimal(a)?, decimal(b)?)) +} + +fn decimal(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err("layer numbers must be unsigned decimal integers".to_string()); + } + value + .parse() + .map_err(|_| "layer number exceeds 65535".to_string()) +} + +pub fn parse_layer_name(value: &str) -> Result<(Layer, String), String> { + let mut parts = value.splitn(3, ':'); + let a = parts.next().ok_or("expected layer:datatype:name")?; + let b = parts.next().ok_or("expected layer:datatype:name")?; + let name = parts + .next() + .filter(|s| !s.is_empty() && !s.contains('\0')) + .ok_or("layer name must be nonempty and contain no NUL")?; + Ok(((decimal(a)?, decimal(b)?), name.to_string())) +} + +fn record(output: &mut Vec, tag: u16, payload: &[u8]) -> Result<(), String> { + let padding = usize::from(tag & 0xff == 6 && !payload.len().is_multiple_of(2)); + let size = payload + .len() + .checked_add(padding + 4) + .and_then(|n| u16::try_from(n).ok()) + .ok_or("metadata label exceeds GDS record size")?; + output.extend_from_slice(&size.to_be_bytes()); + output.extend_from_slice(&tag.to_be_bytes()); + output.extend_from_slice(payload); + if padding != 0 { + output.push(0); + } + Ok(()) +} + +fn label(output: &mut Vec, carrier: Layer, text: &str) -> Result<(), String> { + record(output, RTAG_TEXT, &[])?; + record(output, RTAG_LAYER, &carrier.0.to_be_bytes())?; + record(output, RTAG_TEXTTYPE, &carrier.1.to_be_bytes())?; + record(output, RTAG_XY, &[0; 8])?; + record(output, RTAG_STRING, text.as_bytes())?; + record(output, RTAG_ENDEL, &[]) +} + +/// Consolidate recognized metadata into an explicitly chosen existing cell and +/// carrier. Other records, including timestamps and unknown records, are copied +/// byte-for-byte. Updates override existing names; the input is never modified. +pub fn write_layer_names( + input: &[u8], + cell_name: &str, + carrier: Layer, + updates: &BTreeMap, +) -> Result, String> { + let doc = Document::parse(input)?; + if doc + .diagnostics + .iter() + .any(|i| i.code == "unsupported_metadata_version") + { + return Err("cannot rewrite an unsupported layer-name metadata version".to_string()); + } + let cell = doc + .cells + .get(cell_name) + .ok_or_else(|| format!("cell {cell_name:?} does not exist"))?; + if cell + .objects + .iter() + .any(|o| o.layer == Some(carrier) && !o.metadata) + { + return Err( + "carrier pair already contains ordinary design objects in the selected cell" + .to_string(), + ); + } + let mut names = doc.layer_names.clone(); + for (&pair, name) in updates { + if name.is_empty() || name.contains('\0') { + return Err("layer names must be nonempty and contain no NUL".to_string()); + } + names.insert(pair, name.clone()); + } + let mut legend = Vec::new(); + label(&mut legend, carrier, LAYER_SENTINEL)?; + for ((layer, dtype), name) in names { + label(&mut legend, carrier, &format!("{layer}:{dtype}:{name}"))?; + } + let mut edits: Vec<_> = doc + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| o.metadata) + .map(|o| (o.offset, o.end_offset, None)) + .collect(); + edits.push((cell.end_offset, cell.end_offset, Some(legend.as_slice()))); + edits.sort_by_key(|(start, end, _)| (*start, *end)); + let mut result = Vec::with_capacity(input.len() + legend.len()); + let mut cursor = 0; + for (start, end, replacement) in edits { + result.extend_from_slice(&input[cursor..start]); + if let Some(bytes) = replacement { + result.extend_from_slice(bytes); + } + cursor = end; + } + result.extend_from_slice(&input[cursor..]); + // Validate the final declaration set before the CLI creates an output file. + Document::parse(&result)?; + Ok(result) +} diff --git a/src/review/report.rs b/src/review/report.rs new file mode 100644 index 0000000..7c0c445 --- /dev/null +++ b/src/review/report.rs @@ -0,0 +1,407 @@ +//! Versioned machine-readable diff reports. No consumer needs to parse prose. +use super::*; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; + +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, Default)] +pub struct ReportInputs { + pub old: Option, + pub new: Option, +} + +/// Limits affect evidence only. The default includes every changed object. +#[derive(Clone, Debug, Default)] +pub struct ReportOptions { + pub filter: DisplayFilter, + pub max_objects: Option, + pub hierarchy: bool, +} + +fn layer(pair: Layer) -> Value { + json!({"number": pair.0, "purpose": pair.1}) +} + +fn scope(value: &Scope) -> Value { + json!({"top": value.top, "cell": value.cell, "layer": value.layer.map(layer)}) +} + +fn input(path: Option<&Path>, doc: Option<&Document<'_>>) -> Value { + let units = doc + .filter(|d| d.present) + .and_then(|d| d.header.iter().find(|(tag, _)| *tag == RTAG_UNITS)) + .map(|(tag, data)| field(*tag, data)); + json!({ + "path": path.and_then(Path::to_str), + "path_encoded_hex": path.filter(|p| p.to_str().is_none()) + .map(|p| hex(p.as_os_str().as_encoded_bytes())), + "present": doc.map(|d| d.present), + "units": units + }) +} + +fn envelope(inputs: &ReportInputs, selection: &Scope, options: &ReportOptions) -> Value { + json!({ + "schema_version": SCHEMA_VERSION, + "kind": "diff", + "producer": {"name": "pheon", "version": env!("CARGO_PKG_VERSION")}, + "comparison_profile": PROFILE, + "inputs": {"old": input(inputs.old.as_deref(), None), "new": input(inputs.new.as_deref(), None)}, + "scope": scope(selection), + "coordinates": {"unit": "database_unit", "frame": "cell_local", "bounds": "coordinate_envelope"}, + "status": "failed", + "result": "unknown", + "exit_code": 2, + "summary": null, + "report_selection": { + "cell": options.filter.cell, + "layer": options.filter.layer.map(layer), + "max_objects": options.max_objects.map(|n| n.to_string()), + "truncated": false, + "filtered_cells": "0", "filtered_objects": "0", "limited_objects": "0", "emitted_objects": "0" + }, + "layer_legend": [], + "library_changes": [], "layer_name_changes": [], "cell_changes": [], + "hierarchy": null, + "diagnostics": [] + }) +} + +/// Input/selection failures still produce a valid report. Unknown locations are +/// null rather than inferred by parsing an error's human-readable message. +pub fn failure_report( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + code: &str, + message: &str, + side: Option<&str>, +) -> Value { + let mut report = envelope(inputs, selection, options); + report["diagnostics"] = json!([{ + "code": code, "severity": "error", "message": message, + "side": side, "byte_offset": null, "cell": null, "layer": null, "target": null + }]); + report +} + +fn field(tag: u16, data: &[u8]) -> Value { + let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { + ("xy", json!(points(data))) + } else if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE + ) && data.len().is_multiple_of(2) + { + ( + "uint16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ) + } else { + match tag & 0xff { + 0 if data.is_empty() => ("none", Value::Null), + 1 if data.len().is_multiple_of(2) => ( + "bits", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ), + 2 if data.len().is_multiple_of(2) => ( + "int16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + ), + 3 if data.len().is_multiple_of(4) => ( + "int32", + json!( + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + ), + 5 if data.len().is_multiple_of(8) => ( + "real8", + json!( + data.as_chunks::<8>() + .0 + .iter() + .map(|&v| decode_real8(u64::from_be_bytes(v))) + .collect::>() + ), + ), + 6 => ("ascii", json!(std::str::from_utf8(data).ok())), + _ => ("bytes", Value::Null), + } + }; + json!({"tag": tag, "name": tag_name(tag), "encoding": encoding, "value": value, "payload_hex": hex(data)}) +} + +fn fields(values: &[FieldValue]) -> Vec { + values.iter().map(|f| field(f.tag, &f.data)).collect() +} + +fn object(value: &ObjectChange, id: &str, side: &str) -> Value { + json!({ + "id": id, "kind": kind_name(value.kind), "tag": value.kind, + "source": {"side": side, "byte_offset": value.offset.to_string(), "end_offset": value.end_offset.to_string()}, + "layer": value.layer.map(layer), "bounds": value.bounds, "target": value.target, + "origin": value.origin.filter(|_| matches!(value.kind, RTAG_SREF | RTAG_AREF | RTAG_TEXT)), "width": value.width, "counts": value.counts, + "fields": fields(&value.fields), + "properties": value.properties.iter().map(|(attribute, bytes)| json!({ + "attribute": attribute, "value": std::str::from_utf8(bytes).ok(), "payload_hex": hex(bytes) + })).collect::>() + }) +} + +fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { + let a: BTreeMap<_, _> = old.fields.iter().map(|f| (f.tag, &f.data)).collect(); + let b: BTreeMap<_, _> = new.fields.iter().map(|f| (f.tag, &f.data)).collect(); + a.keys() + .chain(b.keys()) + .copied() + .collect::>() + .into_iter() + .filter(|tag| a.get(tag) != b.get(tag)) + .map(|tag| { + json!({ + "tag": tag, + "before": a.get(&tag).map(|data| field(tag, data)), + "after": b.get(&tag).map(|data| field(tag, data)) + }) + }) + .collect() +} + +impl Comparison { + /// Render evidence from the same documents used to construct this comparison. + /// IDs are local to this report, not persistent object identities. + pub fn json_report( + &self, + old: &Document<'_>, + new: &Document<'_>, + inputs: &ReportInputs, + options: &ReportOptions, + ) -> Value { + let mut report = envelope(inputs, &self.scope, options); + report["inputs"] = json!({"old": input(inputs.old.as_deref(), Some(old)), "new": input(inputs.new.as_deref(), Some(new))}); + let different = !self.cells.is_empty() + || !self.library_fields.is_empty() + || !self.layer_names.is_empty(); + report["status"] = json!(if self.issues.is_empty() { + "complete" + } else { + "incomplete" + }); + report["result"] = json!(if different { + "different" + } else if self.is_equal() { + "equal" + } else { + "unknown" + }); + report["exit_code"] = json!(self.exit_code()); + report["summary"] = json!({ + "counts": if self.issues.is_empty() { "exact" } else { "observed" }, + "changed_cells": self.cells.len().to_string(), + "removed_objects": self.cells.iter().map(|c| c.removed.len()).sum::().to_string(), + "added_objects": self.cells.iter().map(|c| c.added.len()).sum::().to_string(), + "layer_name_changes": self.layer_names.len().to_string(), + "library_field_changes": self.library_fields.len().to_string() + }); + report["layer_legend"] = json!( + self.layer_legend + .iter() + .map(|(pair, name)| json!({"layer": layer(*pair), "name": name})) + .collect::>() + ); + report["layer_name_changes"] = json!( + self.layer_names + .iter() + .map(|c| json!({ + "layer": layer(c.layer), "before": c.old, "after": c.new + })) + .collect::>() + ); + report["library_changes"] = json!(self.library_fields.iter().map(|c| { + if c.tag.is_none() { + json!({"field": "presence", "tag": null, "before": old.present, "after": new.present}) + } else { + json!({"field": c.field, "tag": c.tag, "before": fields(&c.old_fields), "after": fields(&c.new_fields)}) + } + }).collect::>()); + let (mut emitted, mut filtered_objects, mut limited, mut filtered_cells) = + (0_usize, 0_usize, 0_usize, 0_usize); + let mut cells = Vec::new(); + for (cell_index, cell) in self.cells.iter().enumerate() { + if options + .filter + .cell + .as_ref() + .is_some_and(|name| name != &cell.name) + { + filtered_cells += 1; + filtered_objects += cell.removed.len() + cell.added.len(); + continue; + } + let (mut removed, mut added) = (Vec::new(), Vec::new()); + let mut shown = BTreeSet::new(); + for (side, objects, output) in [ + ("old", &cell.removed, &mut removed), + ("new", &cell.added, &mut added), + ] { + for (index, change) in objects.iter().enumerate() { + if options + .filter + .layer + .is_some_and(|l| change.layer != Some(l)) + { + filtered_objects += 1; + } else if options.max_objects.is_some_and(|max| emitted >= max) { + limited += 1; + } else { + let id = format!("c{cell_index}:{side}:{index}"); + output.push(object(change, &id, side)); + shown.insert(id); + emitted += 1; + } + } + } + let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| { + let a = format!("c{cell_index}:old:{}", r.removed); + let b = format!("c{cell_index}:new:{}", r.added); + (shown.contains(&a) && shown.contains(&b)).then(|| json!({ + "removed": a, "added": b, "match_rule": r.match_rule, + "explanation": r.explanation, + "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), + "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties + })) + }).collect(); + cells.push(json!({ + "name": cell.name, + "before": cell.old_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.old_attributes)})), + "after": cell.new_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.new_attributes)})), + "attributes_changed": cell.attributes_changed, + "removed": removed, "added": added, "replacements": replacements + })); + } + report["cell_changes"] = json!(cells); + report["report_selection"]["truncated"] = + json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0); + for (key, count) in [ + ("filtered_cells", filtered_cells), + ("filtered_objects", filtered_objects), + ("limited_objects", limited), + ("emitted_objects", emitted), + ] { + report["report_selection"][key] = json!(count.to_string()); + } + let mut diagnostics: Vec<_> = [("old", old), ("new", new)] + .into_iter() + .flat_map(|(side, doc)| { + doc.diagnostics.iter().map(move |d| { + json!({ + "code": d.code, "severity": "error", "message": d.message, "side": side, + "byte_offset": d.offset.map(|n| n.to_string()), "cell": d.cell, + "layer": d.layer.map(layer), "target": d.target + }) + }) + }) + .collect(); + if old.scope != new.scope { + diagnostics.push(json!({"code": "scope_mismatch", "severity": "error", "message": "comparison scopes differ", "side": null, "byte_offset": null, "cell": null, "layer": null, "target": null})); + } + report["diagnostics"] = json!(diagnostics); + if options.hierarchy { + report["hierarchy"] = graph(old, new, self); + } + report + } +} + +fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { + let a = old.edges(); + let b = new.edges(); + let edges: BTreeSet<_> = a.keys().chain(b.keys()).cloned().collect(); + let names: BTreeSet<_> = old + .cells + .keys() + .chain(new.cells.keys()) + .cloned() + .chain( + edges + .iter() + .flat_map(|(parent, child)| [parent.clone(), child.clone()]), + ) + .collect(); + let direct: BTreeSet<_> = diff.cells.iter().map(|c| c.name.clone()).collect(); + let mut affected = direct.clone(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + let changed_refs: BTreeSet<_> = diff + .cells + .iter() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let a_targets: BTreeSet<_> = a.keys().map(|(_, t)| t.as_str()).collect(); + let b_targets: BTreeSet<_> = b.keys().map(|(_, t)| t.as_str()).collect(); + let state = |doc: &Document<'_>, targets: &BTreeSet<&str>, name: &String| { + if doc.cells.contains_key(name) { + "included" + } else if doc.known_cells.contains(name) { + "outside_scope" + } else if targets.contains(name.as_str()) { + "unresolved" + } else { + "absent" + } + }; + json!({ + "nodes": names.iter().map(|name| json!({ + "cell": name, "before": state(old, &a_targets, name), "after": state(new, &b_targets, name), + "change": if direct.contains(name) { "direct" } else if affected.contains(name) { "affected" } else { "unchanged" } + })).collect::>(), + "edges": edges.iter().map(|edge| json!({ + "parent": edge.0, "target": edge.1, + "before_count": a.get(edge).copied().unwrap_or(0).to_string(), + "after_count": b.get(edge).copied().unwrap_or(0).to_string(), + "reference_changed": changed_refs.contains(edge) + })).collect::>() + }) +} diff --git a/src/review/selection.rs b/src/review/selection.rs new file mode 100644 index 0000000..b87fbe6 --- /dev/null +++ b/src/review/selection.rs @@ -0,0 +1,233 @@ +use super::*; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Scope { + pub top: Option, + pub cell: Option, + pub layer: Option, +} + +/// Presentation filters never modify comparison status, totals, or hierarchy. +#[derive(Clone, Debug, Default)] +pub struct DisplayFilter { + pub cell: Option, + pub layer: Option, +} + +impl Scope { + pub fn description(&self) -> String { + let top = self + .top + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let cell = self + .cell + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let layer = self + .layer + .map(|(a, b)| format!("{a}/{b}")) + .unwrap_or_else(|| "all".to_string()); + format!("scope top={top} cell={cell} layer={layer}") + } + + pub fn validate_pair(&self, old: &Document<'_>, new: &Document<'_>) -> Result<(), String> { + for (kind, name) in [("top", &self.top), ("cell", &self.cell)] { + if let Some(name) = name + && !old.cells.contains_key(name) + && !new.cells.contains_key(name) + { + return Err(format!( + "selected {kind} {name:?} exists in neither revision" + )); + } + } + if let (Some(top), Some(cell)) = (&self.top, &self.cell) + && !old.reachable(top).contains(cell) + && !new.reachable(top).contains(cell) + { + return Err(format!( + "selected cell {cell:?} is not reachable from {top:?}" + )); + } + Ok(()) + } +} + +impl<'a> Document<'a> { + /// A missing Git revision has no cells or declarations. Retain the other + /// revision's library header to avoid inventing a unit/name change. + pub fn empty_like(&self) -> Self { + Self { + records: Vec::new(), + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + issues: Vec::new(), + diagnostics: Vec::new(), + header: self.header.clone(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: self.scope.clone(), + present: false, + } + } + fn reachable(&self, root: &str) -> BTreeSet { + let mut found = BTreeSet::new(); + let mut pending = vec![root.to_string()]; + while let Some(name) = pending.pop() { + if let Some(cell) = self.cells.get(&name) + && found.insert(name) + { + pending.extend(cell.objects.iter().filter_map(|o| o.target.clone())); + } + } + found + } + + pub(super) fn refresh_issues(&mut self) { + self.diagnostics = self.global_issues.clone(); + for (name, cell) in &self.cells { + for issue in &cell.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + for object in &cell.objects { + for issue in &object.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.layer = object.layer; + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + } + } + self.check_graph(); + self.issues = self.diagnostics.iter().map(|d| d.message.clone()).collect(); + } + + /// Project a parsed document. Missing selectors are allowed only for one + /// side of a comparison, after Scope::validate_pair has checked both sides. + pub fn selected(mut self, scope: &Scope, allow_missing: bool) -> Result { + if self.scope != Scope::default() { + return Err("document is already scoped; combine selectors in one Scope".to_string()); + } + if !allow_missing { + scope.validate_pair(&self, &self)?; + } + let mut wanted = scope + .top + .as_ref() + .map(|t| self.reachable(t)) + .unwrap_or_else(|| self.cells.keys().cloned().collect()); + if let Some(cell) = &scope.cell { + wanted.retain(|n| n == cell); + } + // Determine layer coverage before projecting cells: exact-cell scopes + // still need to know whether a referenced definition uses this layer. + let relevant = scope.layer.map(|layer| { + let mut relevant: BTreeSet<_> = self + .cells + .iter() + .filter(|(_, c)| { + !c.issues.is_empty() + || c.objects.iter().any(|o| { + !o.metadata + && (o.layer == Some(layer) + || (!o.issues.is_empty() && o.layer.is_none())) + }) + }) + .map(|(n, _)| n.clone()) + .collect(); + let mut parents = BTreeMap::>::new(); + for (parent, cell) in &self.cells { + for o in &cell.objects { + if let Some(target) = &o.target { + parents + .entry(target.clone()) + .or_default() + .push(parent.clone()); + if !self.known_cells.contains(target) { + relevant.insert(parent.clone()); + } + } + } + } + let mut pending: Vec<_> = relevant.iter().cloned().collect(); + // A cycle cannot be proven to have no effect on a layer. Preserve + // cycles (and their descendants) so graph validation remains visible. + let mut indegree: BTreeMap<_, usize> = + self.cells.keys().map(|n| (n.clone(), 0)).collect(); + for cell in self.cells.values() { + for target in cell.objects.iter().filter_map(|o| o.target.as_ref()) { + if let Some(n) = indegree.get_mut(target) { + *n += 1; + } + } + } + let mut ready: Vec<_> = indegree + .iter() + .filter(|(_, n)| **n == 0) + .map(|(n, _)| n.clone()) + .collect(); + while let Some(name) = ready.pop() { + for target in self.cells[&name] + .objects + .iter() + .filter_map(|o| o.target.as_ref()) + { + if let Some(n) = indegree.get_mut(target) { + *n -= 1; + if *n == 0 { + ready.push(target.clone()); + } + } + } + } + for (name, _) in indegree.iter().filter(|(_, n)| **n != 0) { + if relevant.insert(name.clone()) { + pending.push(name.clone()); + } + } + while let Some(child) = pending.pop() { + for parent in parents.get(&child).into_iter().flatten() { + if relevant.insert(parent.clone()) { + pending.push(parent.clone()); + } + } + } + relevant + }); + self.cells.retain(|name, _| { + wanted.contains(name) && relevant.as_ref().is_none_or(|r| r.contains(name)) + }); + if let (Some(layer), Some(relevant)) = (scope.layer, relevant) { + for cell in self.cells.values_mut() { + cell.objects.retain(|o| { + o.metadata + || o.layer == Some(layer) + || o.target + .as_ref() + .is_some_and(|t| relevant.contains(t) || !self.known_cells.contains(t)) + || (!o.issues.is_empty() && o.layer.is_none()) + }); + } + self.layer_names.retain(|pair, _| *pair == layer); + } else if scope.top.is_some() || scope.cell.is_some() { + let layers: BTreeSet<_> = self + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .filter_map(|o| o.layer) + .collect(); + self.layer_names.retain(|pair, _| layers.contains(pair)); + } + self.scope = scope.clone(); + self.refresh_issues(); + Ok(self) + } +} diff --git a/test.py b/test.py new file mode 100644 index 0000000..448e8e8 --- /dev/null +++ b/test.py @@ -0,0 +1,8 @@ +"""Read a GDSII file through Pheon's supported Python API.""" +import sys + +from pheon import arrow + + +if __name__ == "__main__": + print(arrow.read_path(sys.argv[1])) diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..993caa8 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,20 @@ +use pheon::{basic::encode_real8, records}; + +pub fn record(out: &mut Vec, tag: u16, data: &[u8]) { + out.extend_from_slice(&u16::try_from(data.len() + 4).unwrap().to_be_bytes()); + out.extend_from_slice(&tag.to_be_bytes()); + out.extend_from_slice(data); +} + +pub fn library_header() -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_BGNLIB, &[0; 24]); + record(&mut bytes, records::RTAG_LIBNAME, b"test"); + let units: Vec = [0.001, 1e-9] + .into_iter() + .flat_map(|v| encode_real8(v).unwrap().to_be_bytes()) + .collect(); + record(&mut bytes, records::RTAG_UNITS, &units); + bytes +} diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs new file mode 100644 index 0000000..0e74a5e --- /dev/null +++ b/tests/ffi_errors.rs @@ -0,0 +1,157 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{last_error_message, read_path, read_cells_bytes, scan_bytes, records}; +use std::{ffi::{CStr, CString}, fs, path::PathBuf, ptr}; + +struct TestFile(PathBuf); + +impl TestFile { + fn new(name: &str, contents: &[u8]) -> Self { + let path = std::env::temp_dir().join(format!("pheon-{name}-{}.gds", std::process::id())); + fs::write(&path, contents).unwrap(); + Self(path) + } + + fn cpath(&self) -> CString { + CString::new(self.0.to_str().unwrap()).unwrap() + } +} + +impl Drop for TestFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +fn error_message() -> String { + let message = last_error_message(); + assert!(!message.is_null()); + unsafe { CStr::from_ptr(message) }.to_str().unwrap().to_owned() +} + +#[test] +fn path_errors_preserve_outputs_and_convert_parser_panics() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("path pointer was null")); + assert!(array.is_released()); + + // A mismatched timestamp size currently makes Record::read panic. + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); + let file = TestFile::new("invalid-timestamps", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("Expected record size")); + assert!(array.is_released()); + + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("valid-empty", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(!array.is_released()); + assert!(last_error_message().is_null()); +} + +#[test] +fn other_threads_do_not_clear_or_free_error_messages() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + let pointer = last_error_message(); + let message = error_message(); + + std::thread::spawn(|| { + assert!(last_error_message().is_null()); + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("thread-empty", &bytes); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(last_error_message().is_null()); + }).join().unwrap(); + + assert_eq!(last_error_message(), pointer); + assert_eq!(unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(), message); +} + +#[test] +fn null_arrow_outputs_return_an_error() { + assert_eq!(unsafe { read_path(ptr::null(), ptr::null_mut(), ptr::null_mut()) }, -1); + assert!(error_message().contains("Arrow output pointer was null")); +} + +#[test] +fn large_malformed_inputs_report_offsets_without_dumping_contents() { + let mut bytes = vec![0; 1024 * 1024 + 4]; + bytes[..4].copy_from_slice(&[0, 4, 0xff, 0xff]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + let file = TestFile::new("large-malformed", &bytes); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let scan_message = error_message(); + assert_eq!( + unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, + -1 + ); + let read_message = error_message(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + -1 + ); + let selected_message = error_message(); + for message in [scan_message, read_message, selected_message] { + assert!(message.contains("at byte 4"), "{message}"); + assert!(message.contains("Unexpected record"), "{message}"); + assert!(message.len() < 256); + } + assert!(array.is_released()); +} + +#[test] +fn selected_cell_errors_use_file_offsets_for_bounded_ranges() { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let cell_end = bytes.len(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let range_end = bytes.len() as u64; + bytes.resize(bytes.len() + 1024, 0); + let ranges = [start, range_end]; + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!(error_message().contains(&format!( + "at byte {cell_end}: Cell byte range extends past ENDSTR" + ))); +} diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json new file mode 100644 index 0000000..75c2e4b --- /dev/null +++ b/tests/golden/report-equal.json @@ -0,0 +1,78 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 0, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "equal", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-failed.json b/tests/golden/report-failed.json new file mode 100644 index 0000000..c9b12d9 --- /dev/null +++ b/tests/golden/report-failed.json @@ -0,0 +1,64 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": null, + "cell": null, + "code": "invalid_input", + "layer": null, + "message": "truncated record", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": null, + "units": null + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": null, + "units": null + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "failed", + "summary": null +} diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json new file mode 100644 index 0000000..763de12 --- /dev/null +++ b/tests/golden/report-geometry.json @@ -0,0 +1,360 @@ +{ + "cell_changes": [ + { + "added": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:new:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "438", + "side": "new" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "after": { + "attributes": [], + "byte_offset": "346" + }, + "attributes_changed": false, + "before": { + "attributes": [], + "byte_offset": "346" + }, + "name": "leaf", + "removed": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:old:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "446", + "side": "old" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "replacements": [ + { + "added": "c0:new:0", + "explanation": "BOUNDARY modified; coordinate bounds unchanged", + "field_changes": [ + { + "after": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "before": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "tag": 4099 + } + ], + "match_rule": "unique_kind_layer_bounds", + "properties_changed": false, + "removed": "c0:old:0" + } + ] + } + ], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": { + "edges": [ + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "left" + }, + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "right" + }, + { + "after_count": "2", + "before_count": "2", + "parent": "left", + "reference_changed": false, + "target": "leaf" + }, + { + "after_count": "3", + "before_count": "3", + "parent": "right", + "reference_changed": false, + "target": "leaf" + } + ], + "nodes": [ + { + "after": "included", + "before": "included", + "cell": "CHIP", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "leaf", + "change": "direct" + }, + { + "after": "included", + "before": "included", + "cell": "left", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "right", + "change": "affected" + } + ] + }, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "2", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "1", + "changed_cells": "1", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "1" + } +} diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json new file mode 100644 index 0000000..4319c1f --- /dev/null +++ b/tests/golden/report-incomplete.json @@ -0,0 +1,105 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "old", + "target": null + }, + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "incomplete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "observed", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json new file mode 100644 index 0000000..3e1245a --- /dev/null +++ b/tests/golden/report-metadata.json @@ -0,0 +1,87 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [ + { + "after": "CORE:WG", + "before": "WG", + "layer": { + "number": 1, + "purpose": 0 + } + } + ], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "1", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt new file mode 100644 index 0000000..8811a25 --- /dev/null +++ b/tests/golden/review-canonical.txt @@ -0,0 +1,9 @@ +profile klamath-object-v2 +scope top=all cell=all layer=all +coordinates: integer database units; bounds: coordinate envelopes +HEADER=[600] +LIBNAME="test" +UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54 +layer 1/0 name="WG" +cell "CHIP" + BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt new file mode 100644 index 0000000..57e6c09 --- /dev/null +++ b/tests/golden/review-diff.txt @@ -0,0 +1,7 @@ +profile klamath-object-v2 +scope top=all cell=all layer=all +cell "CHIP" old@Some(62) new@Some(62) + ~ BOUNDARY modified; coordinate bounds unchanged + - @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] + + @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 0, 0, 0] +1 changed cells; 2 of 2 object changes shown; 0 layer-name changes diff --git a/tests/golden/review-hierarchy.mmd b/tests/golden/review-hierarchy.mmd new file mode 100644 index 0000000..dcc963b --- /dev/null +++ b/tests/golden/review-hierarchy.mmd @@ -0,0 +1,10 @@ +flowchart TD + %% scope top=all cell=all layer=all + n0["CHIP [affected]"] + n1["leaf [cell changed]"] + n2["left [affected]"] + n3["right [affected]"] + n0 -->|"x1"| n2 + n0 -->|"x1"| n3 + n2 -->|"x2"| n1 + n3 -->|"x3"| n1 diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt new file mode 100644 index 0000000..fa40d79 --- /dev/null +++ b/tests/golden/review-hierarchy.txt @@ -0,0 +1,6 @@ +scope top=all cell=all layer=all +"CHIP" x1 [affected] + "left" x1 [affected] + "leaf" x2 [cell changed] + "right" x1 [affected] + "leaf" x3 [cell changed] [shown above] diff --git a/tests/numeric_packing.rs b/tests/numeric_packing.rs new file mode 100644 index 0000000..14a6109 --- /dev/null +++ b/tests/numeric_packing.rs @@ -0,0 +1,103 @@ +use pheon::*; +use std::{mem::size_of, slice}; + +// Read the packed prefix of each source slot, as the Python wrapper does. +fn packed_bytes(values: &[T], width: usize) -> Vec { + let bytes = unsafe { slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) }; + bytes + .chunks_exact(size_of::()) + .flat_map(|slot| slot[..width].iter().copied()) + .collect() +} + +macro_rules! check_int2 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 256 as $ty, 32767 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!(packed_bytes(&values, 2), [0, 0, 0, 1, 1, 0, 0x7f, 0xff]); + }}; +} + +macro_rules! check_int4 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 65536 as $ty, 2147483520_i64 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!( + packed_bytes(&values, 4), + [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0x7f, 0xff, 0xff, 0x80] + ); + }}; +} + +#[test] +fn all_source_types_pack_to_big_endian_integer_bytes() { + check_int2!(u16, u16_to_i16); + check_int2!(u32, u32_to_i16); + check_int2!(u64, u64_to_i16); + check_int2!(i16, i16_to_i16); + check_int2!(i32, i32_to_i16); + check_int2!(i64, i64_to_i16); + check_int2!(f32, f32_to_i16); + check_int2!(f64, f64_to_i16); + check_int4!(u32, u32_to_i32); + check_int4!(u64, u64_to_i32); + check_int4!(i32, i32_to_i32); + check_int4!(i64, i64_to_i32); + check_int4!(f32, f32_to_i32); + check_int4!(f64, f64_to_i32); +} + +#[test] +fn signed_limits_and_unsigned_overflows_are_checked() { + let mut signed = [i64::from(i16::MIN), -1]; + assert_eq!(unsafe { i64_to_i16(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!(packed_bytes(&signed, 2), [0x80, 0, 0xff, 0xff]); + let mut signed = [i64::from(i32::MIN), i64::from(i32::MAX)]; + assert_eq!(unsafe { i64_to_i32(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!( + packed_bytes(&signed, 4), + [0x80, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff] + ); + + for value in [32768_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i16(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } + for value in [2147483648_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i32(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } +} + +#[test] +fn nonfinite_floats_and_rounded_up_f32_limit_are_rejected() { + for convert in [f32_to_i16, f32_to_i32] { + for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let mut input = [value]; + let result = unsafe { convert(input.as_mut_ptr(), 1) }; + assert_ne!(result, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + for convert in [f64_to_i16, f64_to_i32] { + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut input = [value]; + assert_ne!(unsafe { convert(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + let mut input = [2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 2147483648_f32); + assert_eq!(input, [2147483648_f32]); + let mut input = [-2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(packed_bytes(&input, 4), [0x80, 0, 0, 0]); +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py new file mode 100644 index 0000000..30367b4 --- /dev/null +++ b/tests/python/test_arrow.py @@ -0,0 +1,105 @@ +"""Public Arrow readers, lifetime management and input validation.""" + +import gc +import gzip +import mmap +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy + +from pheon import PheonError, arrow +from pheon._native import ffi +from test_ffi import layout + + +class ArrowTests(unittest.TestCase): + def test_full_scan_and_selected_reads_own_their_data(self): + data = bytearray(layout()) + full = arrow.read_bytes(data) + scan = arrow.scan_bytes(data) + cells = scan.field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + ) + ) + selected = arrow.read_cells_bytes(data, ranges) + data[:] = b"\0" * len(data) + del data + gc.collect() + self.assertEqual(full.to_pylist(), selected.to_pylist()) + self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"]) + self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"]) + + def test_paths_gzip_magic_and_closed_mmap(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "layout with spaces.gds" + path.write_bytes(layout()) + expected = arrow.read_path(path).to_pylist() + with path.open("rb") as stream: + with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer: + result = arrow.read_bytes(buffer) + self.assertEqual(result.to_pylist(), expected) + path.write_bytes(gzip.compress(layout())) + self.assertEqual(arrow.read_path(path).to_pylist(), expected) + + def test_empty_and_noncontiguous_cell_selections(self): + data = layout() + empty = arrow.read_cells_bytes(data, []) + self.assertEqual(empty.field("cells")[0].as_py(), []) + values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2] + self.assertFalse(values.flags.c_contiguous) + self.assertEqual( + arrow.read_cells_bytes(data, values).to_pylist(), + arrow.read_bytes(data).to_pylist(), + ) + + def test_bad_inputs_fail_before_importing_arrow(self): + with self.assertRaises(PheonError): + arrow.read_bytes(b"invalid") + with self.assertRaises(PheonError): + arrow.scan_bytes(b"") + with self.assertRaises(PheonError): + arrow.read_cells_bytes(layout(), [[0, 10]]) + with self.assertRaises(ValueError): + arrow.read_bytes(memoryview(layout())[::2]) + for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]: + with self.assertRaises(ValueError): + arrow.read_cells_bytes(layout(), ranges) + with self.assertRaises(TypeError): + arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + + def test_import_failure_releases_both_native_outputs(self): + releases = [] + + @ffi.callback("void(struct ArrowArray *)") + def release_array(value): + releases.append("array") + value.release = ffi.NULL + + @ffi.callback("void(struct ArrowSchema *)") + def release_schema(value): + releases.append("schema") + value.release = ffi.NULL + + def export(array, schema): + array.release = release_array + schema.release = release_schema + return 0 + + def fail_import(*args): + raise ValueError("import failed") + + with patch.object( + arrow, + "pyarrow", + SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)), + ): + with self.assertRaisesRegex(ValueError, "import failed"): + arrow._export(export) + self.assertEqual(releases, ["array", "schema"]) diff --git a/tests/python/test_ffi.py b/tests/python/test_ffi.py new file mode 100644 index 0000000..4dba750 --- /dev/null +++ b/tests/python/test_ffi.py @@ -0,0 +1,70 @@ +"""Exercise the explicit C ABI declarations against the actual shared library.""" +import struct +import tempfile +from pathlib import Path +import unittest + +from pheon._native import ffi, lib + + +def record(tag, data=b""): + return struct.pack(">HH", len(data) + 4, tag) + data + + +def layout(): + return b"".join([ + record(0x0002, struct.pack(">h", 600)), + record(0x0102, b"\0" * 24), + record(0x0206, b"test"), + record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")), + record(0x0502, b"\0" * 24), + record(0x0606, b"cell"), + record(0x0700), + record(0x0400), + ]) + + +class FfiTests(unittest.TestCase): + def check_export(self, call): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + self.assertEqual(call(array, schema), 0) + self.assertEqual(lib.last_error_message(), ffi.NULL) + self.assertEqual(array.length, 1) + self.assertEqual(ffi.string(schema.format), b"+s") + self.assertNotEqual(array.release, ffi.NULL) + self.assertNotEqual(schema.release, ffi.NULL) + self.assertGreaterEqual(array.n_children, 7) + self.assertEqual(array.n_children, schema.n_children) + self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit") + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + def test_scan_selected_read_and_path_exports(self): + data = layout() + self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s)) + self.check_export(lambda a, s: lib.read_bytes(data, len(data), a, s)) + # The library header is 62 bytes; the one cell ends just before ENDLIB. + ranges = ffi.new("uint64_t[]", [62, len(data) - 4]) + self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "file with spaces.gds" + path.write_bytes(data) + self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s)) + + def test_failures_leave_outputs_empty_and_expose_error_message(self): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1) + self.assertNotEqual(lib.last_error_message(), ffi.NULL) + self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message())) + self.assertEqual(array.release, ffi.NULL) + self.assertEqual(schema.release, ffi.NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_loader.py b/tests/python/test_loader.py new file mode 100644 index 0000000..4724495 --- /dev/null +++ b/tests/python/test_loader.py @@ -0,0 +1,36 @@ +"""Reject incompatible native libraries before exposing callable functions.""" + +from pathlib import Path +import runpy +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import pheon._native + + +class LoaderTests(unittest.TestCase): + def load(self, library): + with patch("cffi.FFI.dlopen", return_value=library): + return runpy.run_path(str(Path(pheon._native.__file__))) + + def test_matching_versions_are_accepted(self): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: 1, pheon_arrow_schema_version=lambda: 1 + ) + ) + + def test_incompatible_abi_or_schema_is_rejected(self): + for abi, schema in [(2, 1), (1, 2)]: + with self.assertRaisesRegex(ImportError, "Incompatible Pheon library"): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: abi, + pheon_arrow_schema_version=lambda: schema, + ) + ) + + def test_unversioned_library_is_rejected(self): + with self.assertRaisesRegex(ImportError, "version API"): + self.load(SimpleNamespace()) diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py new file mode 100644 index 0000000..44eb84d --- /dev/null +++ b/tests/python/test_packing.py @@ -0,0 +1,66 @@ +import unittest + +import numpy as np + +from pheon import pack_int2, pack_int4 + + +class PackingTests(unittest.TestCase): + def test_scalars_lists_and_dimensions(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]: + for shape in [(), (1,), (2, 2), (2, 2, 2)]: + count = int(np.prod(shape)) if shape else 1 + data = np.arange(count, dtype=dtype).reshape(shape) + before = data.copy() + with self.subTest(width=width, dtype=dtype, shape=shape): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + np.testing.assert_array_equal(data, before) + for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]: + with self.subTest(width=width, data=data): + self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes()) + + def test_noncontiguous_readonly_and_endian_inputs(self): + original = np.arange(48, dtype=np.int64).reshape(6, 8) + readonly = original.copy() + readonly.setflags(write=False) + for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly, + original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]: + before = data.copy() + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + with self.subTest(width=width, strides=data.strides, dtype=data.dtype): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C")) + np.testing.assert_array_equal(data, before) + + def test_signed_limits_and_native_integer_widths(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1 + data = np.array([[low, -1], [0, high]], dtype=np.int64) + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + for dtype in [np.int8, np.uint8, np.int16, np.uint16]: + data = np.array([[1, 2], [3, 4]], dtype=dtype) + self.assertEqual(pack_int2(data), data.astype(">i2").tobytes()) + self.assertEqual(pack_int4(data), data.astype(">i4").tobytes()) + + def test_failed_conversions_preserve_caller_data(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + limit = 1 << (8 * width - 1) + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=np.int64) + before = data.copy() + with self.subTest(width=width, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + for dtype in [np.float32, np.float64]: + for invalid in [np.nan, np.inf, -np.inf]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, invalid=invalid, dtype=dtype): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/real8.rs b/tests/real8.rs new file mode 100644 index 0000000..fc632a1 --- /dev/null +++ b/tests/real8.rs @@ -0,0 +1,75 @@ +use pheon::basic::{decode_real8, encode_real8, pack_real8}; + +#[test] +fn normalized_real8_has_known_encodings_and_exact_values() { + for (value, expected) in [ + (0.0, 0), + (-0.0, 0), + (1.0, 0x4110_0000_0000_0000), + (-2.0, 0xc120_0000_0000_0000), + (0.5, 0x4080_0000_0000_0000), + (0.001, 0x3e41_8937_4bc6_a7f0), + (1e-9, 0x3944_b82f_a09b_5a54), + (2_f64.powi(-260), 0x0010_0000_0000_0000), + ] { + let bits = encode_real8(value).unwrap(); + assert_eq!(bits, expected, "{value:e}"); + assert_eq!(decode_real8(bits), value); + } +} + +#[test] +fn real8_round_trips_each_supported_binary_exponent_and_mantissa_boundary() { + for exponent in -260..=251_i32 { + for mantissa in [ + 0, + 1, + 0x0008_0000_0000_0000, + 0x000f_ffff_ffff_fffe, + 0x000f_ffff_ffff_ffff, + ] { + for sign in [0, 1_u64 << 63] { + let ieee = sign | (((exponent + 1023) as u64) << 52) | mantissa; + let value = f64::from_bits(ieee); + let encoded = encode_real8(value).unwrap(); + assert_ne!(encoded & 0x00f0_0000_0000_0000, 0, "{value:e}"); + assert_eq!(decode_real8(encoded).to_bits(), ieee, "{value:e}"); + } + } + } +} + +#[test] +fn real8_underflow_is_canonical_and_overflow_is_an_error() { + let minimum = 2_f64.powi(-260); + let maximum_exclusive = 2_f64.powi(252); + for magnitude in [ + 0.0, + f64::from_bits(1), + f64::MIN_POSITIVE, + 1e-90, + 1e-80, + f64::from_bits(minimum.to_bits() - 1), + ] { + for value in [magnitude, -magnitude] { + assert_eq!(encode_real8(value).unwrap(), 0, "{value:e}"); + } + } + let largest = f64::from_bits(maximum_exclusive.to_bits() - 1); + assert_eq!(encode_real8(largest).unwrap(), 0x7fff_ffff_ffff_fff8); + assert_eq!(decode_real8(encode_real8(largest).unwrap()), largest); + for value in [ + maximum_exclusive, + -maximum_exclusive, + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] { + assert!(encode_real8(value).is_err(), "{value}"); + let mut buffer = [0x55; 8]; + assert!(pack_real8(&mut buffer, value).is_err()); + assert_eq!(buffer, [0x55; 8]); + } +} diff --git a/tests/review.rs b/tests/review.rs new file mode 100644 index 0000000..71b895b --- /dev/null +++ b/tests/review.rs @@ -0,0 +1,1248 @@ +mod common; +#[path = "review/reports.rs"] +mod reports; + +use common::{library_header, record}; +use pheon::{ + records::*, + review::{Comparison, Document, LAYER_SENTINEL, hierarchy}, +}; +use std::{fs, path::PathBuf, process::Command}; + +fn ascii(bytes: &mut Vec, tag: u16, value: &str) { + let mut value = value.as_bytes().to_vec(); + if value.len() % 2 == 1 { + value.push(0); + } + record(bytes, tag, &value); +} + +fn xy(bytes: &mut Vec, points: &[[i32; 2]]) { + record( + bytes, + RTAG_XY, + &points + .iter() + .flatten() + .flat_map(|v| v.to_be_bytes()) + .collect::>(), + ); +} + +fn layout(cells: &[(&str, Vec>)]) -> Vec { + let mut bytes = library_header(); + for (name, objects) in cells { + record(&mut bytes, RTAG_BGNSTR, &[0; 24]); + ascii(&mut bytes, RTAG_STRNAME, name); + for object in objects { + bytes.extend_from_slice(object); + } + record(&mut bytes, RTAG_ENDSTR, &[]); + } + record(&mut bytes, RTAG_ENDLIB, &[]); + bytes +} + +fn polygon(points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_BOUNDARY, &[]); + record(&mut bytes, RTAG_LAYER, &1_u16.to_be_bytes()); + record(&mut bytes, RTAG_DATATYPE, &0_u16.to_be_bytes()); + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn square() -> Vec { + polygon(&[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]) +} + +fn triangle() -> Vec { + polygon(&[[0, 0], [10, 0], [0, 10], [0, 0]]) +} + +fn reference(target: &str, x: i32, count: i16) -> Vec { + let mut bytes = Vec::new(); + record( + &mut bytes, + if count == 1 { RTAG_SREF } else { RTAG_AREF }, + &[], + ); + ascii(&mut bytes, RTAG_SNAME, target); + if count == 1 { + xy(&mut bytes, &[[x, 0]]); + } else { + let counts: Vec<_> = [count, 1].into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, RTAG_COLROW, &counts); + xy(&mut bytes, &[[x, 0], [x + 100, 0], [x, 0]]); + } + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn label(carrier: (u16, u16), text: &str) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_TEXT, &[]); + record(&mut bytes, RTAG_LAYER, &carrier.0.to_be_bytes()); + record(&mut bytes, RTAG_TEXTTYPE, &carrier.1.to_be_bytes()); + xy(&mut bytes, &[[0, 0]]); + ascii(&mut bytes, RTAG_STRING, text); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn with_property(mut object: Vec, value: &str) -> Vec { + object.truncate(object.len() - 4); + record(&mut object, RTAG_PROPATTR, &17_u16.to_be_bytes()); + ascii(&mut object, RTAG_PROPVALUE, value); + record(&mut object, RTAG_ENDEL, &[]); + object +} + +fn shared(geometry: Vec) -> Vec { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 20, 1)], + ), + ("left", vec![reference("leaf", 0, 2)]), + ("right", vec![reference("leaf", 0, 3)]), + ("leaf", vec![geometry]), + ]) +} + +struct Pair { + name: &'static str, + old: Vec, + new: Vec, + changed_cells: usize, + removed: usize, + added: usize, + metadata: usize, +} + +// Ten deliberately small fixture pairs specify the first object profile. +fn corpus() -> Vec { + let base = layout(&[("CHIP", vec![square(), label((2, 0), "port")])]); + let mut timestamps = base.clone(); + timestamps[10] = 1; // BGNLIB payload, ignored by the object profile. + let meta = |name| { + layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), name), + ], + )]) + }; + vec![ + Pair { + name: "record reordering", + old: base.clone(), + new: layout(&[("CHIP", vec![label((2, 0), "port"), square()])]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "timestamp edit", + old: base, + new: timestamps, + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "polygon start and winding", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[( + "CHIP", + vec![polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])], + )]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "duplicate shape", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![square(), square()])]), + changed_cells: 1, + removed: 0, + added: 1, + metadata: 0, + }, + Pair { + name: "same bounds different geometry", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![triangle()])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "moved reference", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 5, 1)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "array count", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 2)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 0, 3)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "element property", + old: layout(&[("CHIP", vec![with_property(square(), "old")])]), + new: layout(&[("CHIP", vec![with_property(square(), "new")])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "layer name only", + old: meta("1:0:WG"), + new: meta("1:0:CORE:WG"), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 1, + }, + Pair { + name: "shared changed descendant", + old: shared(square()), + new: shared(triangle()), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + ] +} + +#[test] +fn ten_fixture_pairs_have_explicit_expected_changes() { + for pair in corpus() { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert!(diff.issues.is_empty(), "{}: {:?}", pair.name, diff.issues); + assert_eq!(diff.cells.len(), pair.changed_cells, "{}", pair.name); + assert_eq!( + diff.cells.iter().map(|c| c.removed.len()).sum::(), + pair.removed, + "{}", + pair.name + ); + assert_eq!( + diff.cells.iter().map(|c| c.added.len()).sum::(), + pair.added, + "{}", + pair.name + ); + assert_eq!(diff.layer_names.len(), pair.metadata, "{}", pair.name); + let equal = pair.changed_cells + pair.metadata == 0; + assert_eq!(diff.is_equal(), equal, "{}", pair.name); + assert_eq!( + old.canonical_dump() == new.canonical_dump(), + equal, + "{}", + pair.name + ); + assert_eq!(diff.exit_code(), if equal { 0 } else { 1 }); + } +} + +#[test] +fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { + let bytes = layout(&[("CHIP", vec![square()])]); + let doc = Document::parse(&bytes).unwrap(); + let dump = doc.canonical_dump(); + assert!( + dump.ends_with( + "cell \"CHIP\"\n BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0]\n" + ), + "{dump}" + ); + assert!(!dump.contains("BGNSTR")); + assert!( + doc.record_dump() + .contains("0000000000 0002 HEADER size=6 [600]") + ); + assert!(doc.record_dump().contains("BGNSTR")); +} + +#[test] +fn canonical_dump_and_hierarchy_match_versioned_goldens() { + let bytes = layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + )]); + assert_eq!( + Document::parse(&bytes).unwrap().canonical_dump(), + include_str!("golden/review-canonical.txt") + ); + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + hierarchy(&a, &b, Some(&diff), false, 8), + include_str!("golden/review-hierarchy.txt") + ); + assert_eq!( + hierarchy(&a, &b, Some(&diff), true, 8), + include_str!("golden/review-hierarchy.mmd") + ); +} + +#[test] +fn misplaced_library_fields_and_nul_names_are_rejected() { + let mut late = layout(&[("CHIP", vec![])]); + late.truncate(late.len() - 4); + ascii(&mut late, RTAG_LIBNAME, "late"); + record(&mut late, RTAG_ENDLIB, &[]); + assert!( + Document::parse(&late) + .unwrap_err() + .contains("library field outside") + ); + assert!( + Document::parse(&layout(&[("CH\0IP", vec![])])) + .unwrap_err() + .contains("NUL") + ); +} + +#[test] +fn cell_and_property_order_do_not_affect_comparison() { + let a = with_property(with_property(square(), "a"), "b"); + let b = with_property(with_property(square(), "b"), "a"); + let a = layout(&[("z", vec![a]), ("a", vec![])]); + let b = layout(&[("a", vec![]), ("z", vec![b])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + assert!(Comparison::between(&a, &b).is_equal()); + assert_eq!(a.canonical_dump(), b.canonical_dump()); +} + +#[test] +fn metadata_requires_a_sentinel_in_the_same_cell_and_carrier_pair() { + let bytes = layout(&[ + ( + "CHIP", + vec![ + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "2:0:ordinary text"), + ], + ), + ("OTHER", vec![label((900, 7), "3:0:ordinary text")]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!(doc.layer_names.len(), 1); + assert_eq!(doc.layer_names.get(&(1, 0)).unwrap(), "WG"); + assert!(doc.canonical_dump().contains("2:0:ordinary text")); + assert!(doc.canonical_dump().contains("3:0:ordinary text")); + assert!(!doc.canonical_dump().contains(LAYER_SENTINEL)); + let conflicts = layout(&[( + "CHIP", + vec![ + label((9, 0), LAYER_SENTINEL), + label((9, 0), "1:0:A"), + label((9, 0), "1:0:B"), + ], + )]); + assert!( + Document::parse(&conflicts) + .unwrap_err() + .contains("conflicting names") + ); + assert!( + pheon::review::dump_records(&conflicts) + .unwrap() + .contains("1:0:B") + ); + let future = layout(&[("CHIP", vec![label((9, 0), "KLAMATH_LAYER_NAMES_V2")])]); + let future = Document::parse(&future).unwrap(); + assert_eq!(Comparison::between(&future, &future).exit_code(), 2); +} + +#[test] +fn unknown_records_never_produce_a_successful_equality_claim() { + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![unknown])]); + let doc = Document::parse(&bytes).unwrap(); + let diff = Comparison::between(&doc, &doc); + assert!(!diff.is_equal()); + assert_eq!(diff.exit_code(), 2); + assert!(doc.record_dump().contains("7f00")); + assert!(diff.text(0).contains("INCOMPLETE")); +} + +#[test] +fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + hierarchy(&a, &b, Some(&diff), false, 8), + concat!( + "scope top=all cell=all layer=all\n", + "\"CHIP\" x1 [affected]\n", + " \"left\" x1 [affected]\n", + " \"leaf\" x2 [cell changed]\n", + " \"right\" x1 [affected]\n", + " \"leaf\" x3 [cell changed] [shown above]\n", + ) + ); + let graph = hierarchy(&a, &b, Some(&diff), true, 8); + assert_eq!(graph.matches("leaf [cell changed]").count(), 1); + assert!(graph.contains("x2")); + assert!(graph.contains("x3")); + let limited = hierarchy(&a, &b, Some(&diff), false, 1); + assert!(!limited.contains("\"leaf\"")); + assert!(limited.contains("children omitted")); +} + +#[test] +fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { + for pair in corpus() + .into_iter() + .filter(|p| p.name == "moved reference" || p.name == "array count") + { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + let tree = hierarchy(&old, &new, Some(&diff), false, 8); + assert!(tree.contains("[cell unchanged]"), "{tree}"); + assert!(tree.contains("[reference changed]"), "{tree}"); + if pair.name == "array count" { + assert!(tree.contains("x2 -> x3"), "{tree}"); + } + } + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]); + let b = layout(&[("CHIP", vec![])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let tree = hierarchy(&a, &b, Some(&Comparison::between(&a, &b)), false, 8); + assert!(tree.contains("\"leaf\" x1 -> x0"), "{tree}"); +} + +#[test] +fn same_bounds_changes_and_truncated_reports_remain_differences() { + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle()])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + diff.cells[0].removed[0].bounds, + diff.cells[0].added[0].bounds + ); + assert!(diff.text(0).contains("0 of 2 object changes shown")); + assert_eq!(diff.exit_code(), 1); + assert!(diff.cells[0].removed[0].offset > 0); +} + +#[test] +fn unit_changes_report_values_without_claiming_geometry_equivalence() { + let a = layout(&[("CHIP", vec![square()])]); + let units = Document::parse(&a) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + let mut b = a.clone(); + b[units + 12..units + 20].copy_from_slice( + &pheon::basic::encode_real8(2e-9) + .unwrap() + .to_be_bytes(), + ); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!(diff.exit_code(), 1); + assert_eq!(diff.library_fields.len(), 1); + assert_eq!(diff.library_fields[0].field, "UNITS"); + assert!(diff.text(0).contains("UNITS changed")); + assert!(diff.cells.is_empty()); +} + +#[test] +fn dangling_references_cycles_and_duplicate_cell_names_are_reported() { + let missing = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let missing = Document::parse(&missing).unwrap(); + assert!( + missing + .issues + .iter() + .any(|s| s.contains("unresolved reference")) + ); + let cycle = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let cycle = Document::parse(&cycle).unwrap(); + assert!(cycle.issues.iter().any(|s| s.contains("cyclic"))); + assert!(hierarchy(&cycle, &cycle, None, false, 8).len() < 200); + let duplicate = layout(&[("a", vec![]), ("a", vec![])]); + assert!( + Document::parse(&duplicate) + .unwrap_err() + .contains("duplicate cell") + ); +} + +#[test] +fn absolute_transform_flags_are_not_discarded() { + let a = reference("leaf", 0, 1); + let mut b = Vec::new(); + record(&mut b, RTAG_SREF, &[]); + ascii(&mut b, RTAG_SNAME, "leaf"); + record(&mut b, RTAG_STRANS, &[0, 2]); + xy(&mut b, &[[0, 0]]); + record(&mut b, RTAG_ENDEL, &[]); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + assert_eq!( + Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .exit_code(), + 1 + ); +} + +#[test] +fn polygon_normalization_handles_repeated_minimum_vertices() { + let ring = [[0, 0], [4, 0], [4, 4], [0, 0], [-2, 2], [-2, 4], [0, 0]]; + let baseline = layout(&[("CHIP", vec![polygon(&ring)])]); + let baseline = Document::parse(&baseline).unwrap(); + let mut open = ring[..ring.len() - 1].to_vec(); + for _ in 0..2 { + for start in 0..open.len() { + let mut rotated = open[start..] + .iter() + .chain(&open[..start]) + .copied() + .collect::>(); + rotated.push(rotated[0]); + let candidate = layout(&[("CHIP", vec![polygon(&rotated)])]); + let candidate = Document::parse(&candidate).unwrap(); + assert!(Comparison::between(&baseline, &candidate).is_equal()); + } + open.reverse(); + } +} + +#[test] +fn malformed_lengths_fields_and_units_return_errors_without_panicking() { + let good = layout(&[("CHIP", vec![square()])]); + for length in 0..good.len() { + assert!( + Document::parse(&good[..length]).is_err(), + "accepted truncated length {length}" + ); + } + let mut bad = good.clone(); + bad[0..2].copy_from_slice(&3_u16.to_be_bytes()); + assert!(Document::parse(&bad).is_err()); + let mut bad = good.clone(); + let units = Document::parse(&good) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + bad[units + 4..units + 20].fill(0); + assert!(Document::parse(&bad).unwrap_err().contains("UNITS")); + let mut polygon = square(); + polygon.truncate(polygon.len() - 4); + record(&mut polygon, RTAG_LAYER, &[0, 1]); + record(&mut polygon, RTAG_ENDEL, &[]); + assert!( + Document::parse(&layout(&[("CHIP", vec![polygon])])) + .unwrap_err() + .contains("duplicate element field") + ); +} + +#[test] +fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { + let directory = std::env::temp_dir().join(format!("pheon review {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old file.gds"); + let new = directory.join("new file.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let equal = command().arg("diff").arg(&old).arg(&old).output().unwrap(); + assert_eq!(equal.status.code(), Some(0)); + let diff = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "mermaid", "--max-changes", "0"]) + .output() + .unwrap(); + assert_eq!(diff.status.code(), Some(1)); + let text = String::from_utf8(diff.stdout).unwrap(); + assert!(text.contains("0 of 2 object changes shown")); + assert!(text.contains("```mermaid\nflowchart TD")); + let dump = command() + .arg("dump") + .arg(&old) + .args(["--records"]) + .output() + .unwrap(); + assert_eq!(dump.status.code(), Some(0)); + assert!(String::from_utf8(dump.stdout).unwrap().contains("BGNSTR")); + let graph = command() + .arg("hierarchy") + .arg(&old) + .args(["--format", "mermaid"]) + .output() + .unwrap(); + assert_eq!(graph.status.code(), Some(0)); + assert!( + String::from_utf8(graph.stdout) + .unwrap() + .starts_with("flowchart TD") + ); + fs::write(&new, b"invalid").unwrap(); + assert_eq!( + command() + .arg("diff") + .arg(&old) + .arg(&new) + .output() + .unwrap() + .status + .code(), + Some(2) + ); +} + +#[test] +fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { + use pheon::review::write_layer_names; + use std::collections::BTreeMap; + let mut bytes = shared(with_property(square(), "keep this")); + bytes.extend_from_slice(&[0; 32]); + let updates = BTreeMap::from([ + ((1, 0), "WG".to_string()), + ((2, 3), "HEATER:draw".to_string()), + ]); + let output = write_layer_names(&bytes, "CHIP", (900, 7), &updates).unwrap(); + let doc = Document::parse(&output).unwrap(); + assert_eq!(doc.layer_names, updates); + let meta: Vec<_> = doc.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .collect(); + let start = meta.iter().map(|o| o.offset).min().unwrap(); + let end = meta.iter().map(|o| o.end_offset).max().unwrap(); + assert_eq!([&output[..start], &output[end..]].concat(), bytes); + let updated = write_layer_names( + &output, + "CHIP", + (900, 7), + &BTreeMap::from([((1, 0), "CORE".to_string())]), + ) + .unwrap(); + let parsed = Document::parse(&updated).unwrap(); + assert_eq!(parsed.layer_names[&(1, 0)], "CORE"); + assert_eq!(parsed.layer_names[&(2, 3)], "HEATER:draw"); + assert_eq!( + parsed.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .count(), + 3 + ); + let diff = Comparison::between(&Document::parse(&output).unwrap(), &parsed); + assert!(diff.cells.is_empty()); + assert_eq!(diff.layer_names.len(), 1); + assert_eq!( + write_layer_names(&updated, "CHIP", (900, 7), &BTreeMap::new()).unwrap(), + updated + ); +} + +#[test] +fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { + use pheon::review::{parse_layer_name, write_layer_names}; + use std::collections::BTreeMap; + let bytes = layout(&[("CHIP", vec![label((900, 7), "ordinary label")])]); + assert!( + write_layer_names(&bytes, "CHIP", (900, 7), &BTreeMap::new()) + .unwrap_err() + .contains("ordinary design") + ); + assert!( + write_layer_names(&bytes, "missing", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("does not exist") + ); + let bytes = layout(&[("CHIP", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")])]); + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("unsupported") + ); + let bytes = layout(&[("CHIP", vec![square()])]); + for value in ["".to_string(), "bad\0name".to_string(), "x".repeat(65536)] { + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::from([((1, 0), value)])) + .is_err() + ); + } + for text in ["+1:0:name", "1:-1:name", "1:0:", "65536:0:name"] { + assert!(parse_layer_name(text).is_err()); + } +} + +#[test] +fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { + let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let source = directory.join("source.gds"); + let output = directory.join("output.gds"); + let bytes = shared(square()); + fs::write(&source, &bytes).unwrap(); + let _ = fs::remove_file(&output); + let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); + command.arg("name-layers").arg(&source).arg(&output).args([ + "--cell", + "CHIP", + "--carrier", + "900/7", + "--name", + "1:0:WG", + ]); + let result = command.output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(fs::read(&source).unwrap(), bytes); + let generated = fs::read(&output).unwrap(); + assert_eq!( + Document::parse(&generated).unwrap().layer_names[&(1, 0)], + "WG" + ); + assert_eq!(command.output().unwrap().status.code(), Some(2)); + assert_eq!(fs::read(&output).unwrap(), generated); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { + use pheon::review::Scope; + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ("unused", vec![square()]), + ]); + let b = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![triangle()]), + ("unused", vec![triangle()]), + ]); + let top = Scope { + top: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&top, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&top, false).unwrap(); + assert_eq!( + old.cells.keys().map(String::as_str).collect::>(), + ["CHIP", "leaf"] + ); + assert_eq!(Comparison::between(&old, &new).cells.len(), 1); + assert!(old.canonical_dump().contains("scope top=\"CHIP\"")); + let cell = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&cell, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&cell, false).unwrap(); + assert!(old.issues.is_empty()); + assert!(Comparison::between(&old, &new).is_equal()); + let missing = Scope { + top: Some("missing".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&a) + .unwrap() + .selected(&missing, false) + .is_err() + ); + let wrong = Scope { + top: Some("CHIP".to_string()), + cell: Some("unused".to_string()), + layer: None, + }; + assert!( + wrong + .validate_pair(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .is_err() + ); +} + +#[test] +fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { + use pheon::review::Scope; + let mut other_layer = square(); + other_layer[8..10].copy_from_slice(&2_u16.to_be_bytes()); + let a = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 0, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer.clone()]), + ]); + let b = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 5, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer]), + ]); + for (layer, expected) in [((1, 0), 0), ((2, 0), 1)] { + let scope = Scope { + layer: Some(layer), + ..Scope::default() + }; + let old = Document::parse(&a) + .unwrap() + .selected(&scope, false) + .unwrap(); + let new = Document::parse(&b) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!(Comparison::between(&old, &new).exit_code(), expected); + assert!(old.issues.is_empty()); + } +} + +#[test] +fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { + use pheon::review::Scope; + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![square()]), ("unused", vec![unknown])]); + let scope = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .is_empty() + ); + let scope = Scope { + cell: Some("unused".to_string()), + ..Scope::default() + }; + assert!( + !Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .is_empty() + ); + let bytes = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let scope = Scope { + layer: Some((999, 0)), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .iter() + .any(|s| s.contains("cyclic")) + ); +} + +#[test] +fn display_filters_never_change_equality_or_totals() { + use pheon::review::DisplayFilter; + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text_filtered( + 100, + &DisplayFilter { + cell: None, + layer: Some((42, 0)), + }, + ); + assert!(text.contains("display-only")); + assert!(text.contains("0 of 2 object changes shown")); + assert_eq!(diff.exit_code(), 1); +} + +#[test] +fn git_external_driver_handles_changes_additions_deletions_and_errors() { + let directory = std::env::temp_dir().join(format!( + "pheon-git-review-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let inherited_path = std::env::var_os("PATH").unwrap_or_default(); + let git_path = std::env::join_paths( + std::iter::once( + PathBuf::from(env!("CARGO_BIN_EXE_pheon")) + .parent() + .unwrap() + .to_path_buf(), + ) + .chain(std::env::split_paths(&inherited_path)), + ) + .unwrap(); + let git = |args: &[&str]| { + let result = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(args) + .output() + .unwrap(); + assert!( + result.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&result.stderr) + ); + result + }; + git(&["init", "-q"]); + git(&["config", "user.name", "Review Fixture"]); + git(&["config", "user.email", "fixture@example.invalid"]); + git(&["config", "commit.gpgsign", "false"]); + git(&["config", "core.hooksPath", "/dev/null"]); + git(&[ + "config", + "include.path", + concat!(env!("CARGO_MANIFEST_DIR"), "/contrib/git/config"), + ]); + assert_eq!( + String::from_utf8(git(&["config", "--get", "diff.pheon.trustExitCode"]).stdout) + .unwrap() + .trim(), + "false" + ); + fs::write( + directory.join(".gitattributes"), + include_str!("../contrib/git/attributes"), + ) + .unwrap(); + let path = directory.join("chip layout.gds"); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + git(&["add", "."]); + git(&["commit", "-qm", "baseline"]); + fs::write(&path, layout(&[("CHIP", vec![triangle()])])).unwrap(); + let changed = git(&["diff", "--", "chip layout.gds"]); + assert!( + String::from_utf8(changed.stdout) + .unwrap() + .contains("2 of 2 object changes shown") + ); + fs::write(&path, b"invalid").unwrap(); + let invalid = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(["diff", "--", "chip layout.gds"]) + .output() + .unwrap(); + assert!(!invalid.status.success()); + assert!(String::from_utf8_lossy(&invalid.stderr).contains("pheon:")); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + let added = directory.join("added layout.gds"); + fs::write(&added, shared(square())).unwrap(); + git(&["add", "added layout.gds"]); + let result = git(&["diff", "--cached", "--", "added layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("layout added") + ); + fs::remove_file(&path).unwrap(); + git(&["add", "-u"]); + let result = git(&["diff", "--cached", "--", "chip layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("layout deleted") + ); + // Textconv is an alternative to the external driver, with a separate cache. + git(&["commit", "-qm", "replace fixture"]); + git(&["mv", "added layout.gds", "renamed layout.gds"]); + let renamed = directory.join("renamed layout.gds"); + fs::write(&renamed, shared(triangle())).unwrap(); + git(&["add", "renamed layout.gds"]); + let result = git(&["diff", "--cached", "--find-renames"]); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.contains("renamed layout.gds"), "{text}"); + assert!(text.contains("2 of 2 object changes shown"), "{text}"); + git(&["commit", "-qm", "rename fixture"]); + fs::write( + directory.join(".gitattributes"), + "*.gds diff=pheon-text\n", + ) + .unwrap(); + fs::write(&renamed, shared(square())).unwrap(); + let result = git(&["diff", "--", "renamed layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("- BOUNDARY") + ); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn replacement_explanations_are_unique_and_keep_full_evidence() { + for pair in corpus().into_iter().filter(|p| { + [ + "same bounds different geometry", + "moved reference", + "array count", + ] + .contains(&p.name) + }) { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert_eq!(diff.cells[0].replacements.len(), 1); + let text = diff.text(100); + assert!(text.contains(" ~ ")); + assert!(text.contains(" - @")); + assert!(text.contains(" + @")); + match pair.name { + "moved reference" => assert!(text.contains("delta=(5, 0) dbu")), + "array count" => assert!(text.contains("columns/rows 2 x 1 -> 3 x 1")), + _ => assert_eq!(text, include_str!("golden/review-diff.txt")), + } + assert!(!diff.text(0).contains(" ~ ")); + assert!(diff.text(1).contains("1 of 2 object changes shown")); + } + let a = layout(&[("CHIP", vec![square(), square()])]); + let b = layout(&[("CHIP", vec![triangle(), triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!(diff.cells[0].replacements.is_empty()); +} + +#[test] +fn path_width_explanation_preserves_absent_and_zero() { + let path = |width: Option| { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_PATH, &[]); + record(&mut bytes, RTAG_LAYER, &[0, 1]); + record(&mut bytes, RTAG_DATATYPE, &[0, 0]); + if let Some(n) = width { + record(&mut bytes, RTAG_WIDTH, &n.to_be_bytes()); + } + xy(&mut bytes, &[[0, 0], [10, 10]]); + record(&mut bytes, RTAG_ENDEL, &[]); + layout(&[("CHIP", vec![bytes])]) + }; + let a = path(None); + let b = path(Some(0)); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!(diff.text(100).contains("width absent -> 0 dbu")); +} + +#[test] +fn unchanged_layer_names_explain_geometry_changes_once() { + let named = |geometry| { + layout(&[( + "CHIP", + vec![ + geometry, + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + )]) + }; + let a = named(square()); + let b = named(triangle()); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text(100); + assert_eq!(text.matches("name=\"WG\"").count(), 1); + assert!(diff.layer_names.is_empty()); +} + +#[test] +fn an_added_empty_document_is_still_a_presence_change() { + let bytes = layout(&[]); + let doc = Document::parse(&bytes).unwrap(); + let diff = Comparison::between(&doc.empty_like(), &doc); + assert_eq!(diff.exit_code(), 1); + assert!(diff.text(0).contains("library presence:")); + assert!(!diff.text(0).contains("equal under")); +} + +#[test] +fn cli_scope_and_display_filters_have_distinct_exit_behavior() { + let directory = std::env::temp_dir().join(format!("pheon-scope-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + for (options, expected) in [ + (vec!["--cell", "CHIP"], 0), + (vec!["--top", "CHIP"], 1), + (vec!["--layer", "99/0"], 0), + (vec!["--show-cell", "missing"], 1), + (vec!["--show-layer", "99/0"], 1), + (vec!["--top", "missing"], 2), + ] { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(&options) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(expected), + "{options:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + fs::write(&old, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["diff", "--top", "CHIP"]) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn large_polygons_and_billion_instance_arrays_stay_compact() { + let mut points = vec![[0, 0], [0, 1]]; + points.extend((1..8187).map(|x| [x, 1])); + points.extend([[8186, 0], [0, 0]]); + let mut array = Vec::new(); + record(&mut array, RTAG_AREF, &[]); + ascii(&mut array, RTAG_SNAME, "leaf"); + record(&mut array, RTAG_COLROW, &[0x7f, 0xff, 0x7f, 0xff]); + xy(&mut array, &[[0, 0], [32767, 0], [0, 32767]]); + record(&mut array, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![array]), ("leaf", vec![polygon(&points)])]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.edges()[&("CHIP".to_string(), "leaf".to_string())], + 32767 * 32767 + ); + assert_eq!(doc.cells["CHIP"].objects.len(), 1); + assert!(hierarchy(&doc, &doc, None, false, 8).len() < 200); + assert!(Comparison::between(&doc, &doc).is_equal()); +} + +#[test] +fn deep_hierarchy_comparison_and_rendering_are_iterative() { + let names: Vec<_> = (0..2048).map(|i| format!("cell_{i}")).collect(); + let cells = |leaf: Vec| { + names + .iter() + .enumerate() + .map(|(i, name)| { + ( + name.as_str(), + if i + 1 == names.len() { + vec![leaf.clone()] + } else { + vec![reference(&names[i + 1], 0, 1)] + }, + ) + }) + .collect::>() + }; + let a = layout(&cells(square())); + let b = layout(&cells(triangle())); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + let tree = hierarchy(&a, &b, Some(&diff), false, 5); + assert_eq!(diff.cells.len(), 1); + assert!(tree.contains("children omitted")); + assert!(tree.len() < 500); +} diff --git a/tests/review/reports.rs b/tests/review/reports.rs new file mode 100644 index 0000000..f761a8c --- /dev/null +++ b/tests/review/reports.rs @@ -0,0 +1,633 @@ +use super::*; +use pheon::review::{ + DisplayFilter, Scope, + report::{ReportInputs, ReportOptions, failure_report}, +}; +use serde_json::{Value, json}; + +fn inputs() -> ReportInputs { + ReportInputs { + old: Some("before.gds".into()), + new: Some("after.gds".into()), + } +} + +fn report(old: &[u8], new: &[u8], options: &ReportOptions) -> Value { + let a = Document::parse(old).unwrap(); + let b = Document::parse(new).unwrap(); + Comparison::between(&a, &b).json_report(&a, &b, &inputs(), options) +} + +fn unknown(geometry: Vec) -> Vec { + let mut object = geometry; + object.truncate(object.len() - 4); + record(&mut object, 0x7f02, &[0, 42]); + record(&mut object, RTAG_ENDEL, &[]); + layout(&[("CHIP", vec![object])]) +} + +#[test] +fn reports_preserve_fixture_results_totals_and_replacement_links() { + for pair in corpus() { + let r = report(&pair.old, &pair.new, &ReportOptions::default()); + assert_eq!(r["schema_version"], 1); + assert_eq!(r["comparison_profile"], "klamath-object-v2"); + assert_eq!(r["producer"]["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(r["status"], "complete", "{}", pair.name); + assert_eq!( + r["exit_code"], + if pair.changed_cells + pair.metadata == 0 { + 0 + } else { + 1 + } + ); + assert_eq!( + r["summary"]["changed_cells"], + pair.changed_cells.to_string() + ); + assert_eq!(r["summary"]["added_objects"], pair.added.to_string()); + assert_eq!(r["summary"]["removed_objects"], pair.removed.to_string()); + assert_eq!( + r["summary"]["layer_name_changes"], + pair.metadata.to_string() + ); + assert_eq!(r["report_selection"]["truncated"], false); + for cell in r["cell_changes"].as_array().unwrap() { + for replacement in cell["replacements"].as_array().unwrap() { + assert!( + cell["removed"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["removed"]) + ); + assert!( + cell["added"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["added"]) + ); + } + } + assert_eq!(r, report(&pair.old, &pair.new, &ReportOptions::default())); + } +} + +#[test] +fn incomplete_reports_distinguish_known_differences_and_include_structured_locations() { + let a = unknown(square()); + let same = report(&a, &a, &ReportOptions::default()); + assert_eq!(same["status"], "incomplete"); + assert_eq!(same["result"], "unknown"); + assert_eq!(same["exit_code"], 2); + assert_eq!(same["summary"]["counts"], "observed"); + let diagnostic = &same["diagnostics"][0]; + assert_eq!(diagnostic["code"], "unsupported_record"); + assert_eq!(diagnostic["side"], "old"); + assert_eq!(diagnostic["cell"], "CHIP"); + assert_eq!(diagnostic["layer"], json!({"number":1,"purpose":0})); + let offset: usize = diagnostic["byte_offset"].as_str().unwrap().parse().unwrap(); + assert_eq!(&a[offset + 2..offset + 4], &0x7f02_u16.to_be_bytes()); + let changed = report(&a, &unknown(triangle()), &ReportOptions::default()); + assert_eq!(changed["status"], "incomplete"); + assert_eq!(changed["result"], "different"); + assert_eq!(changed["exit_code"], 2); + assert_eq!( + changed["cell_changes"][0]["removed"][0]["fields"] + .as_array() + .unwrap() + .last() + .unwrap()["payload_hex"], + "0x002a" + ); +} + +#[test] +fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { + for location in ["CHIP", "legend"] { + let bytes = if location == "CHIP" { + layout(&[( + "CHIP", + vec![square(), label((900, 7), "KLAMATH_LAYER_NAMES_V2")], + )]) + } else { + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ]) + }; + let source = Document::parse(&bytes).unwrap().diagnostics[0].offset; + for scope in [ + Scope::default(), + Scope { + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + layer: Some((999, 0)), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + ] { + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!(doc.diagnostics.len(), 1, "{scope:?}"); + let diagnostic = &doc.diagnostics[0]; + assert_eq!(diagnostic.code, "unsupported_metadata_version"); + assert_eq!(diagnostic.cell.as_deref(), Some(location)); + assert_eq!(diagnostic.layer, Some((900, 7))); + assert_eq!(diagnostic.offset, source); + let diff = Comparison::between(&doc, &doc); + assert_eq!(diff.exit_code(), 2); + assert!(!diff.is_equal()); + assert!(diff.text(0).contains("INCOMPLETE")); + let json = diff.json_report(&doc, &doc, &inputs(), &ReportOptions::default()); + assert_eq!(json["status"], "incomplete"); + assert_eq!(json["result"], "unknown"); + assert_eq!(json["exit_code"], 2); + assert_eq!(json["diagnostics"].as_array().unwrap().len(), 2); + } + } +} + +#[test] +fn json_is_unbounded_by_default_and_explicit_filters_do_not_change_totals() { + let a = layout(&[("CHIP", vec![])]); + let b = layout(&[("CHIP", vec![square(); 120])]); + let full = report(&a, &b, &ReportOptions::default()); + assert_eq!( + full["cell_changes"][0]["added"].as_array().unwrap().len(), + 120 + ); + let limited = report( + &a, + &b, + &ReportOptions { + max_objects: Some(1), + ..Default::default() + }, + ); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["exit_code"], 1); + assert_eq!(limited["report_selection"]["truncated"], true); + assert_eq!(limited["report_selection"]["limited_objects"], "119"); + assert_eq!(limited["report_selection"]["emitted_objects"], "1"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: Some("absent".into()), + layer: None, + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["cell_changes"], json!([])); + assert_eq!(filtered["report_selection"]["filtered_cells"], "1"); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + assert_eq!(filtered["cell_changes"][0]["added"], json!([])); +} + +#[test] +fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { + let mut a = reference("leaf", 0, 1); + a.truncate(a.len() - 4); + record(&mut a, RTAG_MAG, &0x4110000000000000_u64.to_be_bytes()); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_ENDEL, &[]); + let mut b = a.clone(); + let mag = Document::parse(&layout(&[("CHIP", vec![a.clone()]), ("leaf", vec![])])) + .unwrap() + .records + .into_iter() + .find(|r| r.tag == RTAG_MAG) + .unwrap() + .offset; + // Replace the REAL8 payload within the object, retaining the same decoded value. + let within = a + .windows(8) + .position(|v| v == 0x4110000000000000_u64.to_be_bytes()) + .unwrap(); + b[within..within + 8].copy_from_slice(&0x4201000000000000_u64.to_be_bytes()); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + let r = report(&a, &b, &ReportOptions::default()); + let replacement = &r["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_kind_target"); + let change = &replacement["field_changes"][0]; + assert_eq!(change["tag"], RTAG_MAG); + assert_eq!(change["before"]["value"], change["after"]["value"]); + assert_ne!( + change["before"]["payload_hex"], + change["after"]["payload_hex"] + ); + let removed = &r["cell_changes"][0]["removed"][0]; + assert_eq!(removed["properties"].as_array().unwrap().len(), 2); + assert_eq!(removed["properties"][0]["attribute"], 17); + assert_eq!(removed["properties"][0]["value"], Value::Null); + assert_eq!(removed["properties"][0]["payload_hex"], "0xfffe"); + let offset: usize = removed["source"]["byte_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert!(offset < mag); + let end: usize = removed["source"]["end_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert_eq!(&a[end - 4..end], &[0, 4, 17, 0]); + + let mut path = square(); + path[2..4].copy_from_slice(&RTAG_PATH.to_be_bytes()); + let mut width = path.clone(); + width.truncate(width.len() - 4); + record(&mut width, RTAG_WIDTH, &0_i32.to_be_bytes()); + record(&mut width, RTAG_ENDEL, &[]); + let r = report( + &layout(&[("CHIP", vec![path])]), + &layout(&[("CHIP", vec![width])]), + &ReportOptions::default(), + ); + let change = &r["cell_changes"][0]["replacements"][0]["field_changes"][0]; + assert_eq!(change["tag"], RTAG_WIDTH); + assert_eq!(change["before"], Value::Null); + assert_eq!(change["after"]["value"], json!([0])); +} + +#[test] +fn hierarchy_reports_shared_ancestors_and_symbolic_arrays() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let r = report(&shared(square()), &shared(triangle()), &options); + let graph = &r["hierarchy"]; + assert_eq!(graph["nodes"].as_array().unwrap().len(), 4); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "CHIP") + .unwrap()["change"], + "affected" + ); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "leaf") + .unwrap()["change"], + "direct" + ); + let edge = graph["edges"] + .as_array() + .unwrap() + .iter() + .find(|e| e["parent"] == "right") + .unwrap(); + assert_eq!(edge["before_count"], "3"); + assert_eq!(edge["reference_changed"], false); + let bytes = shared(square()); + let scope = Scope { + cell: Some("CHIP".into()), + ..Default::default() + }; + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + let r = Comparison::between(&doc, &doc).json_report(&doc, &doc, &inputs(), &options); + assert_eq!(r["scope"]["cell"], "CHIP"); + assert_eq!( + r["hierarchy"]["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "left") + .unwrap()["after"], + "outside_scope" + ); + let dangling = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let r = report(&dangling, &dangling, &options); + assert_eq!(r["diagnostics"][0]["code"], "unresolved_reference"); + assert_eq!(r["diagnostics"][0]["target"], "missing"); + assert_eq!(r["hierarchy"]["nodes"][1]["after"], "unresolved"); + let cycle = layout(&[("CHIP", vec![reference("CHIP", 0, 1)])]); + assert_eq!( + report(&cycle, &cycle, &options)["diagnostics"][0]["code"], + "hierarchy_cycle" + ); +} + +#[test] +fn library_attributes_empty_cells_units_and_absence_are_explicit() { + let a = layout(&[]); + let b = layout(&[("empty", vec![])]); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["cell_changes"][0]["before"], Value::Null); + assert_eq!(r["cell_changes"][0]["after"]["attributes"], json!([])); + let mut b = b; + let end = Document::parse(&b).unwrap().cells["empty"].end_offset; + let mut attribute = Vec::new(); + record(&mut attribute, RTAG_STRCLASS, &[0, 1]); + b.splice(end..end, attribute); + let r = report(&layout(&[("empty", vec![])]), &b, &ReportOptions::default()); + assert_eq!( + r["cell_changes"][0]["after"]["attributes"][0]["value"], + json!([1]) + ); + assert_eq!(r["inputs"]["old"]["units"]["encoding"], "real8"); + let doc = Document::parse(&a).unwrap(); + let absent = doc.empty_like(); + let r = Comparison::between(&absent, &doc).json_report( + &absent, + &doc, + &inputs(), + &ReportOptions::default(), + ); + assert_eq!(r["inputs"]["old"]["present"], false); + assert_eq!(r["inputs"]["old"]["units"], Value::Null); + assert_eq!( + r["library_changes"], + json!([{"field":"presence","tag":null,"before":false,"after":true}]) + ); + let mut b = a.clone(); + let units = doc + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + b[units + 12..units + 20].copy_from_slice( + &pheon::basic::encode_real8(2e-9) + .unwrap() + .to_be_bytes(), + ); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["library_changes"][0]["tag"], RTAG_UNITS); + assert_ne!(r["inputs"]["old"]["units"], r["inputs"]["new"]["units"]); +} + +#[test] +fn reports_keep_extreme_coordinates_unsigned_layers_and_timestamp_equality() { + let a = layout(&[("quoted\"\\\n日本", vec![])]); + let mut shape = polygon(&[ + [i32::MIN, i32::MIN], + [i32::MAX, i32::MIN], + [i32::MAX, i32::MAX], + [i32::MIN, i32::MAX], + [i32::MIN, i32::MIN], + ]); + shape[8..10].copy_from_slice(&u16::MAX.to_be_bytes()); + let b = layout(&[("quoted\"\\\n日本", vec![shape])]); + let r = report(&a, &b, &ReportOptions::default()); + let object = &r["cell_changes"][0]["added"][0]; + assert_eq!(object["layer"]["number"], u16::MAX); + assert_eq!( + object["bounds"], + json!([i32::MIN, i32::MIN, i32::MAX, i32::MAX]) + ); + let encoded = serde_json::to_string(&r).unwrap(); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), r); + let mut times = b.clone(); + let doc = Document::parse(&b).unwrap(); + for record in doc + .records + .iter() + .filter(|r| matches!(r.tag, RTAG_BGNLIB | RTAG_BGNSTR)) + { + times[record.offset + 4..record.offset + 28].fill(1); + } + let r = report(&b, ×, &ReportOptions::default()); + assert_eq!(r["result"], "equal"); + assert_eq!(r["exit_code"], 0); +} + +#[cfg(unix)] +#[test] +fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { + use std::os::unix::ffi::OsStringExt; + let directory = std::env::temp_dir().join(format!("pheon json paths {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let path = directory.join(std::ffi::OsString::from_vec(b"file\xff.gds".to_vec())); + fs::write(&path, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(["--format", "json", "--"]) + .arg(&path) + .arg(&path) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + let r: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(r["inputs"]["old"]["path"], Value::Null); + let expected = "0x".to_string() + + &path + .as_os_str() + .as_encoded_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + assert_eq!(r["inputs"]["old"]["path_encoded_hex"], expected); + for name in ["--format", "json"] { + fs::write(directory.join(name), layout(&[])).unwrap(); + } + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .current_dir(&directory) + .args(["diff", "--", "--format", "json"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + assert!( + String::from_utf8(output.stdout) + .unwrap() + .starts_with("profile ") + ); +} + +#[test] +fn machine_reports_match_versioned_goldens() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let equal = layout(&[("CHIP", vec![square()])]); + let metadata = corpus() + .into_iter() + .find(|p| p.name == "layer name only") + .unwrap(); + let samples = [ + ("equal", report(&equal, &equal, &ReportOptions::default())), + ( + "geometry", + report(&shared(square()), &shared(triangle()), &options), + ), + ( + "metadata", + report(&metadata.old, &metadata.new, &ReportOptions::default()), + ), + ( + "incomplete", + report( + &unknown(square()), + &unknown(square()), + &ReportOptions::default(), + ), + ), + ( + "failed", + failure_report( + &inputs(), + &Scope::default(), + &ReportOptions::default(), + "invalid_input", + "truncated record", + Some("new"), + ), + ), + ]; + for (name, mut actual) in samples { + actual["producer"]["version"] = json!("test"); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join(format!("tests/golden/report-{name}.json")); + if std::env::var_os("PHEON_UPDATE_REPORT_GOLDENS").is_some() { + fs::write(&path, serde_json::to_string_pretty(&actual).unwrap() + "\n").unwrap(); + } + let expected: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(actual, expected, "{name}"); + } +} + +#[test] +fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { + let directory = std::env::temp_dir().join(format!("pheon json {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let a = directory.join("before file.gds"); + let b = directory.join("after file.gds"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); + let command = |extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(["--format", "json"]) + .args(extra) + .output() + .unwrap(); + assert!(output.stderr.is_empty(), "{:?}", output.stderr); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + output.status.code().unwrap(), + report["exit_code"].as_i64().unwrap() as i32 + ); + report + }; + let full = command(&["--hierarchy", "graph"]); + assert_eq!(full["result"], "different"); + assert!(full["hierarchy"].is_object()); + let limited = command(&["--max-changes", "0"]); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["report_selection"]["emitted_objects"], "0"); + for extra in [ + &["--top", "missing"][..], + &["--unknown"], + &["--max-changes", "x"], + &["--hierarchy", "mermaid"], + &["--depth", "3"], + ] { + let failure = command(extra); + assert_eq!(failure["status"], "failed"); + assert_eq!(failure["result"], "unknown"); + assert_eq!(failure["summary"], Value::Null); + } + fs::write(&b, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "new"); + assert_eq!(invalid["inputs"]["new"]["path"], b.to_str().unwrap()); + fs::remove_file(&b).unwrap(); + assert_eq!(command(&[])["diagnostics"][0]["code"], "io_error"); + fs::write(&b, fs::read(&a).unwrap()).unwrap(); + assert_eq!(command(&[])["result"], "equal"); + fs::write(&b, unknown(square())).unwrap(); + assert_eq!(command(&[])["status"], "incomplete"); + fs::write( + &b, + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ]), + ) + .unwrap(); + for flags in [ + &["--layer", "1/0"][..], + &["--top", "CHIP"], + &["--cell", "CHIP", "--layer", "1/0"], + ] { + let report = command(flags); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["exit_code"], 2); + assert_eq!( + report["diagnostics"][0]["code"], + "unsupported_metadata_version" + ); + } +} diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs new file mode 100644 index 0000000..8db1d81 --- /dev/null +++ b/tests/selective_reads.rs @@ -0,0 +1,768 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; +use arrow_array::{ + Array, BooleanArray, Float64Array, Int16Array, Int32Array, ListArray, StringArray, StructArray, + UInt32Array, UInt64Array, +}; +use common::{library_header, record}; +use pheon::{ + basic::encode_real8, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_cells_bytes, read_path, records, scan_bytes, +}; +use std::{ + ffi::{CStr, CString}, + ptr, +}; + +fn xy(values: &[i32]) -> Vec { + values.iter().flat_map(|v| v.to_be_bytes()).collect() +} + +fn fixture() -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_XY, &xy(&[3, -4])); + record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]); + record( + &mut bytes, + records::RTAG_MAG, + &encode_real8(2.0).unwrap().to_be_bytes(), + ); + record( + &mut bytes, + records::RTAG_ANGLE, + &encode_real8(90.0).unwrap().to_be_bytes(), + ); + record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]); + record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray { + let list = array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + list.value(row) + .as_any() + .downcast_ref::() + .unwrap() + .clone() +} + +fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + let counts: Vec = counts.into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, records::RTAG_COLROW, &counts); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +#[test] +fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { + let bytes = aref_fixture( + [i32::MIN, i32::MAX], + [i32::MAX - 1, i32::MIN + 1], + [i32::MIN, i32::MIN + 1], + [2, 2], + ); + let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0); + let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0); + for refs in [ + list_struct(&full_cells, "arefs", 0), + list_struct(&scan_cells, "refs", 0), + ] { + let column = u64_value(&refs, "xy0", 0); + let row = u64_value(&refs, "xy1", 0); + assert_eq!((column >> 32) as i32, i32::MAX); + assert_eq!(column as i32, -i32::MAX); + assert_eq!((row >> 32) as i32, 0); + assert_eq!(row as i32, -i32::MAX); + } +} + +#[test] +fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { + for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { + let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); + assert!(read_library(&bytes).is_err()); + assert!(scan_library(&bytes).is_err()); + } + let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); + assert!(read_library(&bytes).is_err()); + assert!(scan_library(&bytes).is_err()); +} + +#[test] +fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { + for (column, row, counts) in [ + ([10, 0], [0, 20], [3, 2]), + ([-10, 0], [0, 20], [3, 2]), + ([0, 10], [20, 0], [3, 2]), + ([20, 0], [0, 10], [2, 3]), + ([20, 0], [-10, 0], [2, 3]), + ] { + let bytes = aref_fixture([0, 0], column, row, counts); + let range = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for error in [ + read_library(&bytes).unwrap_err().1, + scan_library(&bytes).unwrap_err().1, + read_selected_library(&bytes, &range).unwrap_err().1, + ] { + assert!(format!("{error:?}").contains("not an integer database-unit increment")); + } + } + + let bytes = aref_fixture([0, 0], [10, 0], [0, 20], [3, 2]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let range = [library_header().len() as u64, bytes.len() as u64 - 4]; + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + range.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + let path = + std::env::temp_dir().join(format!("pheon-fractional-aref-{}.gds", std::process::id())); + struct Cleanup(std::path::PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + let _cleanup = Cleanup(path.clone()); + std::fs::write(&path, &bytes).unwrap(); + let path = CString::new(path.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { read_path(path.as_ptr(), &mut array, &mut schema) }, + -1 + ); + let message = unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap(); + assert!(message.contains("not an integer database-unit increment")); + assert!(array.is_released()); + + // The object reviewer keeps the exact endpoints, including fractional pitches. + let mut bytes = bytes; + bytes.truncate(bytes.len() - 4); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let doc = pheon::review::Document::parse(&bytes).unwrap(); + assert_eq!( + pheon::review::Comparison::between(&doc, &doc).exit_code(), + 0 + ); +} + +#[test] +fn rectangle_detection_handles_extreme_coordinates_and_preserves_other_polygons() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + let wide = [ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MIN, + ]; + let reversed = [ + i32::MIN, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MIN, + ]; + let diagonal = [0, 0, 10, 10, 0, 10, 10, 0, 0, 0]; + let degenerate = [0, 0, 0, 0, 10, 0, 10, 0, 0, 0]; + for points in [&wide, &reversed, &diagonal, °enerate] { + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + record(&mut bytes, records::RTAG_XY, &xy(points)); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let full = read_library(&bytes).unwrap().1; + let scan = scan_library(&bytes).unwrap().1; + assert_eq!( + full, + read_selected_library(&bytes, &ranges(&scan)).unwrap().1 + ); + let cells = list_struct(&full, "cells", 0); + let rects = list_struct(&cells, "rect_batches", 0); + let values = rects + .column_by_name("rects") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + &[ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX + ] + ); + let boundaries = list_struct(&cells, "boundary_batches", 0); + let values = boundaries + .column_by_name("vertices") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + [&diagonal[..8], °enerate[..8]].concat() + ); +} + +#[test] +fn paths_preserve_missing_explicit_zero_and_negative_widths() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for width in [None, Some(0_i32), Some(-100)] { + record(&mut bytes, records::RTAG_PATH, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, records::RTAG_WIDTH, &width.to_be_bytes()); + } + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 10])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, full) = read_library(&bytes).unwrap(); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + let paths = list_struct(&cells, "paths", 0); + let widths = paths + .column_by_name("width") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + widths.iter().collect::>(), + [None, Some(0), Some(-100)] + ); +} + +fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 { + array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row) +} + +fn ranges(scan: &StructArray) -> Vec<(u64, u64)> { + let cells = list_struct(scan, "cells", 0); + (0..cells.len()) + .map(|i| { + ( + u64_value(&cells, "struct_start_offset", i), + u64_value(&cells, "struct_end_offset", i), + ) + }) + .collect() +} + +#[test] +fn scanned_ranges_reproduce_the_full_library_including_offsets() { + let bytes = fixture(); + let (_, full) = read_library(&bytes).unwrap(); + let (scan_remaining, scan) = scan_library(&bytes).unwrap(); + assert!(scan_remaining.is_empty()); + let ranges = ranges(&scan); + assert_eq!(ranges.len(), 2); + for &(start, end) in &ranges { + assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]); + assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]); + } + let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap(); + assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged. + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8); +} + +#[test] +fn scan_preserves_reference_transforms_and_skips_geometry() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let cells = list_struct(&scan, "cells", 0); + let refs = list_struct(&cells, "refs", 0); + assert_eq!(refs.len(), 2); + assert_eq!(list_struct(&cells, "refs", 1).len(), 0); + let target = refs + .column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(target.values().as_ref(), &[1, 1]); + assert_eq!( + u64_value(&refs, "xy", 0), + (3_u64 << 32) | (-4_i32 as u32 as u64) + ); + assert_eq!(u64_value(&refs, "xy0", 0), 0); + assert_eq!(u64_value(&refs, "xy1", 0), 0); + let counts = refs + .column_by_name("counts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]); + assert!( + refs.column_by_name("invert_y") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1) + ); + assert_eq!( + refs.column_by_name("scale") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + 2.0 + ); + assert_eq!( + refs.column_by_name("angle_rad") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + std::f64::consts::FRAC_PI_2 + ); + assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32); + assert_eq!(u64_value(&refs, "xy1", 1), 10); +} + +#[test] +fn selections_support_subsets_reordering_and_empty_results() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap(); + let leaf_cells = list_struct(&leaf, "cells", 0); + assert_eq!(leaf_cells.len(), 1); + let leaf_names = leaf + .column_by_name("cell_names") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!( + leaf_names + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "leaf" + ); + let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap(); + let reordered_cells = list_struct(&reordered, "cells", 0); + assert_eq!( + u64_value(&reordered_cells, "file_offset", 0), + u64_value(&leaf_cells, "file_offset", 0) + ); + let refs = list_struct(&reordered_cells, "sref_props", 1); + assert_eq!( + refs.column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + let (_, empty) = read_selected_library(&bytes, &[]).unwrap(); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); +} + +#[test] +fn id_tables_preserve_encounter_order_and_selection_local_ids() { + let mut bytes = library_header(); + for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha", 9, 4)] { + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); + if name == "zeta" { + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"alpha\0"); + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &datatype.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + } + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, full) = read_library(&bytes).unwrap(); + let cell_ranges = ranges(&scan); + + let check = |library: &StructArray, names: &[&str], layers: &[u32], ids: &[u32]| { + let values = |name| { + library + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + }; + let name_values = values("cell_names"); + assert_eq!( + name_values + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(Option::unwrap) + .collect::>(), + names + ); + let cells = list_struct(library, "cells", 0); + assert_eq!( + cells + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + ids + ); + if library.column_by_name("layers").is_some() { + let layer_values = values("layers"); + assert_eq!( + layer_values + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + layers + ); + } + }; + let layer_a = (9 << 16) | 4; + let layer_b = (2 << 16) | 8; + check( + &full, + &["zeta", "alpha", "middle"], + &[layer_a, layer_b], + &[0, 2, 1], + ); + check(&scan, &["zeta", "alpha", "middle"], &[], &[0, 2, 1]); + let (_, all_selected) = read_selected_library(&bytes, &cell_ranges).unwrap(); + assert_eq!(all_selected, full); + let (_, subset) = read_selected_library(&bytes, &cell_ranges[1..]).unwrap(); + check(&subset, &["middle", "alpha"], &[layer_b, layer_a], &[0, 1]); + let (_, reversed) = + read_selected_library(&bytes, &[cell_ranges[2], cell_ranges[1], cell_ranges[0]]).unwrap(); + check( + &reversed, + &["alpha", "middle", "zeta"], + &[layer_a, layer_b], + &[0, 1, 2], + ); +} + +#[test] +fn streamed_and_buffered_properties_preserve_duplicates_order_and_empty_lists() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for (kind, properties) in [ + (records::RTAG_BOUNDARY, true), + (records::RTAG_PATH, true), + (records::RTAG_PATH, false), + ] { + record(&mut bytes, kind, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + let vertices = if kind == records::RTAG_BOUNDARY { + xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]) + } else { + xy(&[0, 0, 10, 10]) + }; + record(&mut bytes, records::RTAG_XY, &vertices); + if properties { + for (key, value) in [(7_i16, b"first\0".as_slice()), (7, b"second"), (9, b"")] { + record(&mut bytes, records::RTAG_PROPATTR, &key.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, value); + } + } + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + + let (remaining, full) = read_library(&bytes).unwrap(); + assert!(remaining.is_empty()); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(selected, full); + let cells = list_struct(&full, "cells", 0); + let boundaries = list_struct(&cells, "boundary_props", 0); + let paths = list_struct(&cells, "paths", 0); + assert_eq!(boundaries.len(), 1); + assert_eq!(paths.len(), 2); + for elements in [&boundaries, &paths] { + let properties = list_struct(elements, "properties", 0); + let keys = properties + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(keys.values().as_ref(), &[7, 7, 9]); + let values = properties + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + values.iter().collect::>(), + [Some("first"), Some("second"), Some("")] + ); + } + let empty_properties = paths + .column_by_name("properties") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!empty_properties.is_null(1)); + assert_eq!(empty_properties.value_length(1), 0); +} + +#[test] +fn selected_ranges_must_cover_exactly_one_complete_cell() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (start, end) = ranges[0]; + for range in [ + (end, start), + (start, u64::MAX), + (start, end - 1), + (start + 4, end), + (start, end + 4), + (start, ranges[1].1), + ] { + assert!( + read_selected_library(&bytes, &[range]).is_err(), + "accepted {range:?}" + ); + } +} + +#[test] +fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + 0 + ); + let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); + + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + 0 + ); + let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + let ranges: Vec = ranges(&scan) + .into_iter() + .flat_map(|(a, b)| [a, b]) + .collect(); + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + ranges.len() / 2, + &mut array, + &mut schema, + ) + }, + 0 + ); + let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(selected, read_library(&bytes).unwrap().1); + assert!(last_error_message().is_null()); +} + +#[test] +fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("scan failed") + ); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) }, + -1 + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("ranges pointer was null") + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + usize::MAX, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("range count is too large") + ); + assert!(array.is_released()); +} diff --git a/tests/writing.rs b/tests/writing.rs new file mode 100644 index 0000000..caf8816 --- /dev/null +++ b/tests/writing.rs @@ -0,0 +1,138 @@ +use arrow_array::Array; +use pheon::{ + basic::{pack_bitarray, pack_int2, pack_int4, pack_real8}, + library::{FileHeader, read_library}, + record::{Int2Array, Int4Array, Record, RecordData, RecordHeader}, + records, +}; +use std::io::{self, Write}; + +#[test] +fn record_headers_include_their_own_size() { + let mut bytes = Vec::new(); + assert_eq!(records::HEADER::write(&mut bytes, &600).unwrap(), 6); + assert_eq!(bytes, [0, 6, 0, 2, 2, 88]); + let (remaining, value) = records::HEADER::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(value, 600); + let maximum = RecordHeader { + data_size: u16::MAX - 4, + tag: 0x1003, + }; + assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]); +} + +#[test] +fn integer_array_records_round_trip() { + let mut bytes = Vec::new(); + records::COLROW::write(&mut bytes, &[2, 3]).unwrap(); + let (remaining, values) = records::COLROW::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(values.collect::>(), [2, 3]); + + let mut bytes = Vec::new(); + let values = [i32::MIN, -1, 0, i32::MAX]; + records::XY::write(&mut bytes, &values).unwrap(); + let (remaining, parsed) = records::XY::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed.collect::>(), values); +} + +#[test] +fn scalar_packers_write_only_their_prefix_and_check_short_buffers() { + type Case = (fn(&mut [u8]), &'static [u8]); + let cases: &[Case] = &[ + (|bytes| pack_bitarray(bytes, &[true; 16]), &[0xff, 0xff]), + (|bytes| pack_int2(bytes, -2), &[0xff, 0xfe]), + (|bytes| pack_int4(bytes, -2), &[0xff, 0xff, 0xff, 0xfe]), + ( + |bytes| pack_real8(bytes, 1.0).unwrap(), + &[0x41, 0x10, 0, 0, 0, 0, 0, 0], + ), + ]; + for (pack, expected) in cases { + let mut bytes = [0xa5; 16]; + pack(&mut bytes); + assert_eq!(&bytes[..expected.len()], *expected); + assert!(bytes[expected.len()..].iter().all(|&byte| byte == 0xa5)); + let mut short = vec![0xa5; expected.len() - 1]; + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| pack(&mut short))).is_err() + ); + assert!(short.iter().all(|&byte| byte == 0xa5)); + } + // Invalid REAL8 values are rejected before accessing the output buffer. + assert!(pack_real8(&mut [], f64::INFINITY).is_err()); +} + +#[test] +fn array_packers_reject_nonexact_buffers_before_writing() { + for width in [2, 4] { + let required = 2 * width; + for size in [required - 1, required + 1, required + width] { + let mut bytes = vec![0xa5; size]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if width == 2 { + Int2Array::pack_into(&mut bytes, &[-1, 1]); + } else { + Int4Array::pack_into(&mut bytes, &[-1, 1]); + } + })); + assert!(result.is_err()); + assert!(bytes.iter().all(|&byte| byte == 0xa5)); + } + } +} + +#[test] +fn string_records_pad_only_when_needed_and_support_empty_strings() { + for value in [b"".as_slice(), b"odd", b"even"] { + let mut bytes = Vec::new(); + let expected_size = 4 + value.len() + value.len() % 2; + assert_eq!( + records::LIBNAME::write(&mut bytes, value).unwrap(), + expected_size + ); + assert_eq!(bytes.len(), expected_size); + let (remaining, parsed) = records::LIBNAME::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed, value); + if value.len() % 2 == 1 { + assert_eq!(bytes.last(), Some(&0)); + } + } +} + +struct ShortWriter(Vec); + +impl Write for ShortWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let size = bytes.len().min(1); + self.0.extend_from_slice(&bytes[..size]); + Ok(size) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn library_header_round_trips_through_a_short_writer() { + let header = FileHeader::new(b"odd", 1e-9, 0.001); + let mut writer = ShortWriter(Vec::new()); + let mut written = header.write(&mut writer).unwrap(); + written += records::ENDLIB::write(&mut writer, &()).unwrap(); + assert_eq!(written, writer.0.len()); + let (remaining, library) = read_library(&writer.0).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(library.len(), 1); +} + +#[test] +fn oversized_records_fail_before_writing() { + let mut bytes = Vec::new(); + let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(bytes.is_empty()); +} diff --git a/tools/check_installed.py b/tools/check_installed.py new file mode 100644 index 0000000..707ecbf --- /dev/null +++ b/tools/check_installed.py @@ -0,0 +1,26 @@ +"""Run outside the checkout, without PHEON_LIBRARY, after installing a wheel.""" +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +# A separate interpreter verifies that base packing does not import PyArrow. +subprocess.run( + [sys.executable, "-I", "-c", """ +import importlib.util +import sys +import pheon +from pheon._native import library +assert 'pyarrow' not in sys.modules +assert importlib.util.find_spec('klamath_rs_ext') is None +assert library.parent == __import__('pathlib').Path(pheon.__file__).parent +assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') +"""], + check=True, +) +suite = unittest.defaultTestLoader.discover(str(ROOT / "tests/python")) +if not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(): + raise SystemExit(1) From a384bb2f8962924d06b1f001fabb6f1bdfc685ad Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 16:51:41 -0800 Subject: [PATCH 16/41] Organize documentation into focused user guides --- .github/workflows/packages.yml | 2 +- ARROW_SCHEMA.md | 154 -------------- MANIFEST.in | 2 +- README.md | 358 ++++----------------------------- docs/arrow-schema.md | 133 ++++++++++++ docs/cli.md | 94 +++++++++ docs/comparison.md | 58 ++++++ docs/development.md | 45 +++++ docs/diff-reports.md | 243 +++++++++------------- docs/git.md | 59 ++++++ docs/index.md | 19 ++ docs/layer-names.md | 44 ++++ docs/native-api.md | 68 +++++++ docs/python.md | 97 +++++++++ docs/releases.md | 65 +++--- 15 files changed, 779 insertions(+), 662 deletions(-) delete mode 100644 ARROW_SCHEMA.md create mode 100644 docs/arrow-schema.md create mode 100644 docs/cli.md create mode 100644 docs/comparison.md create mode 100644 docs/development.md create mode 100644 docs/git.md create mode 100644 docs/index.md create mode 100644 docs/layer-names.md create mode 100644 docs/native-api.md create mode 100644 docs/python.md diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 3a0c0bc..710c22f 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -116,7 +116,7 @@ jobs: cp target/release/pheon cli-dist/ fi cp LICENSE.md README.md cli-dist/ - cp -R contrib cli-dist/ + cp -R contrib docs cli-dist/ - uses: actions/upload-artifact@v4 with: name: cli-${{ matrix.os }} diff --git a/ARROW_SCHEMA.md b/ARROW_SCHEMA.md deleted file mode 100644 index 14aa848..0000000 --- a/ARROW_SCHEMA.md +++ /dev/null @@ -1,154 +0,0 @@ -# Arrow Schema Documentation - -Pheon's native `pheon_arrow_schema_version()` returns 1 for the full-read, -scan and selected-read layouts documented here. This version is independent -of the C API version, package release and CLI JSON report schema. The stable -Arrow C Data Interface transports these application-specific layouts; consumers -must check both the native API and application schema before reading data. - -This project uses Apache Arrow to represent GDSII (Graphic Data System II) library data. The schema is optimized for memory efficiency and fast access, employing several packed representations for common data types. - -## Top-Level Structure - -The top-level object is a `StructArray` representing the entire GDSII library. - -| Field | Type | Description | -|---|---|---| -| `meters_per_db_unit` | `Float64` | Database unit size in meters. | -| `user_units_per_db_unit` | `Float64` | Number of user units in one database unit. | -| `lib_name` | `Utf8` | The name of the GDSII library. | -| `mod_time` | `FixedSizeList(Int16, 6)` | Last modification time [year, month, day, hour, minute, second]. | -| `acc_time` | `FixedSizeList(Int16, 6)` | Last access time [year, month, day, hour, minute, second]. | -| `cells` | `List(Cell)` | A list of cells (structures) in the library. | -| `cell_names` | `List(Utf8)` | Names of all cells, indexed by their ID. | -| `layers` | `List(UInt32)` | List of packed layer/datatype pairs, indexed by their ID. | - -## Cell Structure - -Each cell in the `cells` list is a `StructArray`. - -| Field | Type | Description | -|---|---|---| -| `id` | `UInt32` | Unique identifier for the cell. | -| `file_offset` | `UInt64` | Byte offset immediately after the cell's `STRNAME` record (the first element, or `ENDSTR` for an empty cell). Identical for full and selected reads. | -| `srefs` | `List(SRef)` | Simple structure references (no properties). | -| `arefs` | `List(ARef)` | Array references (no properties). | -| `sref_props` | `List(SRefProp)` | Structure references with properties. | -| `aref_props` | `List(ARefProp)` | Array references with properties. | -| `rect_batches` | `List(RectBatch)` | Batches of rectangular boundaries (optimized). | -| `boundary_batches` | `List(BoundaryBatch)` | Batches of generic boundaries. | -| `boundary_props` | `List(BoundaryProp)` | Individual boundaries with properties. | -| `paths` | `List(Path)` | Path elements. | -| `nodes` | `List(BoxNode)` | Node elements. | -| `boxes` | `List(BoxNode)` | Box elements. | -| `texts` | `List(Text)` | Text labels. | - -## Hierarchy Scan Schema - -`scan_library` and `scan_bytes` return the same library metadata and `cell_names` -as a full read, with the following `cells` schema and no `layers` field: - -| Field | Type | Description | -|---|---|---| -| `id` | `UInt32` | Cell ID indexing this result's `cell_names`. | -| `struct_start_offset` | `UInt64` | Inclusive byte offset of the `BGNSTR` record. | -| `struct_end_offset` | `UInt64` | Exclusive byte offset immediately after `ENDSTR`. | -| `refs` | `List(ScanRef)` | SREFs and AREFs, including references with properties, sorted by target ID. | - -Each scan reference has `target`, `invert_y`, `scale`, `angle_rad`, `xy`, `xy0`, -`xy1`, and `counts`, using the ARef types and packed encodings described below. -SREFs use zero basis vectors and counts `(1, 1)`. Properties are omitted from the -scan output; geometry and text are skipped without decoding. - -Pass the half-open scan ranges to `read_selected_library` or `read_cells_bytes` -to obtain the full library schema for just those cells. Each range must contain -exactly one complete cell, from `BGNSTR` through `ENDSTR`. Cells appear in the -requested order. Cell and layer IDs are assigned within each result, so use -`cell_names` and `layers` from that result when resolving them; IDs can differ -from a full read or a scan. An empty selection returns metadata and empty lists. -The Rust random-access reader returns the original input slice as its remainder. - -## Element Types - -### Common Packed Encodings - -- **Layer ID (`UInt32`)**: A packed representation of `(layer, datatype)`. - - `layer = (layer_id >> 16) as i16` - - `datatype = (layer_id & 0xFFFF) as i16` -- **XY Coordinate (`UInt64`)**: A packed representation of a single point `(x, y)`. - - `x = (xy >> 32) as i32` - - `y = (xy & 0xFFFFFFFF) as i32` -- **Array Counts (`UInt32`)**: A packed representation of `(columns, rows)`. - - `columns = (counts >> 16) as i16` - - `rows = (counts & 0xFFFF) as i16` - -### Reference Elements (SRef / ARef) - -References point to other cells in the library. - -**SRef (Simple Reference)**: -- `target`: `UInt32` (Cell ID) -- `invert_y`: `Boolean` (Vertical flip) -- `scale`: `Float64` (Magnification) -- `angle_rad`: `Float64` (Rotation in radians) -- `xy`: `UInt64` (Packed origin point) - -**ARef (Array Reference)**: -- Same as SRef, plus: -- `xy0`: `UInt64` (Packed column basis vector increment) -- `xy1`: `UInt64` (Packed row basis vector increment) -- `counts`: `UInt32` (Packed columns and rows) - -Array increments must be exact signed 32-bit integer database-unit values. Full -reads, hierarchy scans, selected reads, and their FFI entry points reject arrays -whose endpoint/count division is fractional or outside that range. They never -round or truncate placements. The review CLI retains original endpoints and can -compare fractional arrays independently of this Arrow representation. - -**SRefProp / ARefProp**: -- Same as above, plus a `properties` field. - -### Boundary Elements - -Boundaries are grouped into batches for efficiency when they share properties or are simple rectangles. - -**RectBatch**: -- `layer`: `UInt32` (Packed Layer ID) -- `rects`: `List(Int32)` (Flattened list of `[min_x, min_y, max_x, max_y]` for each rectangle) - -**BoundaryBatch**: -- `layer`: `UInt32` (Packed Layer ID) -- `vertices`: `List(Int32)` (Flattened list of all vertices for all boundaries in the batch) -- `vertex_offsets`: `List(UInt32)` (Start index for each boundary in the `vertices` list) - -**BoundaryProp**: -- Individual boundary with `properties`. - -### Other Elements - -**Path**: -- `layer`: `UInt32` -- `path_type`: `Int16` (Optional) -- `extension_start`: `Int32` (Optional) -- `extension_end`: `Int32` (Optional) -- `width`: `Int32` (Optional; null when `WIDTH` is absent) -- `xy`: `List(Int32)` (centerline coordinates) -- `properties`: `List(Property)` - -**Text**: -- `layer`: `UInt32` -- `presentation_horiz`: `UInt8` -- `presentation_vert`: `UInt8` -- `presentation_font`: `UInt8` -- `path_type`: `Int16` -- `width`: `Int32` -- `invert_y`: `Boolean` -- `mag`: `Float64` -- `angle_deg`: `Float64` -- `xy`: `UInt64` (Packed position) -- `string`: `Utf8` -- `properties`: `List(Property)` - -**Property**: -- `key`: `Int16` -- `value`: `Utf8` diff --git a/MANIFEST.in b/MANIFEST.in index a552f5f..ff57a3d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include Cargo.toml Cargo.lock rust-toolchain.toml README.md ARROW_SCHEMA.md LICENSE.md +include Cargo.toml Cargo.lock rust-toolchain.toml README.md LICENSE.md recursive-include src *.rs recursive-include pheon *.py *.h py.typed recursive-include tests *.rs *.py *.json *.txt *.mmd diff --git a/README.md b/README.md index 885fc01..55285ab 100644 --- a/README.md +++ b/README.md @@ -1,337 +1,57 @@ # Pheon -A fast GDSII parser and writer in Rust, with Apache Arrow integration for efficient data handling. +Pheon reads GDSII layouts into Apache Arrow and provides command-line tools for +inspecting and comparing layout revisions. It is written in Rust, with Python +bindings and a C API. -Pheon provides native GDSII codecs and a standalone layout review CLI. The Rust -crate, Python package and executable are all named `pheon`. Python Klamath and -Masque remain separate consumers. The former `klamath_rs_ext` import and -`klamath` executable are not provided as aliases. +- Inspect cells, records, and reference hierarchies. +- Compare layout objects, with text or JSON reports and Git integration. +- Read geometry into Arrow, or scan the hierarchy before loading selected cells. -## Overview +Comparisons ignore timestamps and element order while preserving duplicates and +other object details. They compare object fields and coordinates exactly; see the +[comparison rules](docs/comparison.md) for supported normalization and limits. -`pheon` is designed to handle large GDSII files with high performance. It uses Apache Arrow as its primary in-memory data representation, allowing for efficient memory usage and easy integration with other tools in the Arrow ecosystem (like pandas, polars, or DuckDB via Python bindings). +## Try the command-line tools -## Documentation - -- [Arrow Schema Documentation](ARROW_SCHEMA.md): Detailed information about the Arrow schema used for GDSII data. - -## Features - -- Fast parsing of GDSII files. -- Efficient memory representation using Apache Arrow. -- Support for common GDSII elements: SREF, AREF, Boundary, Path, Text, etc. -- Optimized "batch" representations for rectangles and boundaries sharing the same layer. -- C-compatible FFI for integration with other languages (e.g., Python). - -REAL8 writing uses normalized GDS encodings and represents every in-range `f64` -exactly. Finite magnitudes below `2^-260` underflow to canonical zero; nonfinite -values and magnitudes at or above `2^252` are rejected by the encoder. - -## Usage - -`read_path` exports a library through the Arrow C Data Interface. Allocate -initialized `ArrowArray` and `ArrowSchema` output structures, then check the -returned status: `0` means success and `-1` means failure. Only import the Arrow -outputs after success; the caller owns their release callbacks. Failed calls -leave the outputs unchanged. `pheon/_ffi.h` declares the C API; `test.py` -demonstrates the supported Python reader. - -`last_error_message()` returns a UTF-8, null-terminated error string, or null -after a successful parser call. Errors belong to the calling thread. Copy the -message before making another parser call on that thread, and before the thread -exits; calls on other threads do not invalidate it. Parser panics that unwind -are converted into errors. - -For partial reads, call `scan_bytes` with a borrowed GDSII byte buffer to obtain -cell ranges and references without decoding geometry. Pass the desired ranges -to `read_cells_bytes` to load their full contents. Both calls use the same Arrow -outputs and status/error handling as `read_path`: - -```c -int scan_bytes(const uint8_t *data, size_t size, - struct ArrowArray *array, struct ArrowSchema *schema); -int read_cells_bytes(const uint8_t *data, size_t size, - const uint64_t *ranges, size_t range_count, - struct ArrowArray *array, struct ArrowSchema *schema); -``` - -`ranges` contains alternating inclusive start and exclusive end offsets, with -two native-endian `uint64_t` values per cell. Each range must cover one complete -cell. A null `ranges` pointer is accepted when `range_count` is zero. Likewise, -null `data` is accepted for zero bytes and produces an ordinary parse error for -the empty input. Nonempty buffers must be readable for the duration of the call; -range values must be aligned. Exported Arrow data owns its storage independently -of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan -schema, offset definitions, and ID assignment. - -## Python package and validation - -Python 3.11 or newer is required. Source builds require Rust 1.88 or newer; -installing a prebuilt wheel requires no Rust compiler. The shared library uses -a C ABI rather than Python's or Rust's native ABI, so one platform wheel serves -multiple Python minor versions. CFFI, NumPy and optional PyArrow must themselves -support the selected interpreter. A Rust application uses the crate normally -and compiles it with its own compatible toolchain. The checkout selects the -stable toolchain locally through `rust-toolchain.toml`, without changing your -global default. CI checks both Rust 1.88 and current stable. +From a source checkout, install the CLI with Rust 1.88 or newer: ```sh -python3 -m pip install 'pheon[arrow]' +cargo install --path . --locked --bin pheon +pheon dump chip.gds +pheon diff before.gds after.gds +pheon hierarchy chip.gds --format mermaid ``` +`diff` returns 0 for equality, 1 for differences, and 2 for invalid or incomplete +comparisons. Start with the [CLI guide](docs/cli.md), then set up +[Git integration](docs/git.md) if you review layouts in a repository. + +## Read layouts in Python + +Python 3.11 or newer is required. From a source checkout: + +```sh +python -m pip install '.[arrow]' +``` + +Run Python outside the source checkout to use the installed package: + ```python from pheon import arrow -library = arrow.read_path('chip.gds') # one-row StructArray; also accepts gzip -scan = arrow.scan_bytes(gds_bytes) -cells = scan.field('cells')[0].values -ranges = list(zip(cells.field('struct_start_offset').to_pylist(), - cells.field('struct_end_offset').to_pylist())) -selected = arrow.read_cells_bytes(gds_bytes, ranges[:2]) +layout = arrow.read_path("chip.gds") +print(layout.field("cell_names")[0].as_py()) ``` -`read_bytes` reads a complete borrowed buffer. Full, scanned and selected Arrow -results own their storage independently of that buffer. `PheonError` reports -native parse failures; malformed range shapes and types raise Python value/type -errors before calling native code. Integer packing needs no PyArrow import. +Source installation needs Rust; prebuilt Python wheels bundle the native library. +The [Python guide](docs/python.md) covers installation, selected reads, and integer +packing. The [Arrow reference](docs/arrow-schema.md) describes the returned data. -### Discovery and companion packages +## Learn more -Klamath's `native` extra installs `pheon`; Masque's `arrow` extra installs -`pheon[arrow]`. Klamath imports Pheon's packing helpers, while Masque calls -`pheon.arrow` and translates parse failures to its existing error type. +Browse the [documentation](docs/index.md) for layer names, native integration, +JSON reports, development, and releases. For writing, Pheon provides low-level +GDSII record helpers in Rust. -The Python loader selects the library bundled with the imported `pheon` package, -or the exact file selected by `PHEON_LIBRARY`. It checks C ABI version 1 and -Arrow layout schema version 1 before exposing native functions. Incompatible -or unversioned libraries fail with an import error. Those versions are independent -of package releases and the CLI's JSON report schema. - -`cargo install --path . --locked --bin pheon` installs the CLI, not the Python -package or a discoverable shared library. Klamath does not search PATH or Cargo -directories for it. To reuse an external native build, install the Python -components too and set `PHEON_LIBRARY`; an editable source install is useful -during development. The old library override variables are not recognized. - -The Python packing helpers require NumPy and CFFI, both declared as runtime -dependencies. Setuptools invokes `cargo build --release --lib` and bundles the -C ABI shared library in a platform-specific wheel. Its version comes from -`Cargo.toml`. The C declarations are in `pheon/_ffi.h`; the loader uses -CFFI ABI mode, so no Python extension compilation or Maturin is required. - -```sh -python3 -m pip install . -``` - -To build a source archive and a wheel with the `build` frontend and installed -Setuptools/Wheel tooling: - -```sh -python3 -m build --no-isolation -``` - -Source archives include a Cargo lockfile, and wheel builds use `--locked` so -building from an archive retains the resolved Rust dependency versions. Wheels -are tagged for the build platform and use the Python-independent `py3-none` ABI. -The CLI continues to build directly with Cargo and has no Python dependency. - -For source-tree development, point the CFFI loader at a Cargo-built library: - -```sh -cargo build --release --lib -PHEON_LIBRARY="$PWD/target/release/libpheon.so" \ - python3 -m unittest discover -s tests/python -v -``` - -Use `libpheon.dylib` on macOS or `pheon.dll` on Windows. Without -the explicit override, the loader uses the library bundled inside the package. -After installing a wheel, run its tests outside the source checkout to exercise -the installed package: - -```sh -python3 -I -m unittest discover -s /absolute/path/to/pheon/tests/python -v -``` - -The Python tests cover multidimensional and noncontiguous packing, input -preservation, range errors, and the C declarations for Arrow exports and release -callbacks, input lifetimes and ABI mismatches. Run `cargo test` and -`cargo test --release` for native validation. See [release validation](docs/releases.md) -for wheel builds, platform repair and testing the same artifact on multiple -Python versions. - -## Layout review CLI - -The review CLI is a standalone Rust executable: - -```sh -cargo build --bin pheon -target/debug/pheon dump chip.gds -target/debug/pheon dump chip.gds --records -target/debug/pheon diff old.gds new.gds --hierarchy ascii -target/debug/pheon diff old.gds new.gds --hierarchy mermaid --max-changes 20 -target/debug/pheon hierarchy chip.gds --format mermaid --depth 4 -target/debug/pheon diff old.gds new.gds --top CHIP --layer 1/0 -target/debug/pheon diff old.gds new.gds --show-layer 1/0 -target/debug/pheon diff old.gds new.gds --format json -target/debug/pheon diff old.gds new.gds --format json --hierarchy graph -target/debug/pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 --name 1:0:WG -``` - -The object dump is deterministic and omits source offsets. Record dumps preserve -record order and show offsets, tags, sizes, and values, including records the -object reviewer does not support. They can inspect an invalid metadata declaration -without invoking object-level validation. Zero padding after ENDLIB is accepted. - -The fixed `klamath-object-v2` comparison profile: - -- Matches cells by name and layers by numeric pair. The default scope is the - entire library; explicit selectors are recorded in each report. -- Ignores library/cell timestamps, cell order, element order, and property order. -- Normalizes boundary/box starting vertex and winding, retaining other vertices. -- Preserves duplicate objects and duplicate properties as multisets. -- Compares paths, labels, references, array endpoints/counts, properties, and all - retained flags exactly. Explicit defaults and their omission remain distinct; - SREFs and single-element AREFs remain distinct. REAL8 encodings are retained. -- Reports library fields, including UNITS, separately. It does not rescale files - into a shared grid, infer cell renames, or perform geometric Boolean operations. -- Uses bounding boxes as coordinate envelopes, not evidence of equality. Path - envelopes do not include stroke width; text/reference envelopes are coordinates, - not rendered or recursively transformed bounds. -- Returns an incomplete result for unsupported records, future metadata versions, - unresolved references, or cycles. These cannot produce an equality claim. - -`diff` exits with **0** for equality under this profile, **1** for differences, -and **2** for invalid input or incomplete comparison. `--max-changes` limits the -number of object details displayed, not the comparison. Source offsets are -included in changed-object reports for tracing changes back to record dumps. - -`diff --format json` emits a versioned report with typed before/after evidence, -structured diagnostics, and independent comparison status and completeness. -JSON includes all changes by default. Explicit display filters and `--max-changes` -record omitted evidence without changing totals or exit status. `--hierarchy graph` -adds the complete scoped reference graph, with symbolic array multiplicities; -`--depth` applies to text output only. Timestamp-only changes still return 0. -Input and argument errors also produce JSON when requested. See the -[report contract](docs/diff-reports.md) and -[version 1 JSON Schema](docs/diff-report-v1.schema.json). - -Unmatched objects are paired for explanation only when candidates are unique by -type/layer/bounding box, then by type/layer among the remaining objects. -References additionally match by target. Ambiguous groups stay additions and -removals. Reports explain reference movement, array-count changes, path widths, -and polygons changed within the same bounds. Full before/after records remain -visible, and duplicate counts remain significant. Unchanged layer names appear -once as a legend for changed objects on those layers. - -### Comparison scope and display filters - -`--top NAME` compares that cell and its reachable subtree independently in each -revision. `--cell NAME` compares the exact cell, including its reference records, -without comparing descendant contents. Combine the two to select a cell within -a subtree. A selector absent from both revisions is an error; absence on one -side is an addition/deletion. - -`--layer L/T` selects objects on a numeric layer/type pair, plus references whose -targets use that layer transitively. Cells with no relevant content are omitted. -Unresolved references, cycles, and unsupported records with unknown layer effect -remain incomplete results. Unsupported objects on another known layer can be -excluded. Library fields, including units, remain compared. Unsupported layer-name -version diagnostics are always retained, since those declarations can affect any -layer or cell in the library. Scoped dumps and -hierarchy views accept the same selectors. Scope selection parses the complete -input: malformed record grammar and conflicting metadata are still errors. - -`--show-cell NAME` and `--show-layer L/T` filter object details in a diff report -without changing comparison status, totals, metadata changes, or hierarchy. -References have no numeric layer and are hidden by `--show-layer`. The comparison -scope and any display-only filter are printed explicitly. The Rust API accepts -one combined `Scope`; applying another scope to an already scoped document is -rejected rather than losing reference coverage information. - -Hierarchy output distinguishes directly changed cells, affected ancestors, and -unchanged targets of modified references. It includes edges from both revisions, -so removed branches remain visible. Counts are direct edge multiplicities, -including arrays, rather than expanded placements. Shared definitions are marked -as already shown in text and appear once in Mermaid. `--depth` limits displayed -descendants only. The CLI emits Mermaid source, requiring no diagram renderer. - -Layer names are read from labels in an existing cell. A label with the exact -string `KLAMATH_LAYER_NAMES_V1` identifies an annotation carrier `(layer, texttype)` -pair within that cell. All other labels on that same pair in that same cell must -be `layer:datatype:name` declarations, for example `1:0:WG`. The carrier pair is -not fixed; choose a suitable annotation layer in the design's technology. -Declarations split at the first two colons, allowing colons in names. Numeric -fields are unsigned 16-bit decimal integers. Names are nonempty UTF-8 strings. -Repeated identical declarations are accepted; conflicting declarations fail. -Names are collected across the library. Recognized metadata labels are omitted -from object comparison; mappings are compared separately and printed once per -numeric pair, so renaming a layer does not rewrite every shape description. - -`name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ...` writes -this convention. It consolidates existing recognized declarations into one legend -in the chosen existing cell, retains existing names not explicitly updated, and -overrides the named pairs. The chosen carrier must contain no ordinary design -objects in that cell. Unrelated records, timestamps, unknown records and trailing -padding are copied byte-for-byte. No structures or references are added. Output -must be a new file: the CLI refuses to overwrite an existing destination, including -the input. Invalid declarations, conflicting arguments, oversized labels and -future metadata versions fail before output creation. - -The record-preserving review adapter retains information normalized or omitted -by the Arrow geometry reader; existing Arrow APIs are unchanged. Structured Rust -change records feed the text and hierarchy renderers. LayoutEditor's `LayerNames` -extension, layout extraction/remapping, and geometric Boolean comparison remain -outside the current CLI. - -### Git integration - -Install the executable on PATH with `cargo install --path . --bin pheon`. -The templates in [contrib/git/config](contrib/git/config) and -[contrib/git/attributes](contrib/git/attributes) provide two alternative drivers. -For structural summaries and affected hierarchy, add this attribute: - -```gitattributes -*.gds diff=pheon -``` - -Configure the native external driver in Git config: - -```gitconfig -[diff "pheon"] - command = pheon git-diff --hierarchy ascii -- - binary = true - trustExitCode = false -``` - -The adapter consumes Git's old/new temporary files directly, handles additions, -deletions and renames, and quotes display paths. It returns success after rendering -either equality or differences, as expected by the external-driver protocol. -Invalid or incomplete comparisons retain a nonzero exit and diagnostics. Scope -and display options can precede the final `--` in the configured command. Existing -repositories using `diff=layout` can put the same command under `[diff "layout"]`. - -For ordinary Git line diffs, use canonical textconv instead: - -```gitattributes -*.gds diff=pheon-text -``` - -Configure the driver in Git config: - -```gitconfig -[diff "pheon-text"] - textconv = pheon dump - binary = true - cachetextconv = true -``` - -This is a human review representation, not an applicable GDS patch. Invalidate -Git's textconv cache when changing the dump profile/tool version. For automated -equality checks, use `pheon diff` and its exit status directly. - -`tests/review.rs` contains ten initial fixture pairs, checked-in canonical/text/ -Mermaid goldens, metadata byte-preservation checks, scope and report regressions, -and actual Git integration tests. It also checks near-limit polygon records, -2048-cell hierarchies and arrays representing over a billion placements without -expanding them. Run `cargo test` and `cargo test --release` for debug and optimized -validation. No KLayout installation or Python package is required. +Licensed under [GPL-3.0-only](LICENSE.md). diff --git a/docs/arrow-schema.md b/docs/arrow-schema.md new file mode 100644 index 0000000..97ffecd --- /dev/null +++ b/docs/arrow-schema.md @@ -0,0 +1,133 @@ +# Arrow layout reference + +This reference is for consumers of Pheon's Arrow readers. Each result is a +one-row `StructArray` representing a GDSII library. The +[Python guide](python.md) shows how to obtain and access it. + +Geometry fields named `layer` are **indices into the result's `layers` table**. +Reference `target` and cell `id` values index its `cell_names` table. Resolve IDs +using tables from the same result; a selected read can assign different IDs. + +`pheon_arrow_schema_version()` currently returns 1 for all layouts below. +Consumers of the [C API](native-api.md) must check both this application schema +version and the separate C API version. + +## Full library + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `meters_per_db_unit` | Float64 | Database unit size in meters. | +| `user_units_per_db_unit` | Float64 | User units per database unit. | +| `lib_name` | Utf8 | Library name. | +| `mod_time`, `acc_time` | FixedSizeList(Int16, 6) | Modification/access time: year, month, day, hour, minute, second. | +| `cells` | List(Cell) | Cell records. | +| `cell_names` | List(Utf8) | Names indexed by cell ID. | +| `layers` | List(UInt32) | Packed layer/type pairs indexed by layer ID. | + +### Cell + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into `cell_names`. | +| `file_offset` | UInt64 | Byte offset immediately after STRNAME, before the first element or ENDSTR. | +| `srefs`, `arefs` | List(SRef), List(ARef) | References without properties. | +| `sref_props`, `aref_props` | List(SRefProp), List(ARefProp) | References with properties. | +| `rect_batches` | List(RectBatch) | Optimized rectangular boundaries. | +| `boundary_batches` | List(BoundaryBatch) | Other boundaries without properties. | +| `boundary_props` | List(BoundaryProp) | Boundaries with properties. | +| `paths` | List(Path) | Paths. | +| `nodes`, `boxes` | List(BoxNode) | Node and box elements. | +| `texts` | List(Text) | Text labels. | + +## Hierarchy scan and selected reads + +Scans return the same metadata and `cell_names`, omit `layers`, and replace +`cells` with these records: + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into this result's `cell_names`. | +| `struct_start_offset` | UInt64 | Inclusive offset of BGNSTR. | +| `struct_end_offset` | UInt64 | Exclusive offset after ENDSTR. | +| `refs` | List(ScanRef) | References sorted by target ID, with properties omitted. | + +ScanRef has ARef's fields below. An SREF uses zero basis vectors and counts +`(1, 1)`. Geometry and text are skipped without decoding. + +Pass complete half-open cell ranges to `read_selected_library` in Rust or +`read_cells_bytes` in Python/C. The result uses the full library layout, with +cells in requested order. An empty selection returns metadata and empty lists. +Offsets always refer to the original uncompressed input. The Rust selected +reader returns the original input slice as its remainder. + +## Packed values + +| Value | Storage | Decoding | +| --- | --- | --- | +| Layer/type pair in `layers` | UInt32 | Layer in high 16 bits; datatype/texttype/boxtype/nodetype in low 16 bits. | +| Point or vector | UInt64 | Signed Int32 x in high 32 bits, signed Int32 y in low 32 bits. | +| Array counts | UInt32 | Columns in high 16 bits, rows in low 16 bits. | + +Layer halves preserve the original 16-bit patterns; reinterpret them as signed +Int16 where an integration uses signed GDS layer values. Array counts originate +from positive Int16 values. Flattened coordinate lists below use `[x, y, ...]` +in integer database units. + +## References + +| SRef field | Arrow type | Meaning | +| --- | --- | --- | +| `target` | UInt32 | Target cell ID. | +| `invert_y` | Boolean | Reflection flag. | +| `scale` | Float64 | Magnification. | +| `angle_rad` | Float64 | Rotation in radians. | +| `xy` | UInt64 | Packed origin. | + +ARef adds `xy0` and `xy1` (UInt64 packed column/row increments) and `counts` +(UInt32). SRefProp and ARefProp add `properties: List(Property)`. + +Array increments must be exact signed 32-bit integers. All Arrow readers reject +fractional or out-of-range endpoint/count division rather than rounding it. +One-placement AREFs can be normalized to SREFs. The review CLI retains original +endpoints and distinguishes SREF from AREF independently of this representation. + +## Geometry and text + +Every `layer` below is a UInt32 **table index**, not a packed pair. + +| Type | Fields beyond `layer` | +| --- | --- | +| RectBatch | `rects: List(Int32)`, flattened `[min_x, min_y, max_x, max_y]` rectangles. | +| BoundaryBatch | `vertices: List(Int32)`; `vertex_offsets: List(UInt32)` gives each boundary's start in points, not scalar coordinates. | +| BoundaryProp | `vertices: List(Int32)`; `properties: List(Property)`. | +| BoxNode | `xy: List(Int32)`; `properties: List(Property)`. | +| Path | Fields below. | +| Text | Fields below. | + +| Path field | Arrow type | Meaning | +| --- | --- | --- | +| `path_type` | Int16, nullable | Stored path type. | +| `extension_start`, `extension_end` | Int32, nullable | End extensions. | +| `width` | Int32, nullable | Null when WIDTH is absent. | +| `xy` | List(Int32) | Centerline coordinates. | +| `properties` | List(Property) | Element properties. | + +| Text field | Arrow type | Meaning | +| --- | --- | --- | +| `presentation_horiz`, `presentation_vert`, `presentation_font` | UInt8, nullable | Presentation attributes. | +| `path_type` | Int16, nullable | Stored path type. | +| `width` | Int32, nullable | Stored width. | +| `invert_y` | Boolean, nullable | Reflection flag. | +| `mag` | Float64, nullable | Magnification. | +| `angle_deg` | Float64, nullable | Rotation in degrees. | +| `xy` | UInt64 | Packed position. | +| `string` | Utf8 | Label text. | +| `properties` | List(Property) | Element properties. | + +Property contains `key: Int16` and `value: Utf8`. The reader preserves property +order and duplicates. Property-list fields are nullable in the schema; the +reader normally emits valid lists, including empty ones. + +Geometry batching and reference normalization mean this layout is not a verbatim +record stream; use the [CLI's record dump](cli.md#inspect-a-layout) when original +records matter. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..843bd68 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,94 @@ +# Command-line tools + +Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. +The CLI runs independently of Python. + +## Install + +From a source checkout, with Rust 1.88 or newer: + +```sh +cargo install --path . --locked --bin pheon +pheon --help +``` + +You can also build locally with `cargo build --release --locked --bin pheon` +and run `target/release/pheon` (`pheon.exe` on Windows). + +## Inspect a layout + +```sh +pheon dump chip.gds +pheon dump chip.gds --records +pheon hierarchy chip.gds --format ascii --depth 4 +pheon hierarchy chip.gds --format mermaid --top CHIP +``` + +`dump` shows a normalized layout description. Add `--records` to inspect raw +records, including timestamps. `hierarchy` shows cell references without +expanding array placements. Shared cells are marked rather than repeated +indefinitely. Mermaid output is diagram source for a Mermaid renderer. + +## Compare revisions + +```sh +pheon diff before.gds after.gds +pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 +pheon diff before.gds after.gds --format json > report.json +``` + +Reports identify changed objects and their fields, plus affected ancestors in +the hierarchy. Objects that can be paired unambiguously receive a replacement +explanation; ambiguous changes remain removals and additions. + +| Exit code | Meaning | +| --- | --- | +| 0 | Equal under the comparison rules and selected scope. | +| 1 | Differences found in a complete comparison. | +| 2 | Invalid input, a command error, or an incomplete comparison. | + +A code of 1 is an expected comparison result. A code of 2 can accompany definite +differences, but equality cannot be established. Read the +[comparison rules](comparison.md) before relying on results in automation, and +the [JSON reference](diff-reports.md) to consume reports programmatically. + +## Choose the comparison scope + +```sh +pheon diff before.gds after.gds --top CHIP --layer 1/0 +pheon diff before.gds after.gds --cell RESONATOR +``` + +| Option | Compared content | +| --- | --- | +| `--top NAME` | The named cell and its reachable descendants in each revision. | +| `--cell NAME` | One cell, including its references, without its descendants. | +| `--layer L/T` | Objects on a numeric layer/type pair, plus references leading to that layer. | + +Selectors combine: `--top CHIP --cell RESONATOR` selects that cell within CHIP's +subtree. A name absent from both revisions is an error; presence on only one +side is an addition or deletion. Layer selection omits cells irrelevant to that +layer, but library fields and metadata-version diagnostics remain relevant. +The complete input is parsed before scope selection, so malformed records +outside the scope still fail the command. + +## Control report detail + +```sh +pheon diff before.gds after.gds --show-cell RESONATOR --show-layer 1/0 +pheon diff before.gds after.gds --hierarchy mermaid --depth 4 +``` + +`--show-cell` and `--show-layer` filter displayed changes without changing the +comparison, totals, or exit status. A layer display filter hides references +without a numeric layer. `--max-changes N` limits displayed object occurrences; +JSON includes all changed objects unless this limit is explicit. + +The diff hierarchy marks directly changed cells, affected ancestors, and +unchanged targets of modified references. It includes references from both +revisions so removed branches remain visible. Counts represent direct placements; +arrays remain symbolic. `--depth` limits text diagrams. For a complete graph in +JSON, use `--format json --hierarchy graph`; text depth and display filters do +not trim that graph. + +Next: configure [Git integration](git.md) or add [layer names](layer-names.md). diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 0000000..a64d64a --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,58 @@ +# Comparison rules + +Pheon compares GDSII objects using the `klamath-object-v2` profile. Use it to +review exact layout changes while ignoring incidental ordering and timestamps. +The [CLI guide](cli.md) explains how to choose cells and layers. + +## What counts as equal + +Cells match by name and layers by numeric layer/type pair. By default, the +comparison includes the entire library. + +Pheon ignores library and cell timestamps, cell order, element order, and +property order. Boundary and box rings are normalized for their starting vertex +and winding direction. Other vertices remain significant. Repeated objects and +properties are compared as multisets, so duplicates still count. + +Other retained details compare exactly: paths, labels, reference transforms, +array endpoints and counts, properties, and flags. An explicitly stored default +is distinct from an omitted field. An SREF is distinct from a one-element AREF. +REAL8 encodings are retained, so differently encoded values can compare as +changed even when their decoded values agree. + +Library fields, including UNITS, compare separately. Coordinates are not +rescaled to a shared grid. Cell renames are additions and deletions; Pheon does +not infer identity from similar content. It does not perform geometric Boolean +operations, XOR, or tolerance-based matching. + +## Read change explanations + +Unmatched objects can be paired to explain a replacement when the pairing is +unique by kind, layer, and bounds, then by kind and layer. References also use +their target cell. Ambiguous candidates remain independent removals and +additions. Pairing helps explain the report; it does not establish persistent +object identity or change the underlying counts. + +Reported bounds are envelopes of stored coordinates in cell-local database +units. They exclude path width, rendered text size, and recursively transformed +reference geometry. Account for UNITS before interpreting a physical dimension +or movement. + +Recognized [layer-name annotations](layer-names.md) are removed from ordinary +object comparison. Their name mapping compares separately, so a layer rename +appears once. + +## Know when a result is incomplete + +Unsupported records, future metadata versions, unresolved references, and +hierarchy cycles prevent a complete comparison. Pheon reports known differences +but returns 2 rather than claiming equality. + +Scope can exclude a record known to belong to another layer. Unsupported +records whose layer is unknown can still make a layer-scoped comparison +incomplete. Library fields and future metadata diagnostics remain visible. +Malformed input fails before scope is applied. + +For automation, check completeness as well as differences. The +[JSON report reference](diff-reports.md) describes the status fields and retained +evidence. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..ad4cde9 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,45 @@ +# Development + +This guide is for contributors building and checking Pheon. For usage, start +with the [CLI](cli.md) or [Python](python.md) guide. + +## Build and test + +The checkout selects the stable Rust toolchain; the minimum supported compiler +is 1.88. Python bindings require Python 3.11 or newer. + +```sh +cargo test --locked +cargo test --release --locked +cargo build --release --locked +python -m pip install '.[arrow]' +python -I tools/check_installed.py +``` + +Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler. +The Python package bundles its native library during installation. To test a +newly built library without reinstalling, set `PHEON_LIBRARY` to its absolute +path and run `python -m unittest discover -s tests/python` from the checkout. +Unset this override when checking installed artifacts with `check_installed.py`. + +## Interfaces and reports + +The CFFI declarations live in [pheon/_ffi.h](../pheon/_ffi.h). Keep them aligned +with the exported native functions. The C API version and +[Arrow layout version](arrow-schema.md) are independent of package releases and +the [JSON report schema](diff-reports.md). + +Review tests cover comparison semantics, CLI behavior, symbolic array counts, +and versioned report goldens. To regenerate JSON goldens after deliberately +changing their contract: + +```sh +PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens +``` + +Inspect the resulting diff. Goldens normalize the producer version to `test`; +a package version change alone should not alter them. JSON Schema checks verify +structure, while semantic tests verify links, totals, and result/exit agreement. + +See [building releases](releases.md) for source archives, wheels, and standalone +CLI artifacts. diff --git a/docs/diff-reports.md b/docs/diff-reports.md index 24dcbf3..cd58c92 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -1,4 +1,9 @@ -# Machine-readable diff reports +# JSON diff reports + +Use JSON reports to automate layout review while retaining the evidence behind +a result. The [comparison rules](comparison.md) define equality; the +[version 1 JSON Schema](diff-report-v1.schema.json) defines every field. +A [complete example](../tests/golden/report-geometry.json) shows geometry changes. ```sh pheon diff before.gds after.gds --format json > report.json @@ -6,184 +11,116 @@ pheon diff before.gds after.gds --format json --hierarchy graph pheon diff before.gds after.gds --format json --top CHIP --layer 1/0 ``` -The command writes one UTF-8 JSON document followed by a newline to stdout. -Comparison diagnostics belong to that document. Output write failures are -reported on stderr and return 2; interrupted output is not a valid report. -JSON output currently belongs to `diff`; `dump`, `hierarchy`, and the Git -external driver retain their existing text interfaces. No external process is -used to produce a report. +`diff` writes one UTF-8 JSON document and a newline to stdout, including comparison +diagnostics. Output write failures go to stderr and return 2; interrupted output +is invalid. Other CLI commands retain text output. -The [version 1 JSON Schema](diff-report-v1.schema.json) defines the structure. -[Golden reports](../tests/golden/report-geometry.json) illustrate complete -evidence; other `report-*.json` goldens cover equality, metadata, incomplete -comparison, and failure. Their producer version is replaced with `test` so -package version changes do not alter the fixtures. +## Check the result first -## Versions and determinism - -Pheon emits schema version 1 with producer name `pheon`. This schema has not -had a public release. The comparison profile remains `klamath-object-v2`, and -the `KLAMATH_LAYER_NAMES_V1` GDS marker retains its existing meaning. - -`schema_version` describes the report structure. `comparison_profile` names the -equality rules, currently `klamath-object-v2`. `producer.version` identifies the -implementation. These versions are independent. Consumers interpreting equality -must understand the comparison profile as well as the schema. - -Compatible additions may introduce optional fields. Consumers should ignore -unfamiliar additional fields and handle unfamiliar diagnostic codes and matching -rules without treating them as equality. Removed fields or changed meanings -require a new schema version. The schema intentionally permits additional fields. - -For the same inputs, paths, options, and producer version, serialization is -deterministic. Cells, layer pairs, fields, properties, and graph edges are sorted. -Duplicate objects and properties remain separate occurrences. Reports contain -no generation timestamp or required content fingerprint. - -## Result and exit status - -| `status` | `result` | Exit code | Interpretation | +| `status` | `result` | Exit code | Meaning | | --- | --- | --- | --- | | `complete` | `equal` | 0 | Equal under the recorded profile and scope. | | `complete` | `different` | 1 | Differences found. | -| `incomplete` | `different` | 2 | Definite differences found, with comparison limitations. | +| `incomplete` | `different` | 2 | Definite differences, with comparison limitations. | | `incomplete` | `unknown` | 2 | Equality cannot be established. | -| `failed` | `unknown` | 2 | No comparison result is available. | +| `failed` | `unknown` | 2 | No comparison result. | -Unsupported records, future metadata versions, unresolved references, and -hierarchy cycles make the comparison incomplete. Malformed input, read errors, -invalid scopes, and invalid command arguments produce failed reports. CI must -check completeness as well as differences. An exit code of 1 is a normal -comparison outcome, including when stdout is redirected to a report file. +A code of 1 is an expected result, including when saving a report. Unsupported +records, future metadata, unresolved references, and cycles make a comparison +incomplete. Malformed input, I/O errors, invalid scopes, and command errors fail. -Library and cell timestamps are excluded from the comparison profile. A valid, -complete timestamp-only comparison returns 0. Raw record dumps remain available -for examining those timestamps. +`summary` covers the whole scoped comparison before display filters. Counts are +`exact` for complete results and `observed` for incomplete ones. Failed reports +have a null summary and empty change arrays. Replacement suggestions and affected +ancestors do not add to change counts. -`summary` counts the entire scoped comparison, before display selection. Its -`counts` is `exact` for a complete comparison and `observed` for an incomplete -one. Failed reports have a null summary and empty change arrays. Replacement -suggestions and affected ancestors never add to change counts. +## Scope and evidence selection -## Scope, selection, and numbers +`scope` records `top`, `cell`, and numeric `layer` selectors; null means +unrestricted. The full input is parsed before scope selection. -`scope` records the actual `top`, `cell`, and numeric `layer` selectors. Null -means unrestricted. The complete file is parsed before applying scope, so an -invalid record outside the selected subtree still fails the command. +`report_selection` records display-only cell/layer filters and `max_objects`. +JSON is unbounded by default. Explicit `--max-changes N` limits total emitted +removals plus additions, with removals first within each cell. Selection counters +`filtered_cells`, `filtered_objects`, `limited_objects`, and `emitted_objects` +explain omissions. `truncated` is true when filtering or limits omit evidence. +Library changes, layer-name changes, diagnostics, summaries, and the optional +graph remain. Replacement links appear only if both objects are emitted. -`report_selection` records display-only `cell`, `layer`, and `max_objects`. -JSON has no default object limit. Explicit `--max-changes N` limits the total -number of emitted object occurrences across both sides. Removals precede -additions within each cell. `filtered_cells`, `filtered_objects`, -`limited_objects`, and `emitted_objects` explain evidence selection. `truncated` -is true if filtering or a limit omits any changed cell or object. Library and -layer-name changes, diagnostics, summaries, and the optional graph are retained. -A replacement is emitted only when both linked objects are included. +## Read values safely -All aggregate counts, limits, and byte offsets are unsigned decimal **strings**. -This avoids loss of precision in JSON consumers using binary64 numbers. -Coordinates, numeric tags, layer/type pairs, widths, and array dimensions are -bounded integers represented as JSON numbers. Null means absent or unavailable; -zero and empty arrays do not imply absence. +Aggregate counts, limits, and byte offsets are unsigned decimal **strings** to +avoid JSON number precision loss. Coordinates, numeric tags, layer/type pairs, +widths, and array dimensions are bounded JSON numbers. Null means absent or +unavailable; zero and empty arrays remain meaningful values. -`inputs.old` and `inputs.new` contain input paths, presence, and UNITS fields. -Presence is distinct from an empty library. The Rust API also supports absent -Git revisions using `Document::empty_like`; no units are invented for absent -inputs. Failed reports may have null presence and units. Argument failures can -lack input/selection context because the command could not be parsed. +`inputs.old` and `inputs.new` record paths, presence, and UNITS. An absent revision +is distinct from an empty library. The Rust API supports absent Git revisions +with `Document::empty_like`. Failed reports may lack presence, units, or argument +context. Non-UTF-8 paths use null `path` and `path_encoded_hex`: `0x`-prefixed +platform-native Rust path bytes, not a portable filename encoding. -Paths are UTF-8 when representable. Otherwise `path` is null and -`path_encoded_hex` preserves Rust's platform-native encoded path bytes, prefixed -with `0x`. This is platform-dependent evidence rather than a portable path name. +## Interpret changes -## Change evidence +| Section | Evidence | +| --- | --- | +| `library_changes` | Typed before/after header fields, including unscaled UNITS. Presence changes use a null tag and boolean values. | +| `layer_name_changes` | Before/after names for numeric pairs `{ "number": 1, "purpose": 0 }`. | +| `layer_legend` | Unchanged names available in scope. Annotation labels do not also appear as object changes. | +| `cell_changes` | Name, nullable before/after cell records, attributes, removed/added objects, and replacement suggestions. A null side means absent; an existing empty cell still has a byte offset. | -`library_changes` records changed header fields as typed before/after arrays. -Library presence changes use the special field `presence`, a null tag, and -boolean before/after values. Units are not rescaled to a common grid. +Objects retain kind, numeric tag, layer, target, bounds, source side and half-open +byte range, typed fields, and a sorted property multiset including duplicates. +Convenience fields include origin, width, and reference dimensions. SREF counts +are `[1, 1]`; kind and retained fields still distinguish a one-element AREF. -`layer_name_changes` records numeric pairs with before/after names. A layer -pair is `{ "number": 1, "purpose": 0 }`, where purpose is DATATYPE, TEXTTYPE, -BOXTYPE, or NODETYPE according to the object kind. `layer_legend` contains -unchanged names available in the scope. Embedded annotation labels are excluded -from ordinary object changes. A metadata rename appears once. +Coordinates are cell-local database units. Bounds are stored-coordinate +envelopes, excluding stroke width, rendered text, and recursive target geometry. +Consider UNITS before deriving physical dimensions. -Each `cell_changes` entry has its name, nullable before/after cell records, -attribute evidence, removed and added objects, and replacement suggestions. -A null cell side means the cell is absent in that scope. An existing empty -cell has a byte offset and an empty attribute array. +Fields contain `tag`, display `name`, `encoding`, decoded `value`, and +`payload_hex`. Numeric tags are authoritative. XY values are lists of points; +other numeric values are arrays. Unsupported encodings have null values and +preserved bytes. Invalid UTF-8 strings/properties have null decoded values; +hex preserves their bytes. -Each object retains: +Evidence follows the comparison profile: boundary/box rings are canonicalized, +string padding is removed, and fields/properties are sorted. REAL8 encodings and +explicit defaults remain significant. Source offsets locate original records. -- Its kind and numeric tag, numeric layer pair, target cell, and coordinate envelope. -- Its original source side and byte range `[byte_offset, end_offset)`. -- Typed field records and the sorted property multiset, including duplicates. -- Convenience fields for text/reference origin, path/text width, and reference - array dimensions. SREF dimensions are `[1, 1]`; the kind and retained fields - distinguish an SREF from a one-element AREF and explicit fields from omission. - -Coordinates are cell-local integer database units. `bounds` contains -`[xmin, ymin, xmax, ymax]` of stored coordinates. It excludes path stroke width, -rendered text dimensions, and recursively transformed target geometry. -Equal envelopes do not establish geometric equality. Unit changes must be -considered before deriving physical movements or dimensions. - -A retained field has a numeric `tag`, convenience `name`, `encoding`, decoded -`value`, and `payload_hex`. The numeric tag is authoritative; names are display -aids. XY values are arrays of `[x, y]` pairs. Other numeric fields are arrays -of their declared type. Unsupported encodings retain bytes with a null value. -String and property values are null if their bytes are not valid UTF-8; -`payload_hex` always preserves them without replacement characters. - -Payloads follow the comparison profile: boundary/box rings use canonical start -and winding, string padding is removed, and fields/properties are sorted. They -are comparison evidence, not a verbatim serialization of the source records. -REAL8 payload encodings are retained exactly alongside their decoded values. -Explicit defaults and omitted fields remain distinct. Source ranges allow raw -record inspection when original encoding or order is needed. - -Replacement suggestions link report-local removed/added IDs, with a `match_rule`, -typed `field_changes`, and `properties_changed`. Their explanation is optional -for consumers to display and must not be parsed. Current matching rules are -`unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. -These are explanatory matches, not persistent identity or shape equivalence. -Ambiguous candidates stay independent removals and additions. IDs are unique -within a report and have no meaning across reports. +Replacements link report-local removed/added IDs with `match_rule`, +`field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, +`unique_kind_layer`, or `unique_kind_target`. IDs are not persistent identities; +ambiguous candidates stay separate. Explanation text is for display only. ## Diagnostics and hierarchy -Diagnostics have `code`, `severity`, `message`, input `side`, original byte -offset, cell, layer, and target context. Unknown context is null. Messages are -for display; codes and context support automation. Current limitation codes are -`unsupported_record`, `unsupported_metadata_version`, `unresolved_reference`, -`hierarchy_cycle`, and `scope_mismatch`. Failure codes are `invalid_input`, -`io_error`, `invalid_scope`, and `command_error`. Fatal parser errors currently -retain their location in the human message; their structured byte offset is -null. Consumers must not depend on message wording. +Diagnostics provide `code`, `severity`, `message`, input side, byte offset, cell, +layer, and target; unknown context is null. Use codes and context for automation. +Fatal parser locations currently appear only in the human message, with null +structured offsets. Do not parse message wording. -`--hierarchy graph` includes the full scoped union graph of both revisions. -Otherwise `hierarchy` is null. Nodes are keyed by cell name, with per-side -states `included`, `outside_scope`, `unresolved`, or `absent`. Their `change` -is `direct`, `affected` through a descendant, or `unchanged`. +Limitation codes are `unsupported_record`, `unsupported_metadata_version`, +`unresolved_reference`, `hierarchy_cycle`, and `scope_mismatch`. Failure codes are +`invalid_input`, `io_error`, `invalid_scope`, and `command_error`. -Edges record parent, target, before/after direct placement counts, and whether -the underlying reference records changed. AREF columns times rows contribute -symbolically; arrays are never expanded. Counts are direct edge multiplicities, -not recursively accumulated placements. Shared cells occur once. Removed edges -remain visible, and ancestors are computed over the union of both revisions. -Graph content is independent of display filters and object limits. Text depth -limits and ASCII/Mermaid choices do not apply to JSON graphs. +`--hierarchy graph` includes the full scoped union graph; otherwise `hierarchy` +is null. Nodes are keyed by name, with per-side states `included`, `outside_scope`, +`unresolved`, or `absent`, and change states `direct`, `affected`, or `unchanged`. +Edges record parent, target, before/after direct placement counts, and changed +reference records. Array multiplicities are symbolic, not expanded or recursively +accumulated. Shared cells occur once and removed edges remain. Display filters, +object limits, and text depth do not trim the graph. -## Development +## Versions -`cargo test --test review` checks report semantics, CLI behavior, and versioned -goldens. To intentionally regenerate report goldens after reviewing a contract -change, run: +`schema_version` is 1, `comparison_profile` is `klamath-object-v2`, and +`producer.name` is `pheon`. Structure, equality rules, and producer version evolve +independently. Understand the profile before interpreting equality. Consumers +should tolerate additional optional fields and unknown codes/rules without +assuming equality. Removed fields or changed meanings require a schema change. -```sh -PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens -``` - -Schema validators can validate the goldens or actual CLI output against -`diff-report-v1.schema.json`. Schema validation does not replace semantic checks -such as replacement links, summary totals, or status/exit-code agreement. +Serialization is deterministic for identical inputs, paths, options, and producer +version. Sorted evidence retains duplicates. Reports have no generation timestamp +or required content fingerprint. See [development](development.md) for contract +tests and golden updates. diff --git a/docs/git.md b/docs/git.md new file mode 100644 index 0000000..e1d0c17 --- /dev/null +++ b/docs/git.md @@ -0,0 +1,59 @@ +# Git integration + +Configure Git to show meaningful GDSII changes during review. Install the +[Pheon CLI](cli.md#install) first, then choose one of the following drivers. +Templates are also available in [contrib/git](../contrib/git/). + +## Object comparison + +Use the external driver for object-aware comparison and affected hierarchy. +Add this to your repository's `.gitattributes`: + +```gitattributes +*.gds diff=pheon +``` + +Add this to `.git/config`, or use the same section in your user Git config: + +```gitconfig +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false +``` + +Then run `git diff` as usual. The driver reads Git's temporary old and new files +and handles additions, deletions, and renames. Keep the final `--`: Git appends +its path and revision arguments after it. Put Pheon options before that marker, +for example `--top CHIP` or `--show-layer 1/0`. + +For an existing `diff=layout` attribute, put the configuration under +`[diff "layout"]` instead. Attribute names only select a Git driver. + +The external driver returns 0 after rendering either an equal or different +comparison, and nonzero for invalid or incomplete comparisons. For automation +that needs equality status, run `pheon diff OLD NEW` directly; it returns +0, 1, or 2 as described in the [CLI guide](cli.md). + +## Normalized text comparison + +Use text conversion when you want Git to compare normalized text dumps. Set +`.gitattributes` to: + +```gitattributes +*.gds diff=pheon-text +``` + +Configure the driver: + +```gitconfig +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true +``` + +Git can cache converted text. Invalidate that cache when upgrading to a tool or +comparison profile that changes the dump format, or disable `cachetextconv`. +These diffs are for human review; they are not patches that can be applied to +GDSII files. See the [comparison rules](comparison.md) for normalization details. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f4e876d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,19 @@ +# Pheon documentation + +Start with a guide for the task you want to perform. + +| Task | Guide | +| --- | --- | +| Inspect layouts, compare revisions, or view hierarchy | [Command-line tools](cli.md) | +| Load geometry or selected cells in Python | [Python](python.md) | +| Review GDSII files with Git | [Git integration](git.md) | +| Add readable names to numeric layers | [Layer names](layer-names.md) | +| Understand what a comparison considers equal | [Comparison rules](comparison.md) | + +For integrations, use the [Arrow layout reference](arrow-schema.md), +[Rust and C API guide](native-api.md), or [JSON report reference](diff-reports.md). + +For contributors and maintainers, see [development](development.md) and +[building releases](releases.md). + +Return to the [project introduction](../README.md). diff --git a/docs/layer-names.md b/docs/layer-names.md new file mode 100644 index 0000000..8fa9288 --- /dev/null +++ b/docs/layer-names.md @@ -0,0 +1,44 @@ +# Layer names + +Attach readable names to numeric GDSII layer/type pairs so Pheon can show a +legend in comparison reports. Names are annotations; geometry keeps its numeric +layer assignments. + +## Add or update names + +```sh +pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 \ + --name 1:0:WG --name 2:0:METAL +``` + +Choose an existing cell and an otherwise unused `(layer, texttype)` pair in that +cell as the carrier. The carrier must contain no ordinary design objects. +The output path must be new; Pheon refuses to overwrite any existing file, +including the input. + +The command consolidates the legend into that cell. Existing mappings remain +unless you override them with `--name`. It creates no cells or references, and +copies other records, timestamps, unknown records, and trailing padding byte for +byte. Input and annotation validation finish before the output is created. + +## Annotation format + +A text label whose exact string is `KLAMATH_LAYER_NAMES_V1` identifies a carrier +pair in its cell. Every other text label on that pair in the same cell is a +mapping of the form: + +```text +layer:datatype:name +``` + +For example, `1:0:WG` names pair `(1, 0)` as WG. Numbers are unsigned 16-bit +decimals. The name must be nonempty UTF-8 and may contain colons; only the first +two colons separate fields. The legacy marker remains part of the format. + +Mappings are collected across the library. Identical duplicates are accepted; +conflicting names for the same pair are errors. Recognized annotation labels +are excluded from ordinary object comparison. The name mapping compares +separately, while unchanged names remain available in the report legend. + +Pheon does not currently read LayoutEditor LayerNames metadata, extract layers, +or remap geometry. See the [CLI guide](cli.md) for layer selection during review. diff --git a/docs/native-api.md b/docs/native-api.md new file mode 100644 index 0000000..d3024e5 --- /dev/null +++ b/docs/native-api.md @@ -0,0 +1,68 @@ +# Rust and C integration + +Use the Rust library or C API to read GDSII into Arrow without Python. For the +returned data layout, see the [Arrow reference](arrow-schema.md). + +## Rust + +Add `pheon` as a dependency. The library readers accept GDSII byte slices: + +```rust +use pheon::library::read_library; + +let bytes = std::fs::read("chip.gds").expect("read layout"); +let (_remainder, layout) = read_library(&bytes).expect("parse layout"); +println!("{layout:?}"); +``` + +`read_library` reads geometry; `scan_library` returns metadata, references, and +cell ranges. Pass `(start, end)` ranges from a scan to `read_selected_library` +to load only those cells. Each range must span one complete cell. Readers return +`(remainder, StructArray)` on success. The selected reader returns the original +input as its remainder, since selection is random access. + +The `record`, `records`, and `basic` modules also provide low-level GDSII record +writing. There is no high-level writer for the Arrow layout. REAL8 writing is +exact for representable in-range `f64` values: magnitudes below `2^-260` +underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected. + +## C API + +Build the shared library with `cargo build --release --locked`. The +[header](../pheon/_ffi.h) defines the functions and Arrow C Data structures. +The library is `libpheon.so`, `libpheon.dylib`, or `pheon.dll` on Linux, macOS, +or Windows respectively. + +| Function | Input | Result | +| --- | --- | --- | +| `read_path` | NUL-terminated path | Full layout. | +| `read_bytes` | Byte pointer and size | Full layout. | +| `scan_bytes` | Byte pointer and size | Metadata, references, and cell ranges. | +| `read_cells_bytes` | Bytes and cell ranges | Full layout for selected cells. | + +`read_path` accepts UTF-8 paths on Windows and filesystem bytes on Unix. It reads +uncompressed GDSII; Python's `read_path` adds gzip handling. + +For `read_cells_bytes`, `ranges` points to alternating uint64 start/end offsets +and `range_count` is the number of cells, not the number of offsets. Ranges are +half-open, from BGNSTR through the end of ENDSTR. Range values must be aligned, +native-endian uint64 values. Input buffers must remain readable for the call. +Null buffer pointers are allowed only when their corresponding length is zero. + +### Status and ownership + +Each reader returns 0 on success and -1 on failure. Call `last_error_message()` +on the same thread after a failure. It returns a borrowed NUL-terminated message; +copy it before the next parser call or thread exit. It returns null after a +successful parser call. + +Supply valid, writable, initialized `ArrowArray` and `ArrowSchema` structures +without live allocations to overwrite. A failed call leaves them unchanged. +Successful outputs own their storage independently of the input buffer. Transfer +ownership to an Arrow importer, or call each non-null release callback exactly +once. Do not release structures again after an importer takes ownership. + +`pheon_abi_version()` and `pheon_arrow_schema_version()` both currently return 1. +Check both before using the API. The C calling/ownership contract and Arrow +layout have separate versions; neither is the package version or the +[JSON report schema](diff-reports.md). diff --git a/docs/python.md b/docs/python.md new file mode 100644 index 0000000..7f123ca --- /dev/null +++ b/docs/python.md @@ -0,0 +1,97 @@ +# Python + +Use `pheon.arrow` to load GDSII geometry into PyArrow, scan a hierarchy, or read +only selected cells. The base `pheon` package also provides integer packing. +Python 3.11 or newer is required. + +## Install + +To install a published package with Arrow support: + +```sh +python -m pip install 'pheon[arrow]' +``` + +From a source checkout, use `python -m pip install '.[arrow]'`. Source builds +require Rust 1.88 or newer. Prebuilt wheels bundle the native library and need +no Rust installation. Use `pheon` without the `arrow` extra if you only need +integer packing; importing it does not import PyArrow. + +Run Python outside the source checkout to use the installed package. + +## Read a layout + +```python +from pheon import arrow + +layout = arrow.read_path("chip.gds") +print(layout.field("lib_name")[0].as_py()) +print(layout.field("cell_names")[0].as_py()) +cells = layout.field("cells")[0].values +``` + +`read_path` accepts strings and path-like objects, expands `~`, and recognizes +gzip by its magic bytes. `read_bytes` accepts an uncompressed, contiguous byte +buffer, such as `bytes`, `bytearray`, or a compatible memoryview. + +Every reader returns a one-row `pyarrow.StructArray`. Geometry is grouped by +cell and element kind; cell and layer IDs resolve through tables in the same +result. See the [Arrow reference](arrow-schema.md) for fields and encodings. +The returned arrays own their storage independently of the input buffer. + +## Scan, then read selected cells + +```python +from pathlib import Path +from pheon import arrow + +content = Path("chip.gds").read_bytes() # Uncompressed GDSII bytes. +scan = arrow.scan_bytes(content) +cells = scan.field("cells")[0].values +ranges = list(zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), +)) +selected = arrow.read_cells_bytes(content, ranges[:2]) +``` + +Scanning returns library metadata, cell names, references, and complete cell byte +ranges without decoding geometry or text. Selected reads return the full layout +schema for the requested cells, in the requested order. Use the selected result's +own name and layer tables: IDs may differ between reads. + +Ranges must have shape `(N, 2)` with nonnegative integer offsets. Each half-open +range must span exactly one complete cell, from BGNSTR through ENDSTR. An empty +list returns metadata without cells. For compressed files, decompress the bytes +before scanning or selecting ranges. + +## Pack integers + +```python +import pheon + +int2_bytes = pheon.pack_int2([1, 2, 3]) +int4_bytes = pheon.pack_int4([100, 200]) +``` + +These helpers return big-endian GDSII integer bytes. They accept NumPy arrays, +Python sequences, or scalars and preserve the caller's data, including for +noncontiguous or differently ordered arrays. Pheon currently has no high-level +Python layout writer. + +## Errors and native library selection + +Native parse failures raise `pheon.PheonError`, a subclass of `ValueError`. +Invalid buffer or range arguments raise `ValueError` or `TypeError`; file access +errors use Python's normal I/O exceptions. + +Python loads its bundled native library. Set `PHEON_LIBRARY` to an explicit +library path before importing Pheon to override it during development. The +loader checks C API and Arrow schema compatibility. It does not discover a +Cargo-installed executable or search PATH for a library. + +Klamath's `native` extra installs Pheon's packing helpers; Masque's `arrow` extra +installs Pheon with Arrow support. Install these in the same Python environment +as the companion package. A standalone CLI installation does not supply the +Python bindings. +For direct integration without Python, see the [native API guide](native-api.md). diff --git a/docs/releases.md b/docs/releases.md index 5836f98..332a447 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,51 +1,48 @@ -# Building and validating a release +# Building releases -The source archive, Python wheels and standalone CLI are separate artifacts. -Python wheels contain `pheon` and its shared library; Cargo installation contains -the CLI. Neither installation searches for the other. +This guide is for maintainers producing source archives, Python wheels, and +standalone CLI artifacts. See [development](development.md) for local checks. -Build the source archive and then the wheel from that archive: +## Build from the source archive ```sh python -m pip install build python -m build ``` -The default `build` invocation builds its wheel from the source archive. Both -carry the Cargo version, license and locked dependency resolution. A source build -requires Rust 1.88 or newer. The checkout selects current stable through -`rust-toolchain.toml`; `RUSTUP_TOOLCHAIN=1.88.0` selects the minimum compiler -explicitly for source-package builds. An installed wheel has no Rust runtime -requirement. +The default invocation builds the wheel from the source archive. Both use the +Cargo version, license, and locked dependencies. Source builds require Rust 1.88 +or newer; the checkout selects stable. Use `RUSTUP_TOOLCHAIN=1.88.0` to check the +minimum compiler explicitly. -The package workflow builds the archive once, then uses cibuildwheel to build -from it and repair wheels for Linux x86-64/ARM64, macOS x86-64/ARM64 and Windows -x86-64. Repair uses the platform tools supplied by cibuildwheel. Wheels must keep -the `py3-none` interpreter/ABI tags and a platform tag, including an appropriate -manylinux tag for Linux releases. Local unrepaired `linux_*` wheels are development -artifacts, not portable Linux releases. +Python wheels contain the package and shared library. Cargo installation +provides the CLI. Neither installation discovers or depends on the other. -Each repaired wheel is tested on Python 3.11 through 3.14 without rebuilding it. -Tests run in isolated mode against the installed package and bundled library, -including an import that verifies integer packing does not import PyArrow. To -repeat that check locally after installing a wheel and PyArrow: +## Validate artifacts + +The [package workflow](../.github/workflows/packages.yml) builds one source archive +and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64, +and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag. +Linux releases need an appropriate manylinux tag; unrepaired local `linux_*` +wheels are development artifacts. + +The workflow tests each repaired wheel on Python 3.11–3.14 without rebuilding it. +Tests run in isolated mode against the installed package and bundled library. +To repeat the check after installing a wheel and PyArrow: ```sh python -I /path/to/pheon/tools/check_installed.py ``` -Unset `PHEON_LIBRARY` for artifact tests. It exists to select an explicit external -library during development, and version checks apply equally to that library. -The minimum Rust compiler and current stable compiler run native tests in both -debug and release modes. CLI artifacts contain the executable, license, README -and Git integration templates. They are tied to their build OS and architecture. +Unset `PHEON_LIBRARY` for artifact checks. Native checks cover minimum and stable +Rust in debug and release modes. Standalone CLI artifacts contain the executable, +license, README, documentation, and Git templates; each targets one OS and +architecture. -Workflows upload build artifacts only. They do not publish to PyPI, crates.io, -or a hosted release. The repository URL and checkout directory can be renamed -to `pheon` separately; no remote repository settings are changed by this work. +## Publish separately -The project rename provides no Python import shim or old CLI alias. Companion -releases must migrate their imports and dependencies together. The Python C API -version and Arrow layout version are currently 1. CLI reports use schema version -1 and retain the `klamath-object-v2` comparison profile and existing GDS layer-name -metadata marker. +Workflows upload artifacts; they do not publish to PyPI, crates.io, or a hosted +release. Review the artifacts and test results before publishing them. +Coordinate companion package dependencies when changing Python interfaces. +C API, Arrow layout, JSON schema, and comparison-profile versions have separate +contracts; see the [native API](native-api.md) and [report reference](diff-reports.md). From 54aa7576781572fb1184738b3d116b7181215cf6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 20:44:48 -0800 Subject: [PATCH 17/41] Validate GDSII record sizes through one fallible path Route every record read through size validation and report malformed payloads as parse errors instead of panicking. Apply metadata size constraints through the Record trait, hide Arrow reader internals, and remove unused wrappers and scratch code. Add coverage for direct, skipped, and payload reads, truncated buffers, and FFI error offsets. --- src/basic.rs | 1 - src/elements.rs | 38 +++++++-------- src/lib.rs | 2 +- src/library.rs | 11 ++--- src/misc.py | 39 --------------- src/record.rs | 77 ++++++++++++++--------------- src/records.rs | 41 +++------------- tests/ffi_errors.rs | 6 +-- tests/record_validation.rs | 99 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 148 deletions(-) delete mode 100644 src/misc.py create mode 100644 tests/record_validation.rs diff --git a/src/basic.rs b/src/basic.rs index f089ff3..1189b8e 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -206,7 +206,6 @@ mod tests { fn test_parse_bitarray() { use crate::basic::parse_bitarray; - //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); diff --git a/src/elements.rs b/src/elements.rs index f043ccc..02a8112 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,7 +4,6 @@ /// use crate::records::{ - //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, @@ -13,11 +12,10 @@ use crate::records::{ use crate::records; use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; //OResult +use crate::basic::{IResult, fail, take_bytes}; use std::string::String; use std::collections::HashMap; -//use std::io::Write; use arrow_array::builder::{ StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, @@ -25,8 +23,8 @@ use arrow_array::builder::{ }; -pub type DListBuilder = ListBuilder>; -pub type FListBuilder = FixedSizeListBuilder>; +pub(crate) type DListBuilder = ListBuilder>; +pub(crate) type FListBuilder = FixedSizeListBuilder>; struct PropertyRecord { key: i16, @@ -72,15 +70,15 @@ struct ARefRecord { } #[derive(Clone)] -pub struct ScanRefRecord { - pub target_id: u32, - pub invert_y: bool, - pub scale: f64, - pub angle_rad: f64, - pub xy: u64, - pub xy0: u64, - pub xy1: u64, - pub counts: u32, +pub(crate) struct ScanRefRecord { + pub(crate) target_id: u32, + pub(crate) invert_y: bool, + pub(crate) scale: f64, + pub(crate) angle_rad: f64, + pub(crate) xy: u64, + pub(crate) xy0: u64, + pub(crate) xy1: u64, + pub(crate) counts: u32, } enum RefRecord { @@ -117,7 +115,7 @@ fn insert_layer( } -pub fn read_elements<'a>( +pub(crate) fn read_elements<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, names: &mut HashMap, @@ -293,7 +291,7 @@ pub fn read_elements<'a>( } -pub fn scan_elements<'a>( +pub(crate) fn scan_elements<'a>( input: &'a [u8], names: &mut HashMap, ) -> IResult<'a, Vec> { @@ -583,7 +581,7 @@ fn append_boundary_prop( } -pub fn read_path<'a>( +fn read_path<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -654,7 +652,7 @@ pub fn read_path<'a>( Ok((input, ())) } -pub fn read_boxnode<'a>( +fn read_boxnode<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16, @@ -698,7 +696,7 @@ pub fn read_boxnode<'a>( Ok((input, ())) } -pub fn read_text<'a>( +fn read_text<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -967,7 +965,7 @@ fn read_ref<'a>( /// Preserves record order and duplicate property attributes. /// Stops reading after consuming ENDEL record. /// -pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { +fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; props_builder.append(true); Ok((input, ())) diff --git a/src/lib.rs b/src/lib.rs index 0abcca5..c31b907 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ pub mod basic; pub mod record; pub mod records; -pub mod elements; +mod elements; pub mod library; pub mod review; diff --git a/src/library.rs b/src/library.rs index 1ea4c6b..e455f8b 100644 --- a/src/library.rs +++ b/src/library.rs @@ -3,14 +3,11 @@ /// use std::io::Write; -//use std::collections::HashMap; -pub use crate::record; -pub use crate::record::{RecordHeader, Record}; -pub use crate::records; -pub use crate::elements; -pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -pub use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::record::{RecordHeader, Record}; +use crate::records; +use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +use crate::basic::{IResult, OResult, take_bytes, fail}; use std::string::String; use std::collections::HashMap; diff --git a/src/misc.py b/src/misc.py deleted file mode 100644 index 1e507c7..0000000 --- a/src/misc.py +++ /dev/null @@ -1,39 +0,0 @@ -''' -https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py -''' -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import ctypes -import pyarrow as pa -from pyarrow.cffi import ffi - - c_schema = ffi.new('struct ArrowSchema*') - ptr_schema = int(ffi.cast('uintptr_t', c_schema)) - c_array = ffi.new('struct ArrowArray*') - ptr_array = int(ffi.cast('uintptr_t', c_array)) - - # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) - - # import gc - # gc.collect() # Make sure no Arrow data dangles in a ref cycle - # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) - - arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) - - diff --git a/src/record.rs b/src/record.rs index cb64ea2..25d358a 100644 --- a/src/record.rs +++ b/src/record.rs @@ -4,16 +4,12 @@ use std::io::{self, Write}; -use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; use crate::records; -//#[no_mangle] -//pub extern "C" fn write_record_header( - - #[repr(C)] #[derive(Debug, Clone)] pub struct RecordHeader { @@ -53,7 +49,6 @@ pub trait RecordData { fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; fn pack_into(buf: &mut [u8], data: &Self::InData); - //fn size(data: &Self::BareData<'_>) -> u16; fn pack(data: &Self::InData) -> Self::ByteData; } @@ -73,22 +68,17 @@ pub trait Record { } } - fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { - RecordHeader::read(input) - } - fn write_header(ww: &mut W, data_size: u16) -> OResult { RecordHeader{tag: Self::tag(), data_size}.write(ww) } fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + if let Err(message) = Self::check_size(size) { + return fail(input, message); + } RData::read(input, size) } - fn pack_data(buf: &mut [u8], data: &RData::InData) { - RData::pack_into(buf, data) - } - /// /// Skip to the end of the next occurence of this record. /// @@ -116,8 +106,7 @@ pub trait Record { (input, _) = take_bytes(input, header.data_size)?; (input, header) = RecordHeader::read(input)?; } - let (input, data) = Self::read_data(input, header.data_size)?; - Ok((input, data)) + Self::read_data(input, header.data_size) } fn expect_header(input: &[u8]) -> IResult<'_, u16> { @@ -131,9 +120,7 @@ pub trait Record { fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { let (input, size) = Self::expect_header(input)?; - Self::check_size(size).unwrap(); - let (input, data) = Self::read_data(input, size)?; - Ok((input, data)) + Self::read_data(input, size) } fn write(ww: &mut W, data: &RData::InData) -> OResult { @@ -155,7 +142,9 @@ impl RecordData for BitArray { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_bitarray(input) } @@ -178,7 +167,9 @@ impl RecordData for Int2 { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_int2(input) } @@ -200,7 +191,9 @@ impl RecordData for Int4 { type ByteData = [u8; 4]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 4); + if size != 4 { + return fail(input, format!("Expected record size {}, got {size}", 4)); + } parse_int4(input) } @@ -223,8 +216,9 @@ impl RecordData for Int2Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(2) { + return fail(input, format!("Record size must be divisible by 2, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int2ArrayReader{bytes})) } @@ -267,8 +261,9 @@ impl RecordData for Int4Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(4) { + return fail(input, format!("Record size must be divisible by 4, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int4ArrayReader{bytes})) } @@ -311,7 +306,9 @@ impl RecordData for Real8 { type ByteData = [u8; 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 8); + if size != 8 { + return fail(input, format!("Expected record size {}, got {size}", 8)); + } parse_real8(input) } @@ -333,7 +330,9 @@ impl RecordData for Real8Pair { type ByteData = [u8; 2 * 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 8); + if size != 2 * 8 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 8)); + } let (input, data0) = parse_real8(input)?; let (input, data1) = parse_real8(input)?; Ok((input, (data0, data1))) @@ -350,10 +349,6 @@ impl RecordData for Real8Pair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct ASCII; @@ -385,7 +380,9 @@ impl RecordData for DateTimePair { type ByteData = [u8; 2 * 6 * 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 6 * 2); + if size != 2 * 6 * 2 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2)); + } let (input, data0) = parse_datetime(input)?; let (input, data1) = parse_datetime(input)?; Ok((input, [data0, data1])) @@ -402,10 +399,6 @@ impl RecordData for DateTimePair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct Empty; @@ -415,7 +408,9 @@ impl RecordData for Empty { type ByteData = [u8; 0]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 0); + if size != 0 { + return fail(input, format!("Expected record size {}, got {size}", 0)); + } Ok((input, ())) } @@ -425,6 +420,4 @@ impl RecordData for Empty { fn pack(_data: &Self::InData) -> Self::ByteData { [] } - //fn write(ww: &mut W, data: &Self::BareData) { - //} } diff --git a/src/records.rs b/src/records.rs index 4e86404..5481e2b 100644 --- a/src/records.rs +++ b/src/records.rs @@ -4,8 +4,6 @@ use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; -//use std::io::Write; - // record tags pub const RTAG_HEADER: u16 = 0x0002; pub const RTAG_BGNLIB: u16 = 0x0102; @@ -77,37 +75,13 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300; pub const RTAG_SPACERERROR: u16 = 0x4400; pub const RTAG_CONTACT: u16 = 0x4500; -/* -// data types -pub const DATA_TYPE_NONE: u16 = 0x00; -pub const DATA_TYPE_BIT: u16 = 0x01; -pub const DATA_TYPE_INT16: u16 = 0x02; -pub const DATA_TYPE_INT32: u16 = 0x03; -pub const DATA_TYPE_REAL32: u16 = 0x04; -pub const DATA_TYPE_REAL64: u16 = 0x05; -pub const DATA_TYPE_STR: u16 = 0x06; -pub const MAX_DATA_SIZE: usize = 8; - -/// Returns the size of the given data type in bytes. -pub fn data_size(t: u16) -> Option { - match t { - x if x == DATA_TYPE_NONE => 0, - x if x == DATA_TYPE_BIT => 2, - x if x == DATA_TYPE_INT16 => 2, - x if x == DATA_TYPE_INT32 => 4, - x if x == DATA_TYPE_REAL32 => 4, - x if x == DATA_TYPE_REAL64 => 8, - _ => 0 - } -*/ pub struct HEADER; impl Record for HEADER { fn tag() -> u16 { RTAG_HEADER } fn expected_size() -> Option { Some(2) } } -//impl Record for HEADER; pub struct BGNLIB; impl Record for BGNLIB { @@ -288,9 +262,8 @@ pub struct REFLIBS; impl Record for REFLIBS { fn tag() -> u16 { RTAG_REFLIBS } fn expected_size() -> Option { None } -} -impl REFLIBS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -303,9 +276,8 @@ pub struct FONTS; impl Record for FONTS { fn tag() -> u16 { RTAG_FONTS } fn expected_size() -> Option { None } -} -impl FONTS { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { @@ -330,9 +302,8 @@ pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } fn expected_size() -> Option { None } -} -impl ATTRTABLE { - pub fn check_size(actual_size: usize) -> Result<(), String> { + + fn check_size(actual_size: u16) -> Result<(), String> { if actual_size.is_multiple_of(44) { Ok(()) } else { diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs index 0e74a5e..3b81821 100644 --- a/tests/ffi_errors.rs +++ b/tests/ffi_errors.rs @@ -31,20 +31,20 @@ fn error_message() -> String { } #[test] -fn path_errors_preserve_outputs_and_convert_parser_panics() { +fn path_errors_preserve_outputs_and_clear_after_success() { let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); assert!(error_message().contains("path pointer was null")); assert!(array.is_released()); - // A mismatched timestamp size currently makes Record::read panic. + // A mismatched timestamp size returns a parse error with a byte offset. let mut bytes = Vec::new(); common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); let file = TestFile::new("invalid-timestamps", &bytes); assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); - assert!(error_message().contains("Expected record size")); + assert!(error_message().contains("read failed at byte 10: Expected record size")); assert!(array.is_released()); let mut bytes = common::library_header(); diff --git a/tests/record_validation.rs b/tests/record_validation.rs new file mode 100644 index 0000000..d1e4fa7 --- /dev/null +++ b/tests/record_validation.rs @@ -0,0 +1,99 @@ +mod common; + +use pheon::{ + basic::ErrType, + record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, + }, + records, +}; + +#[test] +fn fixed_record_sizes_are_checked_by_every_read_entry_point() { + let payload = [0; 4]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &payload); + let mut prefixed = Vec::new(); + common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip"); + prefixed.extend_from_slice(&bytes); + + for result in [ + records::HEADER::read(&bytes), + records::HEADER::skip_and_read(&prefixed), + records::HEADER::read_data(&payload, 4), + ] { + let (remaining, reason) = result.unwrap_err(); + assert_eq!(remaining, payload); + assert!(matches!(reason, ErrType::Failed(message) + if message == "Expected record size 2, got 4")); + } + + // The underlying integer-array payload is aligned, but COLROW needs two values. + let payload = [0; 2]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_COLROW, &payload); + assert!(records::COLROW::read(&bytes).is_err()); + assert!(records::COLROW::skip_and_read(&bytes).is_err()); + assert!(records::COLROW::read_data(&payload, 2).is_err()); +} + +#[test] +fn metadata_size_constraints_apply_to_every_read_entry_point() { + macro_rules! check { + ($record:ident, $tag:ident) => {{ + let payload = [b'x'; 44]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + let mut prefixed = common::library_header(); + prefixed.extend_from_slice(&bytes); + assert_eq!(records::$record::read(&bytes).unwrap().1, payload); + assert_eq!( + records::$record::skip_and_read(&prefixed).unwrap().1, + payload + ); + assert_eq!( + records::$record::read_data(&payload, 44).unwrap().1, + payload + ); + + let payload = [b'x'; 46]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + assert!(records::$record::read(&bytes).is_err()); + assert!(records::$record::skip_and_read(&bytes).is_err()); + assert!(records::$record::read_data(&payload, 46).is_err()); + }}; + } + + check!(REFLIBS, RTAG_REFLIBS); + check!(FONTS, RTAG_FONTS); + check!(ATTRTABLE, RTAG_ATTRTABLE); +} + +#[test] +fn malformed_payload_sizes_return_errors_without_panicking() { + let payload = [0; 32]; + assert!(BitArray::read(&payload, 4).is_err()); + assert!(Int2::read(&payload, 4).is_err()); + assert!(Int4::read(&payload, 2).is_err()); + assert!(Real8::read(&payload, 4).is_err()); + assert!(Real8Pair::read(&payload, 8).is_err()); + assert!(DateTimePair::read(&payload, 12).is_err()); + assert!(Empty::read(&payload, 2).is_err()); + assert!(Int2Array::read(&payload, 3).is_err()); + assert!(Int4Array::read(&payload, 6).is_err()); + + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_XY, &payload[..6]); + assert!(records::XY::read(&bytes).is_err()); + assert!(records::XY::skip_and_read(&bytes).is_err()); + assert!(records::XY::read_data(&payload, 6).is_err()); + + // Valid sizes still reject a truncated buffer; an empty string remains valid. + assert!(Int4::read(&payload[..2], 4).is_err()); + assert!(Real8Pair::read(&payload[..8], 16).is_err()); + assert!(DateTimePair::read(&payload[..12], 24).is_err()); + assert!(Int4Array::read(&payload[..4], 8).is_err()); + assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::::new())); +} From a723ba22651806c43aaa8675637553d652d7642f Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 20:44:56 -0800 Subject: [PATCH 18/41] Simplify Python bindings and package builds Read each input path through one opened stream, including gzip detection, and use direct numeric converter lookup while preserving byte-order handling and caller data. Read the package version from installed metadata and report range errors consistently as ValueError. Require the supplied Cargo.lock instead of generating it during builds, use the standard sdist command, and update package checks and development guides. Validation: 15 Python tests pass against source and the rebuilt installed wheel; the source archive and platform wheel build successfully. --- docs/development.md | 10 ++++++---- docs/releases.md | 2 +- pheon/__init__.py | 4 +++- pheon/arrow.py | 9 +++------ pheon/basic.py | 8 +++----- setup.py | 15 +++------------ tests/python/test_arrow.py | 11 +++++++++++ tests/python/test_packing.py | 15 ++++++++------- tools/check_installed.py | 4 ++-- 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/docs/development.md b/docs/development.md index ad4cde9..da07331 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,7 +6,8 @@ with the [CLI](cli.md) or [Python](python.md) guide. ## Build and test The checkout selects the stable Rust toolchain; the minimum supported compiler -is 1.88. Python bindings require Python 3.11 or newer. +is 1.88. Python bindings require Python 3.11 or newer. Source builds require +the tracked `Cargo.lock`; update dependencies explicitly with Cargo. ```sh cargo test --locked @@ -17,14 +18,15 @@ python -I tools/check_installed.py ``` Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler. -The Python package bundles its native library during installation. To test a -newly built library without reinstalling, set `PHEON_LIBRARY` to its absolute +The Python package bundles its native library during installation and reads its +version from installed package metadata. After installation, to test a newly +built library without reinstalling, set `PHEON_LIBRARY` to its absolute path and run `python -m unittest discover -s tests/python` from the checkout. Unset this override when checking installed artifacts with `check_installed.py`. ## Interfaces and reports -The CFFI declarations live in [pheon/_ffi.h](../pheon/_ffi.h). Keep them aligned +The CFFI declarations live in [pheon/_ffi.h](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h). Keep them aligned with the exported native functions. The C API version and [Arrow layout version](arrow-schema.md) are independent of package releases and the [JSON report schema](diff-reports.md). diff --git a/docs/releases.md b/docs/releases.md index 332a447..76e7c7b 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -20,7 +20,7 @@ provides the CLI. Neither installation discovers or depends on the other. ## Validate artifacts -The [package workflow](../.github/workflows/packages.yml) builds one source archive +The [package workflow](https://mpxd.net/code/jan/pheon/src/branch/master/.github/workflows/packages.yml) builds one source archive and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64, and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag. Linux releases need an appropriate manylinux tag; unrepaired local `linux_*` diff --git a/pheon/__init__.py b/pheon/__init__.py index 09bb1d6..b71eb35 100644 --- a/pheon/__init__.py +++ b/pheon/__init__.py @@ -1,8 +1,10 @@ """Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers.""" +from importlib.metadata import version as _version + from ._native import PheonError as PheonError from .basic import pack_int2 as pack_int2 from .basic import pack_int4 as pack_int4 -__version__ = "0.3.0" +__version__ = _version("pheon") diff --git a/pheon/arrow.py b/pheon/arrow.py index 5d90631..5aa6ee1 100644 --- a/pheon/arrow.py +++ b/pheon/arrow.py @@ -43,16 +43,13 @@ def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray: def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: """Read a GDSII file, recognizing gzip by its magic bytes.""" - filename = os.path.abspath(os.path.expanduser(os.fspath(path))) - with open(filename, "rb") as stream: + with open(os.path.expanduser(path), "rb") as stream: compressed = stream.read(2) == b"\x1f\x8b" + stream.seek(0) if compressed: - stream.seek(0) with gzip.GzipFile(fileobj=stream) as source: return read_bytes(source.read()) - # The native Windows path API accepts UTF-8; Unix accepts filesystem bytes. - encoded = filename.encode("utf-8") if os.name == "nt" else os.fsencode(filename) - return _export(lambda array, schema: lib.read_path(encoded, array, schema)) + return read_bytes(stream.read()) def _read_buffer( diff --git a/pheon/basic.py b/pheon/basic.py index 9ca4745..51100aa 100644 --- a/pheon/basic.py +++ b/pheon/basic.py @@ -36,10 +36,8 @@ def _pack_int( arr = numpy.atleast_1d(data) target_dtype = numpy.dtype(f">i{width}") - for dtype, convert in converters.items(): - if arr.dtype != dtype: - continue - + convert = converters.get(arr.dtype.type) if arr.dtype.isnative else None + if convert is not None: arr = numpy.require( arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") ) @@ -60,7 +58,7 @@ def _pack_int( limit = 1 << (width * 8 - 1) if (arr > limit - 1).any() or (arr < -limit).any(): - raise Exception(f"int{width} data out of range: {arr}") + raise ValueError(f"int{width} data out of range: {arr}") return arr.astype(target_dtype).tobytes() diff --git a/setup.py b/setup.py index ca4fd70..17d080a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,6 @@ import tomllib from setuptools import Distribution, setup from setuptools.command.build_py import build_py -from setuptools.command.sdist import sdist from wheel.bdist_wheel import bdist_wheel @@ -16,14 +15,12 @@ ROOT = Path(__file__).resolve().parent VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] -def ensure_lockfile(): - if not (ROOT / "Cargo.lock").is_file(): - subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True) +if not (ROOT / "Cargo.lock").is_file(): + raise FileNotFoundError("Source builds require the supplied Cargo.lock") class CargoBuild(build_py): def run(self): - ensure_lockfile() super().run() result = subprocess.run( ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], @@ -51,12 +48,6 @@ class CargoBuild(build_py): shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) -class CargoSdist(sdist): - def run(self): - ensure_lockfile() - super().run() - - class NativeDistribution(Distribution): def has_ext_modules(self): return True @@ -72,5 +63,5 @@ class NativeWheel(bdist_wheel): setup( version=VERSION, distclass=NativeDistribution, - cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, + cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel}, ) diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py index 30367b4..ed962e2 100644 --- a/tests/python/test_arrow.py +++ b/tests/python/test_arrow.py @@ -2,6 +2,7 @@ import gc import gzip +import io import mmap from pathlib import Path import tempfile @@ -48,6 +49,16 @@ class ArrowTests(unittest.TestCase): path.write_bytes(gzip.compress(layout())) self.assertEqual(arrow.read_path(path).to_pylist(), expected) + def test_path_reader_keeps_using_the_opened_stream(self): + for payload in (layout(), gzip.compress(layout())): + with self.subTest(compressed=payload.startswith(b"\x1f\x8b")): + source = io.BytesIO(payload) + with patch("builtins.open", return_value=source) as opened: + result = arrow.read_path("opened-stream-only.gds") + opened.assert_called_once_with("opened-stream-only.gds", "rb") + self.assertTrue(source.closed) + self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"]) + def test_empty_and_noncontiguous_cell_selections(self): data = layout() empty = arrow.read_cells_bytes(data, []) diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py index 44eb84d..75ce0cd 100644 --- a/tests/python/test_packing.py +++ b/tests/python/test_packing.py @@ -45,13 +45,14 @@ class PackingTests(unittest.TestCase): def test_failed_conversions_preserve_caller_data(self): for pack, width in [(pack_int2, 2), (pack_int4, 4)]: limit = 1 << (8 * width - 1) - for invalid in [limit, -limit - 1]: - data = np.array([[1, 2], [3, invalid]], dtype=np.int64) - before = data.copy() - with self.subTest(width=width, invalid=invalid): - with self.assertRaises(ValueError): - pack(data) - np.testing.assert_array_equal(data, before) + for dtype in [np.dtype("i8"), np.dtype("i8").newbyteorder("S")]: + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, dtype=dtype, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) for dtype in [np.float32, np.float64]: for invalid in [np.nan, np.inf, -np.inf]: data = np.array([[1, 2], [3, invalid]], dtype=dtype) diff --git a/tools/check_installed.py b/tools/check_installed.py index 707ecbf..970dc90 100644 --- a/tools/check_installed.py +++ b/tools/check_installed.py @@ -10,12 +10,12 @@ ROOT = Path(__file__).resolve().parents[1] # A separate interpreter verifies that base packing does not import PyArrow. subprocess.run( [sys.executable, "-I", "-c", """ -import importlib.util import sys +from importlib.metadata import version import pheon from pheon._native import library +assert pheon.__version__ == version('pheon') assert 'pyarrow' not in sys.modules -assert importlib.util.find_spec('klamath_rs_ext') is None assert library.parent == __import__('pathlib').Path(pheon.__file__).parent assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') """], From df62677f5ca51378870e11db29ed3ab0d6ac9eb6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 20:45:06 -0800 Subject: [PATCH 19/41] Make Pheon structural review results self-contained Capture revision metadata, cell presence and attributes, and structured diagnostics in owned comparison results. Derive outcomes and render text, JSON, and hierarchy reports without retaining source documents. Keep derived descriptions and graph analysis in their renderers. Use pheon-structural-v1 and PHEON_LAYER_NAMES_V1 consistently, expose helpers through their defining modules, and remove obsolete re-exports. Update user guides and report fixtures without changing schema version 1. Cover dropped inputs, diagnostic ordering, absent and empty cells, offset independence, and CLI failures. All 92 Rust tests pass in debug and release; report and hierarchy goldens retain their behavior. --- docs/comparison.md | 2 +- docs/diff-report-v1.schema.json | 2 +- docs/diff-reports.md | 4 +- docs/layer-names.md | 4 +- docs/native-api.md | 12 +- src/bin/pheon.rs | 148 +++---- src/review.rs | 588 ++++++++++++++++++---------- src/review/changes.rs | 6 +- src/review/names.rs | 61 +++ src/review/report.rs | 187 ++++----- src/review/selection.rs | 8 +- tests/golden/report-equal.json | 2 +- tests/golden/report-failed.json | 2 +- tests/golden/report-geometry.json | 2 +- tests/golden/report-incomplete.json | 2 +- tests/golden/report-metadata.json | 2 +- tests/golden/review-canonical.txt | 2 +- tests/golden/review-diff.txt | 2 +- tests/review.rs | 150 ++++--- tests/review/reports.rs | 186 +++++++-- tests/selective_reads.rs | 2 +- 21 files changed, 870 insertions(+), 504 deletions(-) diff --git a/docs/comparison.md b/docs/comparison.md index a64d64a..a596f54 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -1,6 +1,6 @@ # Comparison rules -Pheon compares GDSII objects using the `klamath-object-v2` profile. Use it to +Pheon compares GDSII objects using the `pheon-structural-v1` profile. Use it to review exact layout changes while ignoring incidental ordering and timestamps. The [CLI guide](cli.md) explains how to choose cells and layers. diff --git a/docs/diff-report-v1.schema.json b/docs/diff-report-v1.schema.json index fc85eaa..4427d90 100644 --- a/docs/diff-report-v1.schema.json +++ b/docs/diff-report-v1.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Pheon diff report version 1", - "description": "Object-profile comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", + "description": "Structural comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", "type": "object", "required": ["schema_version", "kind", "producer", "comparison_profile", "inputs", "scope", "coordinates", "status", "result", "exit_code", "summary", "report_selection", "layer_legend", "library_changes", "layer_name_changes", "cell_changes", "hierarchy", "diagnostics"], "properties": { diff --git a/docs/diff-reports.md b/docs/diff-reports.md index cd58c92..3408ea5 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -3,7 +3,7 @@ Use JSON reports to automate layout review while retaining the evidence behind a result. The [comparison rules](comparison.md) define equality; the [version 1 JSON Schema](diff-report-v1.schema.json) defines every field. -A [complete example](../tests/golden/report-geometry.json) shows geometry changes. +A [complete example](https://mpxd.net/code/jan/pheon/src/branch/master/tests/golden/report-geometry.json) shows geometry changes. ```sh pheon diff before.gds after.gds --format json > report.json @@ -114,7 +114,7 @@ object limits, and text depth do not trim the graph. ## Versions -`schema_version` is 1, `comparison_profile` is `klamath-object-v2`, and +`schema_version` is 1, `comparison_profile` is `pheon-structural-v1`, and `producer.name` is `pheon`. Structure, equality rules, and producer version evolve independently. Understand the profile before interpreting equality. Consumers should tolerate additional optional fields and unknown codes/rules without diff --git a/docs/layer-names.md b/docs/layer-names.md index 8fa9288..d006037 100644 --- a/docs/layer-names.md +++ b/docs/layer-names.md @@ -23,7 +23,7 @@ byte. Input and annotation validation finish before the output is created. ## Annotation format -A text label whose exact string is `KLAMATH_LAYER_NAMES_V1` identifies a carrier +A text label whose exact string is `PHEON_LAYER_NAMES_V1` identifies a carrier pair in its cell. Every other text label on that pair in the same cell is a mapping of the form: @@ -33,7 +33,7 @@ layer:datatype:name For example, `1:0:WG` names pair `(1, 0)` as WG. Numbers are unsigned 16-bit decimals. The name must be nonempty UTF-8 and may contain colons; only the first -two colons separate fields. The legacy marker remains part of the format. +two colons separate fields. Mappings are collected across the library. Identical duplicates are accepted; conflicting names for the same pair are errors. Recognized annotation labels diff --git a/docs/native-api.md b/docs/native-api.md index d3024e5..ad64a05 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -21,6 +21,16 @@ to load only those cells. Each range must span one complete cell. Readers return `(remainder, StructArray)` on success. The selected reader returns the original input as its remainder, since selection is random access. +Layout-review APIs live in `pheon::review`, with layer-name helpers in +`pheon::review::names` and scope/display selectors in `pheon::review::selection`. + +`Comparison::between(&old, &new)` captures an owned result with revision metadata, +changes, and structured diagnostics. Use `outcome()` for completeness, equality, +and exit status. Its `text`, `json_report`, and `hierarchy` renderers work after +the source documents and byte buffers are dropped. Standalone hierarchy inspection +uses `Document::hierarchy`. Reference counts are captured with the result; +union-graph and ancestor analysis run only when a hierarchy is requested. + The `record`, `records`, and `basic` modules also provide low-level GDSII record writing. There is no high-level writer for the Arrow layout. REAL8 writing is exact for representable in-range `f64` values: magnitudes below `2^-260` @@ -29,7 +39,7 @@ underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected ## C API Build the shared library with `cargo build --release --locked`. The -[header](../pheon/_ffi.h) defines the functions and Arrow C Data structures. +[header](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h) defines the functions and Arrow C Data structures. The library is `libpheon.so`, `libpheon.dylib`, or `pheon.dll` on Linux, macOS, or Windows respectively. diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 646ec6e..1d514b7 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,11 +1,12 @@ use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ - Comparison, DisplayFilter, Document, Scope, dump_records, hierarchy, parse_layer, - parse_layer_name, write_layer_names, + Comparison, Diagnostic, Document, Outcome, Side, dump_records, + names::{parse_layer, parse_layer_name, write_layer_names}, + selection::{DisplayFilter, Scope}, }; use std::{ collections::BTreeMap, - ffi::OsString, + ffi::{OsStr, OsString}, fs, io::{self, Write}, path::PathBuf, @@ -18,7 +19,7 @@ const HELP: &str = "Usage: pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... -Object profile: ignore timestamps and element order, normalize polygon start/winding, +Structural comparison: ignore timestamps and element order, normalize polygon start/winding, preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. "; @@ -93,40 +94,68 @@ fn json_stdout(report: &serde_json::Value) -> Result<(), String> { output.write_all(b"\n").map_err(|e| e.to_string()) } +fn prepare_comparison( + old_path: Option<&OsStr>, + new_path: Option<&OsStr>, + scope: &Scope, +) -> Result { + let read = |path: Option<&OsStr>, side| { + path.map(|path| { + fs::read(path).map_err(|error| { + Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side)) + }) + }) + .transpose() + }; + let old_bytes = read(old_path, Side::Old)?; + let new_bytes = read(new_path, Side::New)?; + let old = old_bytes + .as_deref() + .map(Document::parse) + .transpose() + .map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::Old)))?; + let new = new_bytes + .as_deref() + .map(Document::parse) + .transpose() + .map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::New)))?; + let (old, new) = match (old, new) { + (Some(old), Some(new)) => (old, new), + (None, Some(new)) => (new.empty_like(), new), + (Some(old), None) => { + let new = old.empty_like(); + (old, new) + } + (None, None) => { + return Err(Diagnostic::new( + "invalid_input", + "both Git revisions are absent", + None, + )); + } + }; + scope + .validate_pair(&old, &new) + .map_err(|message| Diagnostic::new("invalid_scope", message, None))?; + let old = old + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::Old)))?; + let new = new + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::New)))?; + Ok(Comparison::between(&old, &new)) +} + fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { let inputs = ReportInputs { old: Some(PathBuf::from(&paths[0])), new: Some(PathBuf::from(&paths[1])), }; - let comparison = || { - let read = |index, side| { - fs::read(&paths[index]) - .map_err(|e| ("io_error", format!("{:?}: {e}", paths[index]), Some(side))) - }; - let old_bytes = read(0, "old")?; - let new_bytes = read(1, "new")?; - let old = Document::parse(&old_bytes).map_err(|e| ("invalid_input", e, Some("old")))?; - let new = Document::parse(&new_bytes).map_err(|e| ("invalid_input", e, Some("new")))?; - scope - .validate_pair(&old, &new) - .map_err(|e| ("invalid_scope", e, None))?; - let old = old - .selected(scope, true) - .map_err(|e| ("invalid_scope", e, Some("old")))?; - let new = new - .selected(scope, true) - .map_err(|e| ("invalid_scope", e, Some("new")))?; - let diff = Comparison::between(&old, &new); - Ok::<_, (&str, String, Option<&str>)>(( - diff.json_report(&old, &new, &inputs, options), - diff.exit_code(), - )) - }; - let (report, status) = match comparison() { - Ok(result) => result, - Err((code, message, side)) => ( - failure_report(&inputs, scope, options, code, &message, side), - 2, + let (report, outcome) = match prepare_comparison(Some(&paths[0]), Some(&paths[1]), scope) { + Ok(diff) => (diff.json_report(&inputs, options), diff.outcome()), + Err(error) => ( + failure_report(&inputs, scope, options, &error), + Outcome::Failed, ), }; if let Err(error) = json_stdout(&report) { @@ -134,7 +163,7 @@ fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Resu eprintln!("pheon: {error}"); return Ok(2); } - Ok(status) + Ok(outcome.exit_code()) } fn run(args: &[OsString]) -> Result { @@ -252,40 +281,25 @@ fn run(args: &[OsString]) -> Result { let output = if command == "dump" { doc.canonical_dump() } else { - hierarchy(&doc, &doc, None, format == "mermaid", depth) + doc.hierarchy(format == "mermaid", depth) }; stdout(&output)?; if command == "hierarchy" { - for issue in &doc.issues { - eprintln!("INCOMPLETE: {issue}"); + for diagnostic in &doc.diagnostics { + eprintln!("INCOMPLETE: {}", diagnostic.message); } } - return Ok(if doc.issues.is_empty() { 0 } else { 2 }); + return Ok(if doc.diagnostics.is_empty() { 0 } else { 2 }); } let git = command == "git-diff"; let old_missing = git && (paths[3] == "000000" || paths[3] == "."); let new_missing = git && (paths[6] == "000000" || paths[6] == "."); - if old_missing && new_missing { - return Err("both Git revisions are absent".to_string()); - } - let old_bytes = if old_missing { - None - } else { - Some(read(&paths[if git { 1 } else { 0 }])?) - }; - let new_bytes = if new_missing { - None - } else { - Some(read(&paths[if git { 4 } else { 1 }])?) - }; - let old = old_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; - let new = new_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; - let old = old.unwrap_or_else(|| new.as_ref().unwrap().empty_like()); - let new = new.unwrap_or_else(|| old.empty_like()); - scope.validate_pair(&old, &new)?; - let old = old.selected(&scope, true)?; - let new = new.selected(&scope, true)?; - let diff = Comparison::between(&old, &new); + let diff = prepare_comparison( + (!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()), + (!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()), + &scope, + ) + .map_err(|error| error.message)?; let mut output = String::new(); if git { if paths.len() == 9 { @@ -307,13 +321,7 @@ fn run(args: &[OsString]) -> Result { if format == "mermaid" { output.push_str("```mermaid\n"); } - output.push_str(&hierarchy( - &old, - &new, - Some(&diff), - format == "mermaid", - depth, - )); + output.push_str(&diff.hierarchy(format == "mermaid", depth)); if format == "mermaid" { output.push_str("```\n"); } @@ -321,10 +329,10 @@ fn run(args: &[OsString]) -> Result { stdout(&output)?; // Git's external driver expects successful rendering for either equality // or differences. Actual comparison/input failures remain nonzero. - Ok(if git && diff.exit_code() == 1 { + Ok(if git && diff.outcome().exit_code() == 1 { 0 } else { - diff.exit_code() + diff.outcome().exit_code() }) } @@ -363,9 +371,7 @@ fn main() -> ExitCode { &ReportInputs::default(), &Scope::default(), &ReportOptions::default(), - "command_error", - &message, - None, + &Diagnostic::new("command_error", message, None), ); if let Err(error) = json_stdout(&report) { eprintln!("pheon: {error}"); diff --git a/src/review.rs b/src/review.rs index 918ddeb..8a63291 100644 --- a/src/review.rs +++ b/src/review.rs @@ -5,22 +5,39 @@ use crate::{basic::decode_real8, record::RecordHeader, records::*}; use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet}, - fmt::Write, + fmt::{self, Write}, }; -mod names; -pub use names::{parse_layer, parse_layer_name, write_layer_names}; -mod selection; -pub use selection::{DisplayFilter, Scope}; -mod changes; -pub use changes::Replacement; +pub mod changes; +pub mod names; pub mod report; +pub mod selection; -/// A comparison limitation with source context independent of its wording. +use changes::Replacement; +use selection::{DisplayFilter, Scope}; + +/// Input revision associated with a diagnostic. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Side { + Old, + New, +} + +impl fmt::Display for Side { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + out.write_str(match self { + Self::Old => "old", + Self::New => "new", + }) + } +} + +/// A review error or comparison limitation, with structured source context. #[derive(Clone, Debug)] pub struct Diagnostic { pub code: &'static str, pub message: String, + pub side: Option, pub offset: Option, pub cell: Option, pub layer: Option, @@ -28,20 +45,79 @@ pub struct Diagnostic { } impl Diagnostic { - fn at(code: &'static str, offset: usize, message: String) -> Self { + pub fn new(code: &'static str, message: impl Into, side: Option) -> Self { Self { code, - message: error(offset, message), - offset: Some(offset), + message: message.into(), + side, + offset: None, cell: None, layer: None, target: None, } } + + fn at(code: &'static str, offset: usize, message: String) -> Self { + Self { + offset: Some(offset), + ..Self::new(code, error(offset, message), None) + } + } } -pub const PROFILE: &str = "klamath-object-v2"; -pub const LAYER_SENTINEL: &str = "KLAMATH_LAYER_NAMES_V1"; +impl fmt::Display for Diagnostic { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(side) = self.side { + write!(out, "{side}: ")?; + } + out.write_str(&self.message) + } +} + +/// The five meaningful combinations of completeness and comparison result. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Outcome { + Equal, + Different, + Incomplete { different: bool }, + Failed, +} + +impl Outcome { + pub fn status(self) -> &'static str { + match self { + Self::Equal | Self::Different => "complete", + Self::Incomplete { .. } => "incomplete", + Self::Failed => "failed", + } + } + + pub fn result(self) -> &'static str { + match self { + Self::Equal => "equal", + Self::Different | Self::Incomplete { different: true } => "different", + Self::Incomplete { different: false } | Self::Failed => "unknown", + } + } + + pub fn exit_code(self) -> u8 { + match self { + Self::Equal => 0, + Self::Different => 1, + _ => 2, + } + } + + pub fn is_equal(self) -> bool { + self == Self::Equal + } + + pub fn is_complete(self) -> bool { + matches!(self, Self::Equal | Self::Different) + } +} + +pub const PROFILE: &str = "pheon-structural-v1"; pub type Layer = (u16, u16); pub type Bounds = [i32; 4]; @@ -81,16 +157,35 @@ impl Object<'_> { .find(|(t, _)| *t == tag) .map(|(_, v)| v.as_ref()) } +} - pub fn description(&self) -> String { - let mut out = kind_name(self.kind).to_string(); - for (tag, value) in &self.key.fields { - write!(out, " {}={}", tag_name(*tag), format_data(*tag, value)).unwrap(); - } - for (attribute, value) in &self.key.properties { - write!(out, " property[{attribute}]={}", quoted(value)).unwrap(); - } - out +fn format_object<'a>( + out: &mut fmt::Formatter<'_>, + kind: u16, + fields: impl IntoIterator, + properties: impl IntoIterator, +) -> fmt::Result { + out.write_str(kind_name(kind))?; + for (tag, value) in fields { + write!(out, " {}={}", tag_name(tag), format_data(tag, value))?; + } + for (attribute, value) in properties { + write!(out, " property[{attribute}]={}", quoted(value))?; + } + Ok(()) +} + +impl fmt::Display for Object<'_> { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + format_object( + out, + self.kind, + self.key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + self.key.properties.iter().copied(), + ) } } @@ -110,7 +205,6 @@ pub struct Document<'a> { pub cells: BTreeMap>, pub layer_names: BTreeMap, /// Unsupported features or invalid reference graphs prevent an equality claim. - pub issues: Vec, pub diagnostics: Vec, header: Vec<(u16, &'a [u8])>, global_issues: Vec, @@ -452,7 +546,6 @@ impl<'a> Document<'a> { records, cells: BTreeMap::new(), layer_names: BTreeMap::new(), - issues: Vec::new(), diagnostics: Vec::new(), header: Vec::new(), global_issues: Vec::new(), @@ -594,64 +687,10 @@ impl<'a> Document<'a> { doc.header.sort(); doc.read_metadata()?; doc.known_cells = doc.cells.keys().cloned().collect(); - doc.refresh_issues(); + doc.refresh_diagnostics(); Ok(doc) } - fn read_metadata(&mut self) -> Result<(), String> { - for (cell_name, cell) in &mut self.cells { - for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { - let text = object.field(RTAG_STRING).unwrap(); - if text.starts_with(b"KLAMATH_LAYER_NAMES_") && text != LAYER_SENTINEL.as_bytes() { - // Layer names apply across the library. An unknown version - // may describe layers used outside this carrier or cell, so - // geometry scope selection cannot establish its relevance. - let mut diagnostic = Diagnostic::at( - "unsupported_metadata_version", - object.offset, - "unsupported layer-name metadata version".to_string(), - ); - diagnostic.cell = Some(cell_name.clone()); - diagnostic.layer = object.layer; - diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); - self.global_issues.push(diagnostic); - } - } - let carriers: BTreeSet<_> = cell - .objects - .iter() - .filter(|o| { - o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) - }) - .filter_map(|o| o.layer) - .collect(); - for object in &mut cell.objects { - if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) - { - continue; - } - let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; - if text != LAYER_SENTINEL { - let ((layer, dtype), name) = - parse_layer_name(&text).map_err(|message| error(object.offset, message))?; - if self - .layer_names - .get(&(layer, dtype)) - .is_some_and(|old| old != &name) - { - return Err(error( - object.offset, - format!("conflicting names for {layer}/{dtype}"), - )); - } - self.layer_names.insert((layer, dtype), name.to_string()); - } - object.metadata = true; - } - } - Ok(()) - } - fn check_graph(&mut self) { let mut unresolved = BTreeMap::new(); for (parent, cell) in &self.cells { @@ -666,6 +705,7 @@ impl<'a> Document<'a> { for ((parent, target), offset) in unresolved { self.diagnostics.push(Diagnostic { code: "unresolved_reference", + side: None, message: format!("cell {parent:?}: unresolved reference {target:?}"), offset: Some(offset), cell: Some(parent.clone()), @@ -686,6 +726,7 @@ impl<'a> Document<'a> { Some(1) => { self.diagnostics.push(Diagnostic { code: "hierarchy_cycle", + side: None, message: format!("cyclic hierarchy involving {name:?}"), offset: self.cells.get(name).map(|c| c.offset), cell: Some(name.to_string()), @@ -722,6 +763,11 @@ impl<'a> Document<'a> { edges } + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + let revision = Revision::from_document(self); + render_hierarchy(&revision, &revision, None, mermaid, max_depth) + } + pub fn canonical_dump(&self) -> String { let mut out = format!( "profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n", @@ -739,11 +785,11 @@ impl<'a> Document<'a> { writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); } for object in cell.objects.iter().filter(|o| !o.metadata) { - writeln!(out, " {}", object.description()).unwrap(); + writeln!(out, " {object}").unwrap(); } } - for issue in &self.issues { - writeln!(out, "INCOMPLETE: {issue}").unwrap(); + for diagnostic in &self.diagnostics { + writeln!(out, "INCOMPLETE: {}", diagnostic.message).unwrap(); } out } @@ -811,11 +857,25 @@ pub struct ObjectChange { pub origin: Option<[i32; 2]>, pub width: Option, pub counts: Option<[u16; 2]>, - pub description: String, pub fields: Vec, pub properties: Vec<(u16, Vec)>, } +impl fmt::Display for ObjectChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + format_object( + out, + self.kind, + self.fields + .iter() + .map(|field| (field.tag, field.data.as_slice())), + self.properties + .iter() + .map(|(attribute, data)| (*attribute, data.as_slice())), + ) + } +} + impl From<&Object<'_>> for ObjectChange { fn from(object: &Object<'_>) -> Self { Self { @@ -842,7 +902,6 @@ impl From<&Object<'_>> for ObjectChange { [1, 1] } }), - description: object.description(), fields: owned_fields( object .key @@ -860,19 +919,39 @@ impl From<&Object<'_>> for ObjectChange { } } +/// An existing cell's source location and retained attributes. +#[derive(Debug)] +pub struct CellRevision { + pub offset: usize, + pub attributes: Vec, +} + +impl From<&Cell<'_>> for CellRevision { + fn from(cell: &Cell<'_>) -> Self { + Self { + offset: cell.offset, + attributes: owned_fields(cell.attributes.iter().copied()), + } + } +} + #[derive(Debug)] pub struct CellChange { pub name: String, - pub old_offset: Option, - pub new_offset: Option, - pub attributes_changed: bool, - pub old_attributes: Vec, - pub new_attributes: Vec, + pub old: Option, + pub new: Option, pub removed: Vec, pub added: Vec, pub replacements: Vec, } +impl CellChange { + pub fn attributes_changed(&self) -> bool { + self.old.as_ref().map(|cell| &cell.attributes) + != self.new.as_ref().map(|cell| &cell.attributes) + } +} + #[derive(Debug)] pub struct LayerNameChange { pub layer: Layer, @@ -880,59 +959,117 @@ pub struct LayerNameChange { pub new: Option, } +/// Library presence or exact record payloads, formatted only when rendered. #[derive(Debug)] -pub struct LibraryFieldChange { - pub field: String, - pub tag: Option, - pub old_fields: Vec, - pub new_fields: Vec, - pub old: Vec, - pub new: Vec, +pub enum LibraryChange { + Presence { + old: bool, + new: bool, + }, + Record { + tag: u16, + old: Vec>, + new: Vec>, + }, +} + +impl fmt::Display for LibraryChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Presence { old, new } => { + let label = |present| if present { "present" } else { "absent" }; + write!(out, "presence: {:?} -> {:?}", [label(*old)], [label(*new)]) + } + Self::Record { tag, old, new } => { + write!(out, "{}: ", tag_name(*tag))?; + out.debug_list() + .entries(old.iter().map(|data| format_data(*tag, data))) + .finish()?; + out.write_str(" -> ")?; + out.debug_list() + .entries(new.iter().map(|data| format_data(*tag, data))) + .finish() + } + } + } +} + +/// Owned revision metadata and reference counts captured before documents are dropped. +#[derive(Debug)] +pub struct Revision { + pub scope: Scope, + pub present: bool, + pub units: Option>, + cells: BTreeSet, + known_cells: BTreeSet, + edges: BTreeMap<(String, String), u64>, +} + +impl Revision { + fn from_document(doc: &Document<'_>) -> Self { + Self { + scope: doc.scope.clone(), + present: doc.present, + units: doc + .header + .iter() + .find(|(tag, _)| *tag == RTAG_UNITS && doc.present) + .map(|(_, data)| data.to_vec()), + cells: doc.cells.keys().cloned().collect(), + known_cells: doc.known_cells.clone(), + edges: doc.edges(), + } + } } #[derive(Debug)] pub struct Comparison { - pub scope: Scope, + pub old: Revision, + pub new: Revision, /// Unchanged names are displayed once when objects on that layer change. pub layer_legend: BTreeMap, - pub library_fields: Vec, + pub library_changes: Vec, pub layer_names: Vec, pub cells: Vec, - pub issues: Vec, + pub diagnostics: Vec, } impl Comparison { pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { let mut result = Self { - scope: old.scope.clone(), + old: Revision::from_document(old), + new: Revision::from_document(new), layer_legend: old .layer_names .iter() .filter(|(pair, name)| new.layer_names.get(pair) == Some(*name)) .map(|(pair, name)| (*pair, name.clone())) .collect(), - library_fields: Vec::new(), + library_changes: Vec::new(), layer_names: Vec::new(), cells: Vec::new(), - issues: Vec::new(), + diagnostics: Vec::new(), }; - if old.scope != new.scope { - result.issues.push("comparison scopes differ".to_string()); - } if old.present != new.present { - result.library_fields.push(LibraryFieldChange { - field: "presence".to_string(), - tag: None, - old_fields: Vec::new(), - new_fields: Vec::new(), - old: vec![if old.present { "present" } else { "absent" }.to_string()], - new: vec![if new.present { "present" } else { "absent" }.to_string()], + result.library_changes.push(LibraryChange::Presence { + old: old.present, + new: new.present, }); } - for (side, doc) in [("old", old), ("new", new)] { + for (side, doc) in [(Side::Old, old), (Side::New, new)] { result - .issues - .extend(doc.issues.iter().map(|i| format!("{side}: {i}"))); + .diagnostics + .extend(doc.diagnostics.iter().cloned().map(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic + })); + } + if old.scope != new.scope { + result.diagnostics.push(Diagnostic::new( + "scope_mismatch", + "comparison scopes differ", + None, + )); } let header_tags: BTreeSet<_> = old .header @@ -954,13 +1091,10 @@ impl Comparison { .map(|(_, value)| *value) .collect(); if a != b { - result.library_fields.push(LibraryFieldChange { - field: tag_name(tag), - tag: Some(tag), - old_fields: owned_fields(a.iter().map(|data| (tag, *data))), - new_fields: owned_fields(b.iter().map(|data| (tag, *data))), - old: a.iter().map(|data| format_data(tag, data)).collect(), - new: b.iter().map(|data| format_data(tag, data)).collect(), + result.library_changes.push(LibraryChange::Record { + tag, + old: a.into_iter().map(<[u8]>::to_vec).collect(), + new: b.into_iter().map(<[u8]>::to_vec).collect(), }); } } @@ -985,15 +1119,8 @@ impl Comparison { let b = new.cells.get(name); let mut change = CellChange { name: name.clone(), - old_offset: a.map(|c| c.offset), - new_offset: b.map(|c| c.offset), - attributes_changed: a.map(|c| &c.attributes) != b.map(|c| &c.attributes), - old_attributes: owned_fields( - a.into_iter().flat_map(|c| c.attributes.iter().copied()), - ), - new_attributes: owned_fields( - b.into_iter().flat_map(|c| c.attributes.iter().copied()), - ), + old: a.map(CellRevision::from), + new: b.map(CellRevision::from), removed: Vec::new(), added: Vec::new(), replacements: Vec::new(), @@ -1030,11 +1157,7 @@ impl Comparison { } } } - if change.old_offset.is_none() - || change.new_offset.is_none() - || change.attributes_changed - || !change.removed.is_empty() - || !change.added.is_empty() + if change.attributes_changed() || !change.removed.is_empty() || !change.added.is_empty() { change.replacements = changes::replacements(&change.removed, &change.added); result.cells.push(change); @@ -1043,21 +1166,21 @@ impl Comparison { result } - pub fn is_equal(&self) -> bool { - self.issues.is_empty() - && self.library_fields.is_empty() - && self.layer_names.is_empty() - && self.cells.is_empty() + pub fn outcome(&self) -> Outcome { + let different = !self.library_changes.is_empty() + || !self.layer_names.is_empty() + || !self.cells.is_empty(); + if !self.diagnostics.is_empty() { + Outcome::Incomplete { different } + } else if different { + Outcome::Different + } else { + Outcome::Equal + } } - pub fn exit_code(&self) -> u8 { - if !self.issues.is_empty() { - 2 - } else if self.is_equal() { - 0 - } else { - 1 - } + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) } /// Limit presentation only: the complete comparison has already been made. @@ -1066,19 +1189,22 @@ impl Comparison { } pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { - let mut out = format!("profile {PROFILE}\n{}\n", self.scope.description()); + let mut out = format!("profile {PROFILE}\n{}\n", self.old.scope.description()); if show.cell.is_some() || show.layer.is_some() { writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap(); } - for change in &self.library_fields { - writeln!( - out, - "library {}: {:?} -> {:?}", - change.field, change.old, change.new - ) - .unwrap(); + for change in &self.library_changes { + writeln!(out, "library {change}").unwrap(); } - if self.library_fields.iter().any(|c| c.field == "UNITS") { + if self.library_changes.iter().any(|change| { + matches!( + change, + LibraryChange::Record { + tag: RTAG_UNITS, + .. + } + ) + }) { out.push_str("UNITS changed; coordinates remain in each file's database units\n"); } for change in &self.layer_names { @@ -1116,9 +1242,9 @@ impl Comparison { out, "cell {:?} old@{:?} new@{:?}{}", cell.name, - cell.old_offset, - cell.new_offset, - if cell.attributes_changed { + cell.old.as_ref().map(|revision| revision.offset), + cell.new.as_ref().map(|revision| revision.offset), + if cell.attributes_changed() { " [cell/header changed]" } else { "" @@ -1135,19 +1261,9 @@ impl Comparison { .layer .is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer)) { - writeln!(out, " ~ {}", replacement.explanation).unwrap(); - writeln!( - out, - " - @{} bbox={:?} {}", - a.offset, a.bounds, a.description - ) - .unwrap(); - writeln!( - out, - " + @{} bbox={:?} {}", - b.offset, b.bounds, b.description - ) - .unwrap(); + writeln!(out, " ~ {}", changes::explain(a, b)).unwrap(); + writeln!(out, " - @{} bbox={:?} {}", a.offset, a.bounds, a).unwrap(); + writeln!(out, " + @{} bbox={:?} {}", b.offset, b.bounds, b).unwrap(); paired_old.insert(replacement.removed); paired_new.insert(replacement.added); shown += 2; @@ -1168,7 +1284,7 @@ impl Comparison { writeln!( out, " {sign} @{} bbox={:?} {}", - object.offset, object.bounds, object.description + object.offset, object.bounds, object ) .unwrap(); shown += 1; @@ -1183,55 +1299,93 @@ impl Comparison { self.layer_names.len() ) .unwrap(); - for issue in &self.issues { - writeln!(out, "INCOMPLETE: {issue}").unwrap(); + // General comparison errors precede revision-specific errors in text. + for diagnostic in self + .diagnostics + .iter() + .filter(|d| d.side.is_none()) + .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) + { + writeln!(out, "INCOMPLETE: {diagnostic}").unwrap(); } - if self.is_equal() { + if self.outcome().is_equal() { out.push_str("equal under this profile\n"); } out } } +/// Graph facts shared by text, Mermaid, and JSON renderers. +struct HierarchyAnalysis<'a> { + old_edges: &'a BTreeMap<(String, String), u64>, + new_edges: &'a BTreeMap<(String, String), u64>, + edges: BTreeSet<(String, String)>, + direct: BTreeSet, + affected: BTreeSet, + changed_refs: BTreeSet<(String, String)>, +} + +impl<'a> HierarchyAnalysis<'a> { + fn between(old: &'a Revision, new: &'a Revision, changes: Option<&[CellChange]>) -> Self { + let old_edges = &old.edges; + let new_edges = &new.edges; + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = changes + .into_iter() + .flatten() + .map(|cell| cell.name.clone()) + .collect(); + let changed_refs: BTreeSet<_> = changes + .into_iter() + .flatten() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + Self { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + } + } +} + /// The graph joins both revisions, retaining removed branches and old callers. -pub fn hierarchy( - old: &Document<'_>, - new: &Document<'_>, - diff: Option<&Comparison>, +fn render_hierarchy( + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, mermaid: bool, max_depth: usize, ) -> String { - let old_edges = old.edges(); - let new_edges = new.edges(); - let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); - let direct: BTreeSet<_> = diff - .into_iter() - .flat_map(|d| d.cells.iter().map(|c| c.name.clone())) - .collect(); - let changed_refs: BTreeSet<_> = diff - .into_iter() - .flat_map(|d| &d.cells) - .flat_map(|c| { - c.removed - .iter() - .chain(&c.added) - .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) - }) - .collect(); - let mut parents = BTreeMap::<&str, Vec<&str>>::new(); - for (parent, child) in &edges { - parents.entry(child).or_default().push(parent); - } - let mut affected = direct.clone(); - let mut pending: Vec<_> = direct.iter().cloned().collect(); - while let Some(child) = pending.pop() { - for parent in parents.get(child.as_str()).into_iter().flatten() { - if affected.insert((*parent).to_string()) { - pending.push((*parent).to_string()); - } - } - } - let visible: BTreeSet<_> = if diff.is_some() { + let HierarchyAnalysis { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + } = HierarchyAnalysis::between(old, new, changes); + let visible: BTreeSet<_> = if changes.is_some() { affected .iter() .cloned() @@ -1239,8 +1393,8 @@ pub fn hierarchy( .collect() } else { old.cells - .keys() - .chain(new.cells.keys()) + .iter() + .chain(new.cells.iter()) .cloned() .chain(edges.iter().map(|(_, t)| t.clone())) .collect() @@ -1292,7 +1446,7 @@ pub fn hierarchy( rows.push((name.clone(), 0, None, false, true)); } let status = |name: &String| { - if !old.cells.contains_key(name) && !new.cells.contains_key(name) { + if !old.cells.contains(name) && !new.cells.contains(name) { if old.known_cells.contains(name) || new.known_cells.contains(name) { " [outside scope]" } else { @@ -1302,7 +1456,7 @@ pub fn hierarchy( " [cell changed]" } else if affected.contains(name) { " [affected]" - } else if diff.is_some() { + } else if changes.is_some() { " [cell unchanged]" } else { "" diff --git a/src/review/changes.rs b/src/review/changes.rs index d4c2f9a..7c8d3b8 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -1,10 +1,11 @@ +//! Unique replacement matches that explain removed and added objects. + use super::*; #[derive(Debug)] pub struct Replacement { pub removed: usize, pub added: usize, - pub explanation: String, /// Explanation matching is never object identity or geometric equivalence. pub match_rule: &'static str, } @@ -56,7 +57,6 @@ pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec Vec String { +pub(super) fn explain(old: &ObjectChange, new: &ObjectChange) -> String { let mut parts = vec![format!( "{} modified{}", kind_name(old.kind), diff --git a/src/review/names.rs b/src/review/names.rs index a3040d5..22d576a 100644 --- a/src/review/names.rs +++ b/src/review/names.rs @@ -1,5 +1,66 @@ +//! Embedded layer names: parsing, recognition, and record-preserving updates. + use super::*; +const LAYER_PREFIX: &[u8] = b"PHEON_LAYER_NAMES_"; +pub const LAYER_SENTINEL: &str = "PHEON_LAYER_NAMES_V1"; + +impl Document<'_> { + pub(super) fn read_metadata(&mut self) -> Result<(), String> { + for (cell_name, cell) in &mut self.cells { + for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { + let text = object.field(RTAG_STRING).unwrap(); + if text.starts_with(LAYER_PREFIX) && text != LAYER_SENTINEL.as_bytes() { + // Layer names apply across the library. An unknown version + // may describe layers used outside this carrier or cell, so + // geometry scope selection cannot establish its relevance. + let mut diagnostic = Diagnostic::at( + "unsupported_metadata_version", + object.offset, + "unsupported layer-name metadata version".to_string(), + ); + diagnostic.cell = Some(cell_name.clone()); + diagnostic.layer = object.layer; + diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); + self.global_issues.push(diagnostic); + } + } + let carriers: BTreeSet<_> = cell + .objects + .iter() + .filter(|o| { + o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) + }) + .filter_map(|o| o.layer) + .collect(); + for object in &mut cell.objects { + if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) + { + continue; + } + let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; + if text != LAYER_SENTINEL { + let ((layer, dtype), name) = + parse_layer_name(&text).map_err(|message| error(object.offset, message))?; + if self + .layer_names + .get(&(layer, dtype)) + .is_some_and(|old| old != &name) + { + return Err(error( + object.offset, + format!("conflicting names for {layer}/{dtype}"), + )); + } + self.layer_names.insert((layer, dtype), name.to_string()); + } + object.metadata = true; + } + } + Ok(()) + } +} + pub fn parse_layer(value: &str) -> Result { let (a, b) = value .split_once('/') diff --git a/src/review/report.rs b/src/review/report.rs index 7c0c445..aa1a0c1 100644 --- a/src/review/report.rs +++ b/src/review/report.rs @@ -27,32 +27,37 @@ fn scope(value: &Scope) -> Value { json!({"top": value.top, "cell": value.cell, "layer": value.layer.map(layer)}) } -fn input(path: Option<&Path>, doc: Option<&Document<'_>>) -> Value { - let units = doc - .filter(|d| d.present) - .and_then(|d| d.header.iter().find(|(tag, _)| *tag == RTAG_UNITS)) - .map(|(tag, data)| field(*tag, data)); +fn input(path: Option<&Path>, revision: Option<&Revision>) -> Value { + let units = revision + .and_then(|r| r.units.as_deref()) + .map(|data| field(RTAG_UNITS, data)); json!({ "path": path.and_then(Path::to_str), "path_encoded_hex": path.filter(|p| p.to_str().is_none()) .map(|p| hex(p.as_os_str().as_encoded_bytes())), - "present": doc.map(|d| d.present), + "present": revision.map(|r| r.present), "units": units }) } -fn envelope(inputs: &ReportInputs, selection: &Scope, options: &ReportOptions) -> Value { +fn envelope( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + outcome: Outcome, + revisions: Option<(&Revision, &Revision)>, +) -> Value { json!({ "schema_version": SCHEMA_VERSION, "kind": "diff", "producer": {"name": "pheon", "version": env!("CARGO_PKG_VERSION")}, "comparison_profile": PROFILE, - "inputs": {"old": input(inputs.old.as_deref(), None), "new": input(inputs.new.as_deref(), None)}, + "inputs": {"old": input(inputs.old.as_deref(), revisions.map(|r| r.0)), "new": input(inputs.new.as_deref(), revisions.map(|r| r.1))}, "scope": scope(selection), "coordinates": {"unit": "database_unit", "frame": "cell_local", "bounds": "coordinate_envelope"}, - "status": "failed", - "result": "unknown", - "exit_code": 2, + "status": outcome.status(), + "result": outcome.result(), + "exit_code": outcome.exit_code(), "summary": null, "report_selection": { "cell": options.filter.cell, @@ -74,18 +79,22 @@ pub fn failure_report( inputs: &ReportInputs, selection: &Scope, options: &ReportOptions, - code: &str, - message: &str, - side: Option<&str>, + error: &Diagnostic, ) -> Value { - let mut report = envelope(inputs, selection, options); - report["diagnostics"] = json!([{ - "code": code, "severity": "error", "message": message, - "side": side, "byte_offset": null, "cell": null, "layer": null, "target": null - }]); + let mut report = envelope(inputs, selection, options, Outcome::Failed, None); + report["diagnostics"] = json!([diagnostic(error)]); report } +fn diagnostic(value: &Diagnostic) -> Value { + json!({ + "code": value.code, "severity": "error", "message": value.message, + "side": value.side.map(|side| side.to_string()), + "byte_offset": value.offset.map(|offset| offset.to_string()), + "cell": value.cell, "layer": value.layer.map(layer), "target": value.target + }) +} + fn field(tag: u16, data: &[u8]) -> Value { let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { ("xy", json!(points(data))) @@ -195,40 +204,24 @@ fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { } impl Comparison { - /// Render evidence from the same documents used to construct this comparison. + /// Render the captured comparison evidence without retaining its source documents. /// IDs are local to this report, not persistent object identities. - pub fn json_report( - &self, - old: &Document<'_>, - new: &Document<'_>, - inputs: &ReportInputs, - options: &ReportOptions, - ) -> Value { - let mut report = envelope(inputs, &self.scope, options); - report["inputs"] = json!({"old": input(inputs.old.as_deref(), Some(old)), "new": input(inputs.new.as_deref(), Some(new))}); - let different = !self.cells.is_empty() - || !self.library_fields.is_empty() - || !self.layer_names.is_empty(); - report["status"] = json!(if self.issues.is_empty() { - "complete" - } else { - "incomplete" - }); - report["result"] = json!(if different { - "different" - } else if self.is_equal() { - "equal" - } else { - "unknown" - }); - report["exit_code"] = json!(self.exit_code()); + pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value { + let outcome = self.outcome(); + let mut report = envelope( + inputs, + &self.old.scope, + options, + outcome, + Some((&self.old, &self.new)), + ); report["summary"] = json!({ - "counts": if self.issues.is_empty() { "exact" } else { "observed" }, + "counts": if outcome.is_complete() { "exact" } else { "observed" }, "changed_cells": self.cells.len().to_string(), "removed_objects": self.cells.iter().map(|c| c.removed.len()).sum::().to_string(), "added_objects": self.cells.iter().map(|c| c.added.len()).sum::().to_string(), "layer_name_changes": self.layer_names.len().to_string(), - "library_field_changes": self.library_fields.len().to_string() + "library_field_changes": self.library_changes.len().to_string() }); report["layer_legend"] = json!( self.layer_legend @@ -244,11 +237,17 @@ impl Comparison { })) .collect::>() ); - report["library_changes"] = json!(self.library_fields.iter().map(|c| { - if c.tag.is_none() { - json!({"field": "presence", "tag": null, "before": old.present, "after": new.present}) - } else { - json!({"field": c.field, "tag": c.tag, "before": fields(&c.old_fields), "after": fields(&c.new_fields)}) + report["library_changes"] = json!(self.library_changes.iter().map(|change| { + match change { + LibraryChange::Presence { old, new } => { + json!({"field": "presence", "tag": null, "before": old, "after": new}) + } + LibraryChange::Record { tag, old, new } => { + let values = |data: &[Vec]| { + data.iter().map(|bytes| field(*tag, bytes)).collect::>() + }; + json!({"field": tag_name(*tag), "tag": tag, "before": values(old), "after": values(new)}) + } } }).collect::>()); let (mut emitted, mut filtered_objects, mut limited, mut filtered_cells) = @@ -293,16 +292,16 @@ impl Comparison { let b = format!("c{cell_index}:new:{}", r.added); (shown.contains(&a) && shown.contains(&b)).then(|| json!({ "removed": a, "added": b, "match_rule": r.match_rule, - "explanation": r.explanation, + "explanation": changes::explain(&cell.removed[r.removed], &cell.added[r.added]), "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties })) }).collect(); cells.push(json!({ "name": cell.name, - "before": cell.old_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.old_attributes)})), - "after": cell.new_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.new_attributes)})), - "attributes_changed": cell.attributes_changed, + "before": cell.old.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})), + "after": cell.new.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})), + "attributes_changed": cell.attributes_changed(), "removed": removed, "added": added, "replacements": replacements })); } @@ -317,37 +316,29 @@ impl Comparison { ] { report["report_selection"][key] = json!(count.to_string()); } - let mut diagnostics: Vec<_> = [("old", old), ("new", new)] - .into_iter() - .flat_map(|(side, doc)| { - doc.diagnostics.iter().map(move |d| { - json!({ - "code": d.code, "severity": "error", "message": d.message, "side": side, - "byte_offset": d.offset.map(|n| n.to_string()), "cell": d.cell, - "layer": d.layer.map(layer), "target": d.target - }) - }) - }) - .collect(); - if old.scope != new.scope { - diagnostics.push(json!({"code": "scope_mismatch", "severity": "error", "message": "comparison scopes differ", "side": null, "byte_offset": null, "cell": null, "layer": null, "target": null})); - } - report["diagnostics"] = json!(diagnostics); + report["diagnostics"] = json!(self.diagnostics.iter().map(diagnostic).collect::>()); if options.hierarchy { - report["hierarchy"] = graph(old, new, self); + report["hierarchy"] = graph(self); } report } } -fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { - let a = old.edges(); - let b = new.edges(); - let edges: BTreeSet<_> = a.keys().chain(b.keys()).cloned().collect(); +fn graph(diff: &Comparison) -> Value { + let old = &diff.old; + let new = &diff.new; + let HierarchyAnalysis { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + } = HierarchyAnalysis::between(old, new, Some(&diff.cells)); let names: BTreeSet<_> = old .cells - .keys() - .chain(new.cells.keys()) + .iter() + .chain(new.cells.iter()) .cloned() .chain( edges @@ -355,34 +346,10 @@ fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { .flat_map(|(parent, child)| [parent.clone(), child.clone()]), ) .collect(); - let direct: BTreeSet<_> = diff.cells.iter().map(|c| c.name.clone()).collect(); - let mut affected = direct.clone(); - let mut parents = BTreeMap::<&str, Vec<&str>>::new(); - for (parent, child) in &edges { - parents.entry(child).or_default().push(parent); - } - let mut pending: Vec<_> = direct.iter().cloned().collect(); - while let Some(child) = pending.pop() { - for parent in parents.get(child.as_str()).into_iter().flatten() { - if affected.insert((*parent).to_string()) { - pending.push((*parent).to_string()); - } - } - } - let changed_refs: BTreeSet<_> = diff - .cells - .iter() - .flat_map(|c| { - c.removed - .iter() - .chain(&c.added) - .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) - }) - .collect(); - let a_targets: BTreeSet<_> = a.keys().map(|(_, t)| t.as_str()).collect(); - let b_targets: BTreeSet<_> = b.keys().map(|(_, t)| t.as_str()).collect(); - let state = |doc: &Document<'_>, targets: &BTreeSet<&str>, name: &String| { - if doc.cells.contains_key(name) { + let old_targets: BTreeSet<_> = old_edges.keys().map(|(_, t)| t.as_str()).collect(); + let new_targets: BTreeSet<_> = new_edges.keys().map(|(_, t)| t.as_str()).collect(); + let state = |doc: &Revision, targets: &BTreeSet<&str>, name: &String| { + if doc.cells.contains(name) { "included" } else if doc.known_cells.contains(name) { "outside_scope" @@ -394,13 +361,13 @@ fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { }; json!({ "nodes": names.iter().map(|name| json!({ - "cell": name, "before": state(old, &a_targets, name), "after": state(new, &b_targets, name), + "cell": name, "before": state(old, &old_targets, name), "after": state(new, &new_targets, name), "change": if direct.contains(name) { "direct" } else if affected.contains(name) { "affected" } else { "unchanged" } })).collect::>(), "edges": edges.iter().map(|edge| json!({ "parent": edge.0, "target": edge.1, - "before_count": a.get(edge).copied().unwrap_or(0).to_string(), - "after_count": b.get(edge).copied().unwrap_or(0).to_string(), + "before_count": old_edges.get(edge).copied().unwrap_or(0).to_string(), + "after_count": new_edges.get(edge).copied().unwrap_or(0).to_string(), "reference_changed": changed_refs.contains(edge) })).collect::>() }) diff --git a/src/review/selection.rs b/src/review/selection.rs index b87fbe6..820352c 100644 --- a/src/review/selection.rs +++ b/src/review/selection.rs @@ -1,3 +1,5 @@ +//! Comparison scope and display-only filters. + use super::*; #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -64,7 +66,6 @@ impl<'a> Document<'a> { records: Vec::new(), cells: BTreeMap::new(), layer_names: BTreeMap::new(), - issues: Vec::new(), diagnostics: Vec::new(), header: self.header.clone(), global_issues: Vec::new(), @@ -86,7 +87,7 @@ impl<'a> Document<'a> { found } - pub(super) fn refresh_issues(&mut self) { + pub(super) fn refresh_diagnostics(&mut self) { self.diagnostics = self.global_issues.clone(); for (name, cell) in &self.cells { for issue in &cell.issues { @@ -106,7 +107,6 @@ impl<'a> Document<'a> { } } self.check_graph(); - self.issues = self.diagnostics.iter().map(|d| d.message.clone()).collect(); } /// Project a parsed document. Missing selectors are allowed only for one @@ -227,7 +227,7 @@ impl<'a> Document<'a> { self.layer_names.retain(|pair, _| layers.contains(pair)); } self.scope = scope.clone(); - self.refresh_issues(); + self.refresh_diagnostics(); Ok(self) } } diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json index 75c2e4b..2ef560a 100644 --- a/tests/golden/report-equal.json +++ b/tests/golden/report-equal.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", diff --git a/tests/golden/report-failed.json b/tests/golden/report-failed.json index c9b12d9..b4de07a 100644 --- a/tests/golden/report-failed.json +++ b/tests/golden/report-failed.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json index 763de12..ba0797f 100644 --- a/tests/golden/report-geometry.json +++ b/tests/golden/report-geometry.json @@ -225,7 +225,7 @@ ] } ], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json index 4319c1f..b1871dd 100644 --- a/tests/golden/report-incomplete.json +++ b/tests/golden/report-incomplete.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json index 3e1245a..2b219da 100644 --- a/tests/golden/report-metadata.json +++ b/tests/golden/report-metadata.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt index 8811a25..9ef3ac6 100644 --- a/tests/golden/review-canonical.txt +++ b/tests/golden/review-canonical.txt @@ -1,4 +1,4 @@ -profile klamath-object-v2 +profile pheon-structural-v1 scope top=all cell=all layer=all coordinates: integer database units; bounds: coordinate envelopes HEADER=[600] diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt index 57e6c09..bb10311 100644 --- a/tests/golden/review-diff.txt +++ b/tests/golden/review-diff.txt @@ -1,4 +1,4 @@ -profile klamath-object-v2 +profile pheon-structural-v1 scope top=all cell=all layer=all cell "CHIP" old@Some(62) new@Some(62) ~ BOUNDARY modified; coordinate bounds unchanged diff --git a/tests/review.rs b/tests/review.rs index 71b895b..3f49221 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -5,7 +5,7 @@ mod reports; use common::{library_header, record}; use pheon::{ records::*, - review::{Comparison, Document, LAYER_SENTINEL, hierarchy}, + review::{Comparison, Document, names::LAYER_SENTINEL}, }; use std::{fs, path::PathBuf, process::Command}; @@ -251,7 +251,12 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { let old = Document::parse(&pair.old).unwrap(); let new = Document::parse(&pair.new).unwrap(); let diff = Comparison::between(&old, &new); - assert!(diff.issues.is_empty(), "{}: {:?}", pair.name, diff.issues); + assert!( + diff.diagnostics.is_empty(), + "{}: {:?}", + pair.name, + diff.diagnostics + ); assert_eq!(diff.cells.len(), pair.changed_cells, "{}", pair.name); assert_eq!( diff.cells.iter().map(|c| c.removed.len()).sum::(), @@ -267,14 +272,14 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { ); assert_eq!(diff.layer_names.len(), pair.metadata, "{}", pair.name); let equal = pair.changed_cells + pair.metadata == 0; - assert_eq!(diff.is_equal(), equal, "{}", pair.name); + assert_eq!(diff.outcome().is_equal(), equal, "{}", pair.name); assert_eq!( old.canonical_dump() == new.canonical_dump(), equal, "{}", pair.name ); - assert_eq!(diff.exit_code(), if equal { 0 } else { 1 }); + assert_eq!(diff.outcome().exit_code(), if equal { 0 } else { 1 }); } } @@ -317,11 +322,11 @@ fn canonical_dump_and_hierarchy_match_versioned_goldens() { let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); assert_eq!( - hierarchy(&a, &b, Some(&diff), false, 8), + diff.hierarchy(false, 8), include_str!("golden/review-hierarchy.txt") ); assert_eq!( - hierarchy(&a, &b, Some(&diff), true, 8), + diff.hierarchy(true, 8), include_str!("golden/review-hierarchy.mmd") ); } @@ -352,7 +357,8 @@ fn cell_and_property_order_do_not_affect_comparison() { let b = layout(&[("a", vec![]), ("z", vec![b])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); - assert!(Comparison::between(&a, &b).is_equal()); + assert_ne!(a.cells["z"].offset, b.cells["z"].offset); + assert!(Comparison::between(&a, &b).outcome().is_equal()); assert_eq!(a.canonical_dump(), b.canonical_dump()); } @@ -393,9 +399,12 @@ fn metadata_requires_a_sentinel_in_the_same_cell_and_carrier_pair() { .unwrap() .contains("1:0:B") ); - let future = layout(&[("CHIP", vec![label((9, 0), "KLAMATH_LAYER_NAMES_V2")])]); + let future = layout(&[("CHIP", vec![label((9, 0), "PHEON_LAYER_NAMES_V2")])]); let future = Document::parse(&future).unwrap(); - assert_eq!(Comparison::between(&future, &future).exit_code(), 2); + assert_eq!( + Comparison::between(&future, &future).outcome().exit_code(), + 2 + ); } #[test] @@ -407,8 +416,8 @@ fn unknown_records_never_produce_a_successful_equality_claim() { let bytes = layout(&[("CHIP", vec![unknown])]); let doc = Document::parse(&bytes).unwrap(); let diff = Comparison::between(&doc, &doc); - assert!(!diff.is_equal()); - assert_eq!(diff.exit_code(), 2); + assert!(!diff.outcome().is_equal()); + assert_eq!(diff.outcome().exit_code(), 2); assert!(doc.record_dump().contains("7f00")); assert!(diff.text(0).contains("INCOMPLETE")); } @@ -421,7 +430,7 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); assert_eq!( - hierarchy(&a, &b, Some(&diff), false, 8), + diff.hierarchy(false, 8), concat!( "scope top=all cell=all layer=all\n", "\"CHIP\" x1 [affected]\n", @@ -431,11 +440,11 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { " \"leaf\" x3 [cell changed] [shown above]\n", ) ); - let graph = hierarchy(&a, &b, Some(&diff), true, 8); + let graph = diff.hierarchy(true, 8); assert_eq!(graph.matches("leaf [cell changed]").count(), 1); assert!(graph.contains("x2")); assert!(graph.contains("x3")); - let limited = hierarchy(&a, &b, Some(&diff), false, 1); + let limited = diff.hierarchy(false, 1); assert!(!limited.contains("\"leaf\"")); assert!(limited.contains("children omitted")); } @@ -449,7 +458,7 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let old = Document::parse(&pair.old).unwrap(); let new = Document::parse(&pair.new).unwrap(); let diff = Comparison::between(&old, &new); - let tree = hierarchy(&old, &new, Some(&diff), false, 8); + let tree = diff.hierarchy(false, 8); assert!(tree.contains("[cell unchanged]"), "{tree}"); assert!(tree.contains("[reference changed]"), "{tree}"); if pair.name == "array count" { @@ -463,7 +472,7 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let b = layout(&[("CHIP", vec![])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); - let tree = hierarchy(&a, &b, Some(&Comparison::between(&a, &b)), false, 8); + let tree = Comparison::between(&a, &b).hierarchy(false, 8); assert!(tree.contains("\"leaf\" x1 -> x0"), "{tree}"); } @@ -479,7 +488,7 @@ fn same_bounds_changes_and_truncated_reports_remain_differences() { diff.cells[0].added[0].bounds ); assert!(diff.text(0).contains("0 of 2 object changes shown")); - assert_eq!(diff.exit_code(), 1); + assert_eq!(diff.outcome().exit_code(), 1); assert!(diff.cells[0].removed[0].offset > 0); } @@ -494,17 +503,19 @@ fn unit_changes_report_values_without_claiming_geometry_equivalence() { .unwrap() .offset; let mut b = a.clone(); - b[units + 12..units + 20].copy_from_slice( - &pheon::basic::encode_real8(2e-9) - .unwrap() - .to_be_bytes(), - ); + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); - assert_eq!(diff.exit_code(), 1); - assert_eq!(diff.library_fields.len(), 1); - assert_eq!(diff.library_fields[0].field, "UNITS"); + assert_eq!(diff.outcome().exit_code(), 1); + assert!(matches!( + diff.library_changes.as_slice(), + [pheon::review::LibraryChange::Record { + tag: RTAG_UNITS, + .. + }] + )); assert!(diff.text(0).contains("UNITS changed")); assert!(diff.cells.is_empty()); } @@ -515,17 +526,22 @@ fn dangling_references_cycles_and_duplicate_cell_names_are_reported() { let missing = Document::parse(&missing).unwrap(); assert!( missing - .issues + .diagnostics .iter() - .any(|s| s.contains("unresolved reference")) + .any(|d| d.code == "unresolved_reference") ); let cycle = layout(&[ ("a", vec![reference("b", 0, 1)]), ("b", vec![reference("a", 0, 1)]), ]); let cycle = Document::parse(&cycle).unwrap(); - assert!(cycle.issues.iter().any(|s| s.contains("cyclic"))); - assert!(hierarchy(&cycle, &cycle, None, false, 8).len() < 200); + assert!( + cycle + .diagnostics + .iter() + .any(|d| d.code == "hierarchy_cycle") + ); + assert!(cycle.hierarchy(false, 8).len() < 200); let duplicate = layout(&[("a", vec![]), ("a", vec![])]); assert!( Document::parse(&duplicate) @@ -547,6 +563,7 @@ fn absolute_transform_flags_are_not_discarded() { let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); assert_eq!( Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .outcome() .exit_code(), 1 ); @@ -568,7 +585,11 @@ fn polygon_normalization_handles_repeated_minimum_vertices() { rotated.push(rotated[0]); let candidate = layout(&[("CHIP", vec![polygon(&rotated)])]); let candidate = Document::parse(&candidate).unwrap(); - assert!(Comparison::between(&baseline, &candidate).is_equal()); + assert!( + Comparison::between(&baseline, &candidate) + .outcome() + .is_equal() + ); } open.reverse(); } @@ -672,7 +693,7 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { #[test] fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { - use pheon::review::write_layer_names; + use pheon::review::names::write_layer_names; use std::collections::BTreeMap; let mut bytes = shared(with_property(square(), "keep this")); bytes.extend_from_slice(&[0; 32]); @@ -720,7 +741,7 @@ fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { #[test] fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { - use pheon::review::{parse_layer_name, write_layer_names}; + use pheon::review::names::{parse_layer_name, write_layer_names}; use std::collections::BTreeMap; let bytes = layout(&[("CHIP", vec![label((900, 7), "ordinary label")])]); assert!( @@ -733,7 +754,7 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { .unwrap_err() .contains("does not exist") ); - let bytes = layout(&[("CHIP", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")])]); + let bytes = layout(&[("CHIP", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")])]); assert!( write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::new()) .unwrap_err() @@ -788,7 +809,7 @@ fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { #[test] fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { - use pheon::review::Scope; + use pheon::review::selection::Scope; let a = layout(&[ ("CHIP", vec![reference("leaf", 0, 1)]), ("leaf", vec![square()]), @@ -817,8 +838,8 @@ fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { }; let old = Document::parse(&a).unwrap().selected(&cell, false).unwrap(); let new = Document::parse(&b).unwrap().selected(&cell, false).unwrap(); - assert!(old.issues.is_empty()); - assert!(Comparison::between(&old, &new).is_equal()); + assert!(old.diagnostics.is_empty()); + assert!(Comparison::between(&old, &new).outcome().is_equal()); let missing = Scope { top: Some("missing".to_string()), ..Scope::default() @@ -843,7 +864,7 @@ fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { #[test] fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { - use pheon::review::Scope; + use pheon::review::selection::Scope; let mut other_layer = square(); other_layer[8..10].copy_from_slice(&2_u16.to_be_bytes()); let a = layout(&[ @@ -875,14 +896,17 @@ fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { .unwrap() .selected(&scope, false) .unwrap(); - assert_eq!(Comparison::between(&old, &new).exit_code(), expected); - assert!(old.issues.is_empty()); + assert_eq!( + Comparison::between(&old, &new).outcome().exit_code(), + expected + ); + assert!(old.diagnostics.is_empty()); } } #[test] fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { - use pheon::review::Scope; + use pheon::review::selection::Scope; let mut unknown = square(); unknown.truncate(unknown.len() - 4); record(&mut unknown, 0x7f00, &[]); @@ -897,7 +921,7 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .is_empty() ); let scope = Scope { @@ -909,7 +933,7 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .is_empty() ); let bytes = layout(&[ @@ -925,15 +949,15 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .iter() - .any(|s| s.contains("cyclic")) + .any(|d| d.code == "hierarchy_cycle") ); } #[test] fn display_filters_never_change_equality_or_totals() { - use pheon::review::DisplayFilter; + use pheon::review::selection::DisplayFilter; let a = layout(&[("CHIP", vec![square()])]); let b = layout(&[("CHIP", vec![triangle()])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); @@ -946,7 +970,7 @@ fn display_filters_never_change_equality_or_totals() { ); assert!(text.contains("display-only")); assert!(text.contains("0 of 2 object changes shown")); - assert_eq!(diff.exit_code(), 1); + assert_eq!(diff.outcome().exit_code(), 1); } #[test] @@ -1057,11 +1081,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!(text.contains("renamed layout.gds"), "{text}"); assert!(text.contains("2 of 2 object changes shown"), "{text}"); git(&["commit", "-qm", "rename fixture"]); - fs::write( - directory.join(".gitattributes"), - "*.gds diff=pheon-text\n", - ) - .unwrap(); + fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); fs::write(&renamed, shared(square())).unwrap(); let result = git(&["diff", "--", "renamed layout.gds"]); assert!( @@ -1145,13 +1165,25 @@ fn unchanged_layer_names_explain_geometry_changes_once() { } #[test] -fn an_added_empty_document_is_still_a_presence_change() { +fn library_presence_changes_distinguish_empty_and_absent_revisions() { let bytes = layout(&[]); let doc = Document::parse(&bytes).unwrap(); - let diff = Comparison::between(&doc.empty_like(), &doc); - assert_eq!(diff.exit_code(), 1); - assert!(diff.text(0).contains("library presence:")); - assert!(!diff.text(0).contains("equal under")); + let absent = doc.empty_like(); + let added = Comparison::between(&absent, &doc); + assert_eq!(added.outcome().exit_code(), 1); + assert!( + added + .text(0) + .contains("library presence: [\"absent\"] -> [\"present\"]\n") + ); + assert!(!added.text(0).contains("equal under")); + let removed = Comparison::between(&doc, &absent); + assert_eq!(removed.outcome().exit_code(), 1); + assert!( + removed + .text(0) + .contains("library presence: [\"present\"] -> [\"absent\"]\n") + ); } #[test] @@ -1213,8 +1245,8 @@ fn large_polygons_and_billion_instance_arrays_stay_compact() { 32767 * 32767 ); assert_eq!(doc.cells["CHIP"].objects.len(), 1); - assert!(hierarchy(&doc, &doc, None, false, 8).len() < 200); - assert!(Comparison::between(&doc, &doc).is_equal()); + assert!(doc.hierarchy(false, 8).len() < 200); + assert!(Comparison::between(&doc, &doc).outcome().is_equal()); } #[test] @@ -1241,7 +1273,7 @@ fn deep_hierarchy_comparison_and_rendering_are_iterative() { let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); - let tree = hierarchy(&a, &b, Some(&diff), false, 5); + let tree = diff.hierarchy(false, 5); assert_eq!(diff.cells.len(), 1); assert!(tree.contains("children omitted")); assert!(tree.len() < 500); diff --git a/tests/review/reports.rs b/tests/review/reports.rs index f761a8c..bad49d9 100644 --- a/tests/review/reports.rs +++ b/tests/review/reports.rs @@ -1,7 +1,8 @@ use super::*; use pheon::review::{ - DisplayFilter, Scope, + Diagnostic, Outcome, Side, report::{ReportInputs, ReportOptions, failure_report}, + selection::{DisplayFilter, Scope}, }; use serde_json::{Value, json}; @@ -15,7 +16,7 @@ fn inputs() -> ReportInputs { fn report(old: &[u8], new: &[u8], options: &ReportOptions) -> Value { let a = Document::parse(old).unwrap(); let b = Document::parse(new).unwrap(); - Comparison::between(&a, &b).json_report(&a, &b, &inputs(), options) + Comparison::between(&a, &b).json_report(&inputs(), options) } fn unknown(geometry: Vec) -> Vec { @@ -26,12 +27,109 @@ fn unknown(geometry: Vec) -> Vec { layout(&[("CHIP", vec![object])]) } +#[test] +fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { + let comparison = { + let mut old_bytes = shared(square()); + let mut new_bytes = shared(triangle()); + let old = Document::parse(&old_bytes).unwrap(); + let new = Document::parse(&new_bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + drop(old); + drop(new); + old_bytes.fill(0); + new_bytes.fill(0); + comparison + }; + assert_eq!(comparison.outcome(), Outcome::Different); + assert!(comparison.old.present && comparison.new.present); + assert_eq!(comparison.old.units.as_ref().unwrap().len(), 16); + let mut actual = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + ..Default::default() + }, + ); + actual["producer"]["version"] = json!("test"); + let expected: Value = + serde_json::from_str(include_str!("../golden/report-geometry.json")).unwrap(); + assert_eq!(actual, expected); + assert_eq!( + comparison.hierarchy(false, 8), + include_str!("../golden/review-hierarchy.txt") + ); + assert_eq!( + comparison.hierarchy(true, 8), + include_str!("../golden/review-hierarchy.mmd") + ); + // Display choices affect evidence, not the captured outcome or full graph. + let limited = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + max_objects: Some(0), + ..Default::default() + }, + ); + assert_eq!(limited["hierarchy"], actual["hierarchy"]); + assert_eq!(limited["summary"], actual["summary"]); + assert_eq!(limited["exit_code"], 1); + assert!(comparison.text(0).contains("0 of 2 object changes shown")); +} + +#[test] +fn comparison_diagnostics_capture_both_sides_and_scope_mismatch_once() { + let comparison = { + let bytes = unknown(square()); + let old = Document::parse(&bytes) + .unwrap() + .selected( + &Scope { + cell: Some("CHIP".to_string()), + ..Default::default() + }, + false, + ) + .unwrap(); + let mut new = Document::parse(&bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + // Changes to source documents cannot alter an existing result. + new.scope = old.scope.clone(); + new.diagnostics.clear(); + comparison + }; + assert_eq!( + comparison.outcome(), + Outcome::Incomplete { different: false } + ); + assert_eq!(comparison.diagnostics.len(), 3); + assert_eq!(comparison.diagnostics[0].side, Some(Side::Old)); + assert_eq!(comparison.diagnostics[1].side, Some(Side::New)); + assert_eq!(comparison.diagnostics[2].code, "scope_mismatch"); + assert_eq!(comparison.diagnostics[2].side, None); + let report = comparison.json_report(&inputs(), &ReportOptions::default()); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["result"], "unknown"); + assert_eq!(report["exit_code"], 2); + assert_eq!(report["scope"]["cell"], "CHIP"); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 3); + assert_eq!(report["diagnostics"][0]["side"], "old"); + assert_eq!(report["diagnostics"][1]["side"], "new"); + assert_eq!(report["diagnostics"][2]["code"], "scope_mismatch"); + let text = comparison.text(0); + let scope = text.find("INCOMPLETE: comparison scopes differ").unwrap(); + let old = text.find("INCOMPLETE: old:").unwrap(); + let new = text.find("INCOMPLETE: new:").unwrap(); + assert!(scope < old && old < new); +} + #[test] fn reports_preserve_fixture_results_totals_and_replacement_links() { for pair in corpus() { let r = report(&pair.old, &pair.new, &ReportOptions::default()); assert_eq!(r["schema_version"], 1); - assert_eq!(r["comparison_profile"], "klamath-object-v2"); + assert_eq!(r["comparison_profile"], "pheon-structural-v1"); assert_eq!(r["producer"]["version"], env!("CARGO_PKG_VERSION")); assert_eq!(r["status"], "complete", "{}", pair.name); assert_eq!( @@ -110,12 +208,12 @@ fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { let bytes = if location == "CHIP" { layout(&[( "CHIP", - vec![square(), label((900, 7), "KLAMATH_LAYER_NAMES_V2")], + vec![square(), label((900, 7), "PHEON_LAYER_NAMES_V2")], )]) } else { layout(&[ ("CHIP", vec![square()]), - ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), ]) }; let source = Document::parse(&bytes).unwrap().diagnostics[0].offset; @@ -159,10 +257,10 @@ fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { assert_eq!(diagnostic.layer, Some((900, 7))); assert_eq!(diagnostic.offset, source); let diff = Comparison::between(&doc, &doc); - assert_eq!(diff.exit_code(), 2); - assert!(!diff.is_equal()); + assert_eq!(diff.outcome().exit_code(), 2); + assert!(!diff.outcome().is_equal()); assert!(diff.text(0).contains("INCOMPLETE")); - let json = diff.json_report(&doc, &doc, &inputs(), &ReportOptions::default()); + let json = diff.json_report(&inputs(), &ReportOptions::default()); assert_eq!(json["status"], "incomplete"); assert_eq!(json["result"], "unknown"); assert_eq!(json["exit_code"], 2); @@ -339,7 +437,7 @@ fn hierarchy_reports_shared_ancestors_and_symbolic_arrays() { .unwrap() .selected(&scope, false) .unwrap(); - let r = Comparison::between(&doc, &doc).json_report(&doc, &doc, &inputs(), &options); + let r = Comparison::between(&doc, &doc).json_report(&inputs(), &options); assert_eq!(r["scope"]["cell"], "CHIP"); assert_eq!( r["hierarchy"]["nodes"] @@ -369,6 +467,14 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { let r = report(&a, &b, &ReportOptions::default()); assert_eq!(r["cell_changes"][0]["before"], Value::Null); assert_eq!(r["cell_changes"][0]["after"]["attributes"], json!([])); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); + let removed = report(&b, &a, &ReportOptions::default()); + assert_eq!( + removed["cell_changes"][0]["before"]["attributes"], + json!([]) + ); + assert_eq!(removed["cell_changes"][0]["after"], Value::Null); + assert_eq!(removed["cell_changes"][0]["attributes_changed"], true); let mut b = b; let end = Document::parse(&b).unwrap().cells["empty"].end_offset; let mut attribute = Vec::new(); @@ -379,15 +485,11 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { r["cell_changes"][0]["after"]["attributes"][0]["value"], json!([1]) ); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); assert_eq!(r["inputs"]["old"]["units"]["encoding"], "real8"); let doc = Document::parse(&a).unwrap(); let absent = doc.empty_like(); - let r = Comparison::between(&absent, &doc).json_report( - &absent, - &doc, - &inputs(), - &ReportOptions::default(), - ); + let r = Comparison::between(&absent, &doc).json_report(&inputs(), &ReportOptions::default()); assert_eq!(r["inputs"]["old"]["present"], false); assert_eq!(r["inputs"]["old"]["units"], Value::Null); assert_eq!( @@ -401,11 +503,8 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { .find(|r| r.tag == RTAG_UNITS) .unwrap() .offset; - b[units + 12..units + 20].copy_from_slice( - &pheon::basic::encode_real8(2e-9) - .unwrap() - .to_be_bytes(), - ); + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); let r = report(&a, &b, &ReportOptions::default()); assert_eq!(r["library_changes"][0]["tag"], RTAG_UNITS); assert_ne!(r["inputs"]["old"]["units"], r["inputs"]["new"]["units"]); @@ -530,9 +629,7 @@ fn machine_reports_match_versioned_goldens() { &inputs(), &Scope::default(), &ReportOptions::default(), - "invalid_input", - "truncated record", - Some("new"), + &Diagnostic::new("invalid_input", "truncated record", Some(Side::New)), ), ), ]; @@ -580,6 +677,24 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { ); report }; + let check_text_failure = |report: &Value, extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(extra) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + format!( + "pheon: {}\n", + report["diagnostics"][0]["message"].as_str().unwrap() + ) + ); + }; let full = command(&["--hierarchy", "graph"]); assert_eq!(full["result"], "different"); assert!(full["hierarchy"].is_object()); @@ -597,14 +712,35 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { assert_eq!(failure["status"], "failed"); assert_eq!(failure["result"], "unknown"); assert_eq!(failure["summary"], Value::Null); + if extra.first() == Some(&"--top") { + assert_eq!(failure["diagnostics"][0]["code"], "invalid_scope"); + assert_eq!(failure["diagnostics"][0]["side"], Value::Null); + check_text_failure(&failure, extra); + } } fs::write(&b, b"invalid").unwrap(); let invalid = command(&[]); assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); assert_eq!(invalid["diagnostics"][0]["side"], "new"); assert_eq!(invalid["inputs"]["new"]["path"], b.to_str().unwrap()); + check_text_failure(&invalid, &[]); + fs::write(&a, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "old"); + check_text_failure(&invalid, &[]); + fs::write(&a, shared(square())).unwrap(); fs::remove_file(&b).unwrap(); - assert_eq!(command(&[])["diagnostics"][0]["code"], "io_error"); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "new"); + check_text_failure(&missing, &[]); + fs::remove_file(&a).unwrap(); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "old"); + check_text_failure(&missing, &[]); + fs::write(&a, shared(square())).unwrap(); fs::write(&b, fs::read(&a).unwrap()).unwrap(); assert_eq!(command(&[])["result"], "equal"); fs::write(&b, unknown(square())).unwrap(); @@ -613,7 +749,7 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { &b, layout(&[ ("CHIP", vec![square()]), - ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), ]), ) .unwrap(); diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs index 8db1d81..aaef2ab 100644 --- a/tests/selective_reads.rs +++ b/tests/selective_reads.rs @@ -203,7 +203,7 @@ fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { record(&mut bytes, records::RTAG_ENDLIB, &[]); let doc = pheon::review::Document::parse(&bytes).unwrap(); assert_eq!( - pheon::review::Comparison::between(&doc, &doc).exit_code(), + pheon::review::Comparison::between(&doc, &doc).outcome().exit_code(), 0 ); } From 31afef0529259508c5a15325f99d10b9b42a10ee Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 16 Nov 2026 21:28:54 -0800 Subject: [PATCH 20/41] Reject invalid record sizes and return malformed-input errors Validate payload sizes before record/header writes and make header packing return io::Result when the total size overflows the format limit. Return located parse errors for invalid UTF-8 and timestamp year overflow. Validate cell timestamps consistently in full reads, selected reads, and hierarchy scans while retaining scan behavior for skipped text geometry. Add regression coverage for preserved writer outputs, padded metadata sizes, malformed strings, timestamp boundaries, bounded errors, and Python bindings. Document the fallible header packing API. Validation: all 97 Rust tests pass in debug and release; all 16 Python tests pass against the rebuilt native library. --- docs/native-api.md | 5 ++ src/basic.rs | 21 +++-- src/elements.rs | 8 +- src/library.rs | 24 +++--- src/record.rs | 16 ++-- tests/parser_errors.rs | 155 +++++++++++++++++++++++++++++++++++++ tests/python/test_arrow.py | 25 ++++++ tests/record_validation.rs | 25 +++++- tests/writing.rs | 56 +++++++++++++- 9 files changed, 307 insertions(+), 28 deletions(-) create mode 100644 tests/parser_errors.rs diff --git a/docs/native-api.md b/docs/native-api.md index ad64a05..5e9f9ab 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -36,6 +36,11 @@ writing. There is no high-level writer for the Arrow layout. REAL8 writing is exact for representable in-range `f64` values: magnitudes below `2^-260` underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected. +Record writes check payload sizes before emitting bytes. Header packing returns +an `io::Result`, including an error when the total size exceeds the format limit. +Arrow readers return parse errors for invalid UTF-8 strings and timestamp years +that exceed the Int16 range after conversion from the 1900 epoch. + ## C API Build the shared library with `cargo build --release --locked`. The diff --git a/src/basic.rs b/src/basic.rs index 1189b8e..4bda079 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -79,12 +79,23 @@ pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { let mut buf = [0_i16; 6]; - let mut input = input; - for bb in &mut buf { - (input, *bb) = parse_int2(input)?; + let (mut remaining, year) = parse_int2(input)?; + let Some(year) = year.checked_add(1900) else { + return fail( + input, + "Timestamp year exceeds the Int16 range after adding 1900".to_string(), + ); + }; + buf[0] = year; + for bb in &mut buf[1..] { + (remaining, *bb) = parse_int2(remaining)?; } - buf[0] += 1900; // Year is from 1900 - Ok((input, buf)) + Ok((remaining, buf)) +} + +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + String::from_utf8(bytes) + .map_err(|error| ErrType::Failed(format!("Invalid UTF-8 string: {}", error.utf8_error()))) } diff --git a/src/elements.rs b/src/elements.rs index 02a8112..32fd9c8 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -12,7 +12,7 @@ use crate::records::{ use crate::records; use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; +use crate::basic::{IResult, decode_utf8, fail, take_bytes}; use std::string::String; use std::collections::HashMap; @@ -784,7 +784,7 @@ fn read_text<'a>( xy_builder.append_value(xy); let (input, string_bytes) = STRING::read(input)?; - let string = String::from_utf8(string_bytes).unwrap(); + let string = decode_utf8(string_bytes).map_err(|error| (input, error))?; let string_builder = text_builder.field_builder::(10).unwrap(); string_builder.append_value(string); @@ -856,7 +856,7 @@ fn read_ref<'a>( let mut colrow = None; let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; - let struct_name = String::from_utf8(struct_name_bytes).unwrap(); + let struct_name = decode_utf8(struct_name_bytes).map_err(|error| (input, error))?; let next_id = names.len(); let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); @@ -986,7 +986,7 @@ fn walk_properties<'a>( on_property(PropertyRecord { key, - value: String::from_utf8(value_bytes).unwrap(), + value: decode_utf8(value_bytes).map_err(|error| (input, error))?, }); } (input, header) = RecordHeader::read(input)?; diff --git a/src/library.rs b/src/library.rs index e455f8b..d4db8f2 100644 --- a/src/library.rs +++ b/src/library.rs @@ -7,7 +7,7 @@ use std::io::Write; use crate::record::{RecordHeader, Record}; use crate::records; use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::basic::{ErrType, IResult, OResult, decode_utf8, take_bytes, fail}; use std::string::String; use std::collections::HashMap; @@ -324,13 +324,14 @@ fn make_scan_builder() -> StructBuilder { fn append_file_header( lib_builder: &mut StructBuilder, header: &FileHeader, - ) { + ) -> Result<(), ErrType> { + let name = decode_utf8(header.name.clone())?; let dbu_builder = lib_builder.field_builder::(0).unwrap(); dbu_builder.append_value(header.meters_per_db_unit); let uu_builder = lib_builder.field_builder::(1).unwrap(); uu_builder.append_value(header.user_units_per_db_unit); let libname_builder = lib_builder.field_builder::(2).unwrap(); - libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); + libname_builder.append_value(name); let mtl_builder = lib_builder.field_builder::(3).unwrap(); let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); mt_builder.append_values(&header.mod_time, &[true; 6]); @@ -339,6 +340,7 @@ fn append_file_header( let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); at_builder.append_values(&header.acc_time, &[true; 6]); atl_builder.append(true); + Ok(()) } @@ -441,14 +443,16 @@ where let (mut input, mut header) = RecordHeader::read(input)?; while header.tag != records::RTAG_ENDLIB { let record_start = base_offset + input_size - input.len() - 4; - (input, _) = take_bytes(input, header.data_size)?; if header.tag == records::RTAG_BGNSTR { + (input, _) = records::BGNSTR::read_data(input, header.data_size)?; let name_bytes; (input, name_bytes) = records::STRNAME::read(input)?; - let name = String::from_utf8(name_bytes).unwrap(); + let name = decode_utf8(name_bytes).map_err(|error| (input, error))?; let id = intern_name(names, name); let (next_input, ()) = on_struct(input, record_start, id, names)?; input = next_input; + } else { + (input, _) = take_bytes(input, header.data_size)?; } (input, header) = RecordHeader::read(input)?; } @@ -461,7 +465,7 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_library_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -497,7 +501,7 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_scan_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -541,8 +545,8 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { /// as its remainder; it does not advance a sequential parsing cursor. pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { let mut lib_builder = make_library_builder(); - let (_, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + let (header_end, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (header_end, error))?; let mut names = HashMap::::new(); let mut layers = HashMap::::new(); @@ -558,7 +562,7 @@ pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IRes let struct_input = &input[start..end]; let (struct_input, _) = records::BGNSTR::read(struct_input)?; let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; - let name = String::from_utf8(name_bytes).unwrap(); + let name = decode_utf8(name_bytes).map_err(|error| (struct_input, error))?; let file_offset = end - struct_input.len(); let id = intern_name(&mut names, name); diff --git a/src/record.rs b/src/record.rs index 25d358a..6de51d1 100644 --- a/src/record.rs +++ b/src/record.rs @@ -27,15 +27,16 @@ impl RecordHeader { Ok((input, RecordHeader{tag, data_size})) } - pub fn pack_into(&self) -> [u8; 4] { - let size = self.data_size.checked_add(4).expect("Record too big!"); + pub fn pack_into(&self) -> io::Result<[u8; 4]> { + let size = self.data_size.checked_add(4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; let [size_hi, size_lo] = size.to_be_bytes(); let [tag_hi, tag_lo] = self.tag.to_be_bytes(); - [size_hi, size_lo, tag_hi, tag_lo] + Ok([size_hi, size_lo, tag_hi, tag_lo]) } pub fn write(&self, ww: &mut W) -> OResult { - let bytes = self.pack_into(); + let bytes = self.pack_into()?; ww.write_all(&bytes)?; Ok(bytes.len()) } @@ -69,6 +70,8 @@ pub trait Record { } fn write_header(ww: &mut W, data_size: u16) -> OResult { + Self::check_size(data_size) + .map_err(|message| io::Error::new(io::ErrorKind::InvalidInput, message))?; RecordHeader{tag: Self::tag(), data_size}.write(ww) } @@ -126,9 +129,8 @@ pub trait Record { fn write(ww: &mut W, data: &RData::InData) -> OResult { let packed_data = RData::pack(data); let data_bytes = packed_data.as_ref(); - let len = u16::try_from(data_bytes.len()).ok() - .filter(|len| *len <= u16::MAX - 4) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + let len = u16::try_from(data_bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; Self::write_header(ww, len)?; ww.write_all(data_bytes)?; Ok(usize::from(len) + 4) diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs new file mode 100644 index 0000000..eca1116 --- /dev/null +++ b/tests/parser_errors.rs @@ -0,0 +1,155 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{ + basic::ErrType, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_bytes, read_cells_bytes, + record::RecordHeader, + records, scan_bytes, +}; +use std::ffi::CStr; + +fn fixture() -> (Vec, (u64, u64)) { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_SREF, &[]); + common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_TEXT, &[]); + common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]); + common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_STRING, b"label\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let end = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + (bytes, (start, end)) +} + +fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range { + let mut offset = 0; + loop { + let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap(); + let start = offset + 4; + let end = start + usize::from(header.data_size); + if header.tag == tag { + return start..end; + } + offset = end; + } +} + +fn check_ffi_error( + expected: &str, + call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32, +) { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(call(&mut array, &mut schema), -1); + let pointer = last_error_message(); + assert!(!pointer.is_null()); + let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); + assert!(error.contains(expected), "{error}"); + assert!( + error.len() < 256, + "error should not dump the input: {error}" + ); + assert!(array.is_released()); +} + +fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) { + let ranges = [range]; + let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)]; + if scan { + results.push(scan_library(bytes)); + } else { + // Hierarchy scans deliberately skip text geometry and its string payloads. + assert!(scan_library(bytes).is_ok()); + } + for result in results { + let (remaining, error) = result.unwrap_err(); + assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr()); + assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message))); + } + let expected = format!("at byte {offset}: {message}"); + check_ffi_error(&expected, |a, s| unsafe { + read_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + let offsets = [range.0, range.1]; + check_ffi_error(&expected, |a, s| unsafe { + read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s) + }); + if scan { + check_ffi_error(&expected, |a, s| unsafe { + scan_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + } +} + +#[test] +fn invalid_utf8_returns_located_parse_errors_in_each_reader() { + let (bytes, range) = fixture(); + assert!(read_library(&bytes).is_ok()); + assert!(read_selected_library(&bytes, &[range]).is_ok()); + assert!(scan_library(&bytes).is_ok()); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + records::RTAG_STRING, + records::RTAG_PROPVALUE, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0xff; + // UTF-8 is checked at the current parser cursor, after the record/header. + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + check_read_errors( + &malformed, + range, + offset, + "Invalid UTF-8 string:", + tag != records::RTAG_STRING, + ); + } +} + +#[test] +fn large_invalid_utf8_strings_do_not_dump_their_contents() { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, records::RTAG_PROPVALUE); + let mut replacement = Vec::new(); + let mut value = vec![b'a'; 60_000]; + value[30_000] = 0xff; + common::record(&mut replacement, records::RTAG_PROPVALUE, &value); + let added = replacement.len() - (payload.end - payload.start + 4); + bytes.splice(payload.start - 4..payload.end, replacement); + let range = (range.0, range.1 + added as u64); + let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end; + check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true); +} + +#[test] +fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() { + for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] { + for timestamp in [0, 1] { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, tag); + let offset = payload.start + timestamp * 12; + bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes()); + check_read_errors(&bytes, range, offset, "Timestamp year", true); + } + } +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py index ed962e2..64db3f2 100644 --- a/tests/python/test_arrow.py +++ b/tests/python/test_arrow.py @@ -85,6 +85,31 @@ class ArrowTests(unittest.TestCase): with self.assertRaises(TypeError): arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + def test_malformed_strings_and_years_raise_located_parse_errors(self): + data = layout() + cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0] + start = cell["struct_start_offset"] + ranges = [[start, cell["struct_end_offset"]]] + cases = [ + (38, b"\xff", "Invalid UTF-8"), + (start + 32, b"\xff", "Invalid UTF-8"), + ] + cases += [ + (offset, b"\x7f\xff", "Timestamp year") + for offset in (10, 22, start + 4, start + 16) + ] + for offset, value, message in cases: + malformed = bytearray(data) + malformed[offset:offset + len(value)] = value + for read in ( + arrow.read_bytes, + arrow.scan_bytes, + lambda payload: arrow.read_cells_bytes(payload, ranges), + ): + with self.subTest(offset=offset, reader=read): + with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"): + read(malformed) + def test_import_failure_releases_both_native_outputs(self): releases = [] diff --git a/tests/record_validation.rs b/tests/record_validation.rs index d1e4fa7..5fd55d3 100644 --- a/tests/record_validation.rs +++ b/tests/record_validation.rs @@ -1,7 +1,7 @@ mod common; use pheon::{ - basic::ErrType, + basic::{ErrType, parse_datetime}, record::{ ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, Record, RecordData, @@ -38,6 +38,29 @@ fn fixed_record_sizes_are_checked_by_every_read_entry_point() { assert!(records::COLROW::read_data(&payload, 2).is_err()); } +#[test] +fn timestamp_year_conversion_checks_the_int16_boundaries() { + for stored_year in [i16::MIN, -1900, 0, i16::MAX - 1900] { + let mut payload = [0; 12]; + payload[..2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, date) = parse_datetime(&payload).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(i32::from(date[0]), i32::from(stored_year) + 1900); + } + for stored_year in [i16::MAX - 1899, i16::MAX] { + for timestamp in [0, 1] { + let mut payload = [0; 24]; + let offset = timestamp * 12; + payload[offset..offset + 2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, error) = DateTimePair::read(&payload, 24).unwrap_err(); + assert_eq!(remaining.as_ptr(), payload[offset..].as_ptr()); + assert!( + matches!(error, ErrType::Failed(message) if message.contains("Timestamp year")) + ); + } + } +} + #[test] fn metadata_size_constraints_apply_to_every_read_entry_point() { macro_rules! check { diff --git a/tests/writing.rs b/tests/writing.rs index caf8816..dc07994 100644 --- a/tests/writing.rs +++ b/tests/writing.rs @@ -19,7 +19,7 @@ fn record_headers_include_their_own_size() { data_size: u16::MAX - 4, tag: 0x1003, }; - assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]); + assert_eq!(maximum.pack_into().unwrap(), [0xff, 0xff, 0x10, 0x03]); } #[test] @@ -135,4 +135,58 @@ fn oversized_records_fail_before_writing() { let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); assert!(bytes.is_empty()); + + // Header packing and direct header writes reject every overflowing size too. + for data_size in u16::MAX - 3..=u16::MAX { + let header = RecordHeader { + tag: records::RTAG_XY, + data_size, + }; + assert_eq!( + header.pack_into().unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + header.write(&mut bytes).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(bytes.is_empty()); + } +} + +#[test] +fn invalid_record_sizes_preserve_the_output() { + let mut bytes = vec![0xa5]; + for count in [0, 1, 3] { + let error = records::COLROW::write(&mut bytes, &vec![2; count]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } + let error = records::COLROW::write_header(&mut bytes, 2).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + + macro_rules! check_metadata { + ($record:ident) => { + for count in [43, 44, 45] { + let payload = vec![b'x'; count]; + let mut bytes = vec![0xa5]; + let result = records::$record::write(&mut bytes, &payload); + if count == 45 { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } else { + // Validation uses the padded record size, not the input length. + assert_eq!(result.unwrap(), 48); + assert_eq!(records::$record::read(&bytes[1..]).unwrap().1, payload); + } + } + let error = records::$record::write_header(&mut bytes, 46).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + }; + } + check_metadata!(REFLIBS); + check_metadata!(FONTS); + check_metadata!(ATTRTABLE); } From 4ea14085f7a44453ed51bb754b7edc50721d6283 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 15:07:40 -0800 Subject: [PATCH 21/41] Clarify Pheon installation and first-use examples --- README.md | 2 +- docs/arrow-schema.md | 2 +- docs/cli.md | 14 +++++++++----- docs/python.md | 41 ++++++++++++++++++++--------------------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 55285ab..470c168 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Run Python outside the source checkout to use the installed package: from pheon import arrow layout = arrow.read_path("chip.gds") -print(layout.field("cell_names")[0].as_py()) +print(layout.field("lib_name")[0].as_py()) ``` Source installation needs Rust; prebuilt Python wheels bundle the native library. diff --git a/docs/arrow-schema.md b/docs/arrow-schema.md index 97ffecd..a0abd94 100644 --- a/docs/arrow-schema.md +++ b/docs/arrow-schema.md @@ -21,7 +21,7 @@ version and the separate C API version. | `lib_name` | Utf8 | Library name. | | `mod_time`, `acc_time` | FixedSizeList(Int16, 6) | Modification/access time: year, month, day, hour, minute, second. | | `cells` | List(Cell) | Cell records. | -| `cell_names` | List(Utf8) | Names indexed by cell ID. | +| `cell_names` | List(Utf8) | Names indexed by cell ID, including targets without cell definitions. | | `layers` | List(UInt32) | Packed layer/type pairs indexed by layer ID. | ### Cell diff --git a/docs/cli.md b/docs/cli.md index 843bd68..e01228d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,11 +1,14 @@ # Command-line tools Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. -The CLI runs independently of Python. +Use uncompressed `.gds` files with the CLI. It runs independently of Python. ## Install -From a source checkout, with Rust 1.88 or newer: +If you downloaded a CLI archive, extract it and add the directory containing +`pheon` (`pheon.exe` on Windows) to your `PATH`. Run `pheon --help` to check it. + +To install from a source checkout, use Rust 1.88 or newer: ```sh cargo install --path . --locked --bin pheon @@ -37,9 +40,10 @@ pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 pheon diff before.gds after.gds --format json > report.json ``` -Reports identify changed objects and their fields, plus affected ancestors in -the hierarchy. Objects that can be paired unambiguously receive a replacement -explanation; ambiguous changes remain removals and additions. +Text reports group changes by cell. A `-` line shows an object from the old file +and a `+` line shows one from the new file. A `~` line explains a likely +replacement when Pheon can pair the two uniquely. Request a hierarchy view to +see which parent cells are affected. | Exit code | Meaning | | --- | --- | diff --git a/docs/python.md b/docs/python.md index 7f123ca..fa48cf4 100644 --- a/docs/python.md +++ b/docs/python.md @@ -6,16 +6,16 @@ Python 3.11 or newer is required. ## Install -To install a published package with Arrow support: +To install a PyPI release with Arrow support: ```sh python -m pip install 'pheon[arrow]' ``` -From a source checkout, use `python -m pip install '.[arrow]'`. Source builds -require Rust 1.88 or newer. Prebuilt wheels bundle the native library and need -no Rust installation. Use `pheon` without the `arrow` extra if you only need -integer packing; importing it does not import PyArrow. +For a local wheel, pass its file path followed by `[arrow]` in the same quotes. +Prebuilt wheels bundle the native library. From a source checkout, use +`python -m pip install '.[arrow]'` with Rust 1.88 or newer. The base `pheon` +package is enough for integer packing. Run Python outside the source checkout to use the installed package. @@ -25,19 +25,20 @@ Run Python outside the source checkout to use the installed package. from pheon import arrow layout = arrow.read_path("chip.gds") -print(layout.field("lib_name")[0].as_py()) -print(layout.field("cell_names")[0].as_py()) +names = layout.field("cell_names")[0].as_py() cells = layout.field("cells")[0].values +print([names[cell_id] for cell_id in cells.field("id").to_pylist()]) ``` `read_path` accepts strings and path-like objects, expands `~`, and recognizes gzip by its magic bytes. `read_bytes` accepts an uncompressed, contiguous byte buffer, such as `bytes`, `bytearray`, or a compatible memoryview. -Every reader returns a one-row `pyarrow.StructArray`. Geometry is grouped by -cell and element kind; cell and layer IDs resolve through tables in the same -result. See the [Arrow reference](arrow-schema.md) for fields and encodings. -The returned arrays own their storage independently of the input buffer. +Every reader returns a one-row `pyarrow.StructArray`. The example uses each +cell's ID to look up its name. The name table can also contain reference targets +without cell definitions. Geometry is grouped by cell and element kind. The +returned arrays own their storage independently of the input buffer. +See the [Arrow reference](arrow-schema.md) for fields and encodings. ## Scan, then read selected cells @@ -79,19 +80,17 @@ Python sequences, or scalars and preserve the caller's data, including for noncontiguous or differently ordered arrays. Pheon currently has no high-level Python layout writer. -## Errors and native library selection +## Handle read errors Native parse failures raise `pheon.PheonError`, a subclass of `ValueError`. Invalid buffer or range arguments raise `ValueError` or `TypeError`; file access errors use Python's normal I/O exceptions. -Python loads its bundled native library. Set `PHEON_LIBRARY` to an explicit -library path before importing Pheon to override it during development. The -loader checks C API and Arrow schema compatibility. It does not discover a -Cargo-installed executable or search PATH for a library. +## Use from Klamath or Masque -Klamath's `native` extra installs Pheon's packing helpers; Masque's `arrow` extra -installs Pheon with Arrow support. Install these in the same Python environment -as the companion package. A standalone CLI installation does not supply the -Python bindings. -For direct integration without Python, see the [native API guide](native-api.md). +Klamath's `native` extra installs Pheon's packing helpers. Masque's `arrow` extra +installs Pheon with Arrow support. Use the same Python environment as the +companion package. + +See [development](development.md) to test a locally built native library from +Python, or the [native API guide](native-api.md) to call the library directly. From 58adf0cc9937d93914c54f8c48e0958d3acd80b4 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 15:07:51 -0800 Subject: [PATCH 22/41] Build Windows wheels locally and document release builds --- .gitignore | 2 + contrib/wheels/manylinux-image.txt | 1 + docs/releases.md | 122 +++++++++++++++++++++-------- setup.py | 20 ++++- 4 files changed, 110 insertions(+), 35 deletions(-) create mode 100644 contrib/wheels/manylinux-image.txt diff --git a/.gitignore b/.gitignore index e67c112..c2a3759 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ __pycache__/ /build/ /dist/ +/.cache/ +/.venv/ /*.egg-info/ diff --git a/contrib/wheels/manylinux-image.txt b/contrib/wheels/manylinux-image.txt new file mode 100644 index 0000000..a8468dd --- /dev/null +++ b/contrib/wheels/manylinux-image.txt @@ -0,0 +1 @@ +quay.io/pypa/manylinux_2_28_x86_64@sha256:e0adf18521ad35192df733a6c62d35df7801957d24e0470238aacecfab2188a3 diff --git a/docs/releases.md b/docs/releases.md index 76e7c7b..4384205 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,48 +1,108 @@ # Building releases -This guide is for maintainers producing source archives, Python wheels, and -standalone CLI artifacts. See [development](development.md) for local checks. +Build packages locally for distribution. For installing and using Pheon, start +with the [CLI](cli.md) or [Python](python.md) guide. -## Build from the source archive +## Start a release + +Use Python 3.11+ and Rust 1.88+. From the repository root, create a build +environment and put the source archive in an empty `dist/release/` directory. ```sh -python -m pip install build -python -m build +python3 -m venv .venv +. .venv/bin/activate +python -m pip install build twine +python -m build --sdist --outdir dist/release ``` -The default invocation builds the wheel from the source archive. Both use the -Cargo version, license, and locked dependencies. Source builds require Rust 1.88 -or newer; the checkout selects stable. Use `RUSTUP_TOOLCHAIN=1.88.0` to check the -minimum compiler explicitly. +Use `python -m build --wheel` to build a wheel for your own machine. For a +portable Linux wheel, use the container below. See [development](development.md) +for compiler checks. -Python wheels contain the package and shared library. Cargo installation -provides the CLI. Neither installation discovers or depends on the other. +Python wheels bundle the native library. Install the CLI separately with Cargo. + +## Build Linux x86-64 wheels + +On x86-64 Linux, use Podman and a Rust toolchain installed through `rustup`. +Download the [pinned manylinux image](../contrib/wheels/manylinux-image.txt) +once and save it for future releases. + +```sh +pheon_wheel_cache="$PWD/.cache/wheels" +pheon_image=$(cat contrib/wheels/manylinux-image.txt) +pheon_cached_image=localhost/pheon-manylinux:2.28-2026.10.26-1 +mkdir -p "$pheon_wheel_cache" +podman pull --platform linux/amd64 "$pheon_image" +podman tag "$pheon_image" "$pheon_cached_image" +podman save --format oci-archive \ + --output "$pheon_wheel_cache/manylinux_2_28_x86_64.oci.tar" \ + "$pheon_cached_image" +``` + +If the image is missing from Podman, restore it with +`podman load --input .cache/wheels/manylinux_2_28_x86_64.oci.tar`. +Build from the source archive using the local Rust toolchain and Cargo cache. + +```sh +python -m pip install 'cibuildwheel==4.3.5' +cargo fetch --locked --target x86_64-unknown-linux-gnu +rust_toolchain=$(rustc --print sysroot) +cargo_registry="${CARGO_HOME:-$HOME/.cargo}/registry" +export CIBW_CONTAINER_ENGINE="podman; disable_host_mount: true; create_args: \ + --volume '$PWD/.cache/wheels:/pheon-cache' \ + --volume '$rust_toolchain:/rust-toolchain:ro' \ + --volume '$cargo_registry:/pheon-cache/cargo/registry:ro'" +export CIBW_MANYLINUX_X86_64_IMAGE=localhost/pheon-manylinux:2.28-2026.10.26-1 +export CIBW_ENVIRONMENT_LINUX='PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo CARGO_NET_OFFLINE=true PIP_CACHE_DIR=/pheon-cache/pip' +export CIBW_BUILD='cp31{1,2,3,4}-manylinux_x86_64' +export CIBW_TEST_EXTRAS=arrow +export CIBW_TEST_COMMAND='python -I {project}/tools/check_installed.py' +unset PHEON_LIBRARY +python -m cibuildwheel --platform linux --archs x86_64 \ + --output-dir dist/release dist/release/pheon-*.tar.gz +``` + +cibuildwheel repairs the wheel and tests the same file on Python 3.11–3.14. +The container downloads any Python dependencies that it needs. + +## Build Windows x86-64 packages from Linux + +Install MinGW-w64 with `x86_64-w64-mingw32-gcc` on `PATH`. From the project +environment, build the Windows wheel and the standalone CLI: + +```sh +rustup target add x86_64-pc-windows-gnu +PHEON_BUILD_TARGET=x86_64-pc-windows-gnu python -m build --wheel --outdir dist/release +cargo build --locked --release --target x86_64-pc-windows-gnu --bin pheon +``` + +The `py3-none-win_amd64` wheel lands in `dist/release/`. The executable is +`target/x86_64-pc-windows-gnu/release/pheon.exe`. Test the wheel and CLI on Windows +before publishing. ## Validate artifacts -The [package workflow](https://mpxd.net/code/jan/pheon/src/branch/master/.github/workflows/packages.yml) builds one source archive -and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64, -and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag. -Linux releases need an appropriate manylinux tag; unrepaired local `linux_*` -wheels are development artifacts. - -The workflow tests each repaired wheel on Python 3.11–3.14 without rebuilding it. -Tests run in isolated mode against the installed package and bundled library. -To repeat the check after installing a wheel and PyArrow: +Check the Python package metadata and verify that the packaged crate builds. ```sh -python -I /path/to/pheon/tools/check_installed.py +python -m twine check --strict dist/release/*.tar.gz dist/release/*.whl +cargo publish --dry-run --locked ``` -Unset `PHEON_LIBRARY` for artifact checks. Native checks cover minimum and stable -Rust in debug and release modes. Standalone CLI artifacts contain the executable, -license, README, documentation, and Git templates; each targets one OS and -architecture. +Wheel filenames use `py3-none` plus a platform tag, such as +`manylinux_2_28_x86_64` or `win_amd64`. Use repaired `manylinux_*` wheels for +Linux releases. -## Publish separately +To test a wheel manually, install it with its `arrow` extra in a test +environment on the target OS. Clear `PHEON_LIBRARY` so the checks load the +bundled library. Replace the example path with the wheel you built. -Workflows upload artifacts; they do not publish to PyPI, crates.io, or a hosted -release. Review the artifacts and test results before publishing them. -Coordinate companion package dependencies when changing Python interfaces. -C API, Arrow layout, JSON schema, and comparison-profile versions have separate -contracts; see the [native API](native-api.md) and [report reference](diff-reports.md). +```sh +python -m pip install "path/to/pheon-wheel.whl[arrow]" +python -I tools/check_installed.py +``` + +Build standalone CLI artifacts for each OS and architecture. Include the +executable, license, README, documentation, and Git templates. + +Publish the validated Python packages to PyPI and the Rust crate to crates.io. diff --git a/setup.py b/setup.py index 17d080a..7a22f81 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ """Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" import json +import os from pathlib import Path import shutil import subprocess @@ -13,6 +14,11 @@ from wheel.bdist_wheel import bdist_wheel ROOT = Path(__file__).resolve().parent VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] +BUILD_TARGET = os.environ.get("PHEON_BUILD_TARGET") +WINDOWS_TARGET = "x86_64-pc-windows-gnu" + +if BUILD_TARGET is not None and BUILD_TARGET != WINDOWS_TARGET: + raise ValueError(f"PHEON_BUILD_TARGET must be {WINDOWS_TARGET}, got {BUILD_TARGET!r}") if not (ROOT / "Cargo.lock").is_file(): @@ -22,8 +28,11 @@ if not (ROOT / "Cargo.lock").is_file(): class CargoBuild(build_py): def run(self): super().run() + command = ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"] + if BUILD_TARGET: + command.extend(["--target", BUILD_TARGET]) result = subprocess.run( - ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], + command, cwd=ROOT, capture_output=True, text=True, ) sys.stderr.write(result.stderr) @@ -43,8 +52,8 @@ class CargoBuild(build_py): if Path(filename).suffix in (".so", ".dylib", ".dll"): library = Path(filename) result.check_returncode() - if library is None: - raise RuntimeError("Cargo did not produce the pheon shared library") + if library is None or (BUILD_TARGET and library.name != "pheon.dll"): + raise RuntimeError("Cargo did not produce the expected Pheon shared library") shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) @@ -55,7 +64,9 @@ class NativeDistribution(Distribution): class NativeWheel(bdist_wheel): def get_tag(self): - # CFFI ABI mode needs the host platform, but no CPython extension ABI. + # CFFI ABI mode needs a platform tag and no CPython extension ABI. + if BUILD_TARGET == WINDOWS_TARGET: + return "py3", "none", "win_amd64" _, _, platform = super().get_tag() return "py3", "none", platform @@ -64,4 +75,5 @@ setup( version=VERSION, distclass=NativeDistribution, cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel}, + options={"build": {"build_base": f"build/{BUILD_TARGET}"}} if BUILD_TARGET else {}, ) From 6e4b37104669ee25e12d7e3ab5e0ba6c2f937df8 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 16:14:53 -0800 Subject: [PATCH 23/41] Read gzip-compressed GDSII directly in the CLI Detect gzip headers on every CLI input and decode all members before parsing. Keep report offsets relative to the decompressed stream and report corrupt archives as invalid input. Recognize OASIS headers and explain how to provide GDSII data. Cover mixed compressed and plain comparisons, corrupt trailers, and layer-name output from compressed input. --- Cargo.lock | 42 ++++++++++++++++++++ Cargo.toml | 1 + docs/cli.md | 3 +- docs/diff-reports.md | 3 +- docs/layer-names.md | 2 + src/bin/pheon.rs | 39 +++++++++++++++--- src/review.rs | 10 +++++ tests/review.rs | 94 +++++++++++++++++++++++++++++++++++++++++++- 8 files changed, 185 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a525b0..4220cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.11" @@ -156,12 +162,31 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -254,6 +279,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num" version = "0.4.3" @@ -340,6 +375,7 @@ version = "0.3.0" dependencies = [ "arrow-array", "arrow-schema", + "flate2", "serde_json", ] @@ -411,6 +447,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "syn" version = "2.0.100" diff --git a/Cargo.toml b/Cargo.toml index 49e7ad5..c40323a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] arrow-array = {version = "^54", features = ["ffi"]} arrow-schema = "^54" +flate2 = "1" serde_json = "1" diff --git a/docs/cli.md b/docs/cli.md index e01228d..95ea2cf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,8 @@ # Command-line tools Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. -Use uncompressed `.gds` files with the CLI. It runs independently of Python. +The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets +refer to the decompressed layout. The CLI runs independently of Python. ## Install diff --git a/docs/diff-reports.md b/docs/diff-reports.md index 3408ea5..ef870a5 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -86,7 +86,8 @@ hex preserves their bytes. Evidence follows the comparison profile: boundary/box rings are canonicalized, string padding is removed, and fields/properties are sorted. REAL8 encodings and -explicit defaults remain significant. Source offsets locate original records. +explicit defaults remain significant. For gzip inputs, source offsets locate +records in the decompressed layout. Replacements link report-local removed/added IDs with `match_rule`, `field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, diff --git a/docs/layer-names.md b/docs/layer-names.md index d006037..505b08d 100644 --- a/docs/layer-names.md +++ b/docs/layer-names.md @@ -16,6 +16,8 @@ cell as the carrier. The carrier must contain no ordinary design objects. The output path must be new; Pheon refuses to overwrite any existing file, including the input. +The output is uncompressed GDSII, including when the input is gzip-compressed. + The command consolidates the legend into that cell. Existing mappings remain unless you override them with `--name`. It creates no cells or references, and copies other records, timestamps, unknown records, and trailing padding byte for diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 1d514b7..80145b4 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,3 +1,4 @@ +use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ Comparison, Diagnostic, Document, Outcome, Side, dump_records, @@ -8,7 +9,7 @@ use std::{ collections::BTreeMap, ffi::{OsStr, OsString}, fs, - io::{self, Write}, + io::{self, Read, Write}, path::PathBuf, process::ExitCode, }; @@ -21,9 +22,36 @@ const HELP: &str = "Usage: Structural comparison: ignore timestamps and element order, normalize polygon start/winding, preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Gzip-compressed GDSII input is recognized by its header. Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. "; +fn read_input(path: &OsStr) -> Result, Diagnostic> { + let io_error = + |error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None); + let mut file = fs::File::open(path).map_err(io_error)?; + let mut bytes = Vec::new(); + (&mut file) + .take(2) + .read_to_end(&mut bytes) + .map_err(io_error)?; + if bytes == b"\x1f\x8b" { + let mut decoded = Vec::new(); + MultiGzDecoder::new(bytes.as_slice().chain(file)) + .read_to_end(&mut decoded) + .map_err(|error| { + Diagnostic::new( + "invalid_input", + format!("{path:?}: gzip decompression failed: {error}"), + None, + ) + })?; + return Ok(decoded); + } + file.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + fn value(args: &[OsString], index: &mut usize) -> Result { *index += 1; args.get(*index) @@ -61,7 +89,7 @@ fn name_layers(args: &[OsString]) -> Result { if names.is_empty() { return Err("provide at least one --name".to_string()); } - let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; + let input = read_input(&args[0]).map_err(|error| error.message)?; let output = write_layer_names( &input, &cell.ok_or("provide --cell")?, @@ -101,8 +129,9 @@ fn prepare_comparison( ) -> Result { let read = |path: Option<&OsStr>, side| { path.map(|path| { - fs::read(path).map_err(|error| { - Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side)) + read_input(path).map_err(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic }) }) .transpose() @@ -267,7 +296,7 @@ fn run(args: &[OsString]) -> Result { { return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); } - let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); + let read = |path: &OsString| read_input(path).map_err(|error| error.message); if raw { if scope != Scope::default() { return Err("record dumps do not accept object scope selectors".to_string()); diff --git a/src/review.rs b/src/review.rs index 8a63291..9bb17a8 100644 --- a/src/review.rs +++ b/src/review.rs @@ -500,6 +500,16 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result Result>, String> { + if input.starts_with(b"\x1f\x8b") { + return Err(error( + 0, + "gzip-compressed input. Decompress it before reading GDSII", + )); + } + if input.starts_with(b"%SEMI-OASIS\r\n") { + return Err(error(0, "OASIS input. Export the layout as GDSII")); + } + let mut records = Vec::new(); let mut offset = 0; while offset < input.len() { diff --git a/tests/review.rs b/tests/review.rs index 3f49221..979c391 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -43,6 +43,14 @@ fn layout(cells: &[(&str, Vec>)]) -> Vec { bytes } +fn gzip(data: &[u8]) -> Vec { + use std::io::Write; + + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + fn polygon(points: &[[i32; 2]]) -> Vec { let mut bytes = Vec::new(); record(&mut bytes, RTAG_BOUNDARY, &[]); @@ -283,6 +291,21 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { } } +#[test] +fn review_identifies_gzip_and_oasis_input() { + for (bytes, expected) in [ + (&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"), + (&b"%SEMI-OASIS\r\n"[..], "OASIS input"), + ] { + let error = Document::parse(bytes).unwrap_err(); + assert!( + error.starts_with(&format!("at byte 0: {expected}")), + "{error}" + ); + assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error); + } +} + #[test] fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { let bytes = layout(&[("CHIP", vec![square()])]); @@ -691,6 +714,73 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { ); } +#[test] +fn cli_decodes_gzip_by_header_and_checks_the_trailer() { + let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let plain = directory.join("plain.gds"); + let compressed = directory.join("compressed.gds"); + let data = shared(square()); + // Splitting a record between gzip members must preserve the decompressed layout. + let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat(); + fs::write(&plain, &data).unwrap(); + fs::write(&compressed, &encoded).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let dump = |path: &PathBuf| { + let output = command() + .arg("dump") + .arg(path) + .arg("--records") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout + }; + assert_eq!(dump(&plain), dump(&compressed)); + let compare = || { + let output = command() + .arg("diff") + .arg(&plain) + .arg(&compressed) + .args(["--format", "json"]) + .output() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + (output.status.code(), report) + }; + let (code, report) = compare(); + assert_eq!(code, Some(0)); + assert_eq!(report["result"], "equal"); + + // Corrupt the last CRC after all the GDSII bytes have already been decoded. + let crc = encoded.len() - 8; + encoded[crc] ^= 0xff; + fs::write(&compressed, encoded).unwrap(); + let (code, report) = compare(); + assert_eq!(code, Some(2)); + assert_eq!(report["status"], "failed"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "invalid_input"); + assert_eq!(diagnostic["side"], "new"); + assert!( + diagnostic["message"] + .as_str() + .unwrap() + .contains("gzip decompression failed") + ); +} + #[test] fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { use pheon::review::names::write_layer_names; @@ -776,9 +866,9 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); - let source = directory.join("source.gds"); + let source = directory.join("source.gds.gz"); let output = directory.join("output.gds"); - let bytes = shared(square()); + let bytes = gzip(&shared(square())); fs::write(&source, &bytes).unwrap(); let _ = fs::remove_file(&output); let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); From d95c378690aaf9a29583ef0e5f4a47cd248ffea6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 16:15:05 -0800 Subject: [PATCH 24/41] Add a reproducible upstream GDSII corpus check Pin 30 layouts from five upstream repositories and verify downloads and license files by SHA-256. Exercise full reads, scans, selected reads, and CLI self-comparisons with a per-layout JSON report. Cache the inputs locally for offline reruns. The baseline exposes a CLI rejection of KLayout's padded-name fixture. The gzip fixture passes directly through the CLI. --- contrib/gds-corpus.toml | 71 ++++++++++++++++ docs/development.md | 14 ++++ tools/check_gds_corpus.py | 167 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 contrib/gds-corpus.toml create mode 100644 tools/check_gds_corpus.py diff --git a/contrib/gds-corpus.toml b/contrib/gds-corpus.toml new file mode 100644 index 0000000..25eeefb --- /dev/null +++ b/contrib/gds-corpus.toml @@ -0,0 +1,71 @@ +# Pinned GDSII test inputs. Each digest covers the original upstream bytes. + +[gdstk] +repo = "heitzmann/gdstk" +revision = "1099be134da34d4afd9e5fb1f4f0dbae9ef4c17a" +license = "BSL-1.0" +license_sha256 = "beb8e42e9d6b4284e03304d05a81a0755200a965fc8d0a5e0aea1e84cf805d6e" + +[gdstk.layouts] +"tests/proof_lib.gds" = "af7b95ed191ebe32438c0f0cb5ed824ba2a98673cfa9ce6f1a9ff5a827c3dbf7" + +[gdsfactory] +repo = "gdsfactory/gdsfactory" +revision = "7407836dd5d872c704fc06f38562a2ffc88049dc" +license = "MIT" +license_sha256 = "cdace89991b496bd052909790d00ece97feab566afd964d0883d2c0605462b5b" + +[gdsfactory.layouts] +"tests/gds/alphabet_3top_cells.gds" = "09b75d5bfeb0051e4972a4d9fc26df5d342604f7ed2a0994376cd7c77e929019" +"tests/gds/big_rect.gds" = "de71342365dc4bd84dbb03307537b7ca2b4dffab98efdf88713cc451e44e6ac2" +"tests/gds/mmi1x2.gds" = "e12e81a9d5b9dc8463cce05915f567bc7174f848a52c7e092dc8a602819dafa0" +"tests/gds/mzi2x2.gds" = "49499f13c888de137eb26342c45ea379d9ea2ecd82fc990780d38fa014002d38" +"tests/gds/small_rect.gds" = "345480096d17bc724ae32cc0f42bd915a39c6e64806b43054d687755afd5faba" +"tests/gds/straight.gds" = "dc5e971b5be91864895870ef70a5ae959bd52dab5971caa0dcfc9aa6e13fa40d" + +[klayout] +repo = "KLayout/klayout" +revision = "53911acd6dfaa113c188d11d20b2ae290aa60a6b" +license = "GPL-3.0 (upstream LICENSE)" +license_sha256 = "589ed823e9a84c56feb95ac58e7cf384626b9cbf4fda2a907bc36e103de1bad2" + +[klayout.layouts] +"samples/lvs/ringo.gds" = "f24d4d11ac7941021d0b0c15d8b32ced1c6735cbd4da68de1482830ca45be3bd" +"testdata/drc/texts.gds" = "ee8ddf7de65ebccf9772fd98cb5d45313e353f8a3455d162611320885061f907" +"testdata/algo/deep_texts_l1.gds" = "253d24491a097102a729ef7f101c9b7acf7684c4a90c16d212ac91d9087e3b4d" +"testdata/gds/arefs.gds" = "fadde330ade22d0e8b92e4040377dcccb6e7a572d9d69df6b3fa722a2bfbcdcd" +"testdata/gds/arefs_skew.gds" = "a681a47cd1d2d74b881d0ab10c3d6fd6c9674c819cf39c077936a4328511b1ae" +"testdata/gds/arefs_skew2.gds" = "363dd2650b4a3011cbe33a6d8febccc77b22dd9b7563f087029a1c5afbe2ae94" +"testdata/gds/basic_instances.gds" = "6ccfffd590c159a5ab450c8eed826afb8e1d68c9ae2f85ec089ab21c0b40ec1b" +"testdata/gds/issue-1353.gds" = "f0e9cbe2b0582d766b3e6e1737617865ca5e74dc525f7bc2872e09ffe6a82482" +"testdata/gds/issue_1835.gds" = "b2d78e8ed4f43e63738857c7aa81606c8ccc5ca0910e3d4e4569c058415d9171" +"testdata/gds/issue_893.gds" = "30a2bf58ea8ae5b36f6df67c5d5221bc257e916b985bc1e7a74cf19ba244eff8" +"testdata/gds/issue_893_au.gds" = "e33cbc5ae2a3f3e9779f6c1cbaf89762c33644c76836c96445e887820dd4eab6" +"testdata/gds/t166_au.gds.gz" = "bd5f0eda73cf13daae77d61eb2a2d9376f2752e66cd458307cfee0e4d5b5ea57" + +[sky130] +repo = "google/skywater-pdk-libs-sky130_fd_sc_hd" +revision = "ac7fb61f06e6470b94e8afdf7c25268f62fbd7b1" +license = "Apache-2.0" +license_sha256 = "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + +[sky130.layouts] +"cells/inv/sky130_fd_sc_hd__inv_1.gds" = "18a9cfa399957c7c7b5fc75175e03fc921d56ecc6b8de8fae34ba301bb7914c4" +"cells/inv/sky130_fd_sc_hd__inv_4.gds" = "1e0fb091a3e4583297e7124d68542e4f29ee6d1f711e80dcb41fc1ed8465c0ce" +"cells/nand2/sky130_fd_sc_hd__nand2_1.gds" = "4e7fa6255df901f59151791d6e8ff4c421b63c1ea3822d6d5d25b48221782a3a" +"cells/dfxtp/sky130_fd_sc_hd__dfxtp_1.gds" = "2b0eaaeedd5b7b2c359942098aee0cef69387108e02e10bc09940680e4cc7ec3" +"cells/tapvpwrvgnd/sky130_fd_sc_hd__tapvpwrvgnd_1.gds" = "642f8841345c7c09ec910fe7893ae803ab760606adfab65229e2aedb055005a5" +"cells/fill/sky130_fd_sc_hd__fill_1.gds" = "9f718b6252a027fc769c2e4298c182e1c4ee3b160fefcd6acf7d947d5c6c3453" + +[ihp] +repo = "IHP-GmbH/IHP-Open-PDK" +revision = "e73adb4045731ddec21634146d536380057f6a26" +license = "Apache-2.0" +license_sha256 = "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + +[ihp.layouts] +"ihp-sg13g2/libs.ref/sg13g2_pr/gds/sg13g2_pr.gds" = "3c640413b455354da11ec4db01c25f165489bcfd4cdec0f79559a9f78331ab42" +"ihp-sg13g2/libs.ref/sg13g2_stdcell/gds/sg13g2_stdcell.gds" = "ab9f2802c78935d4463093c24860f9d3f244057a839179fb19fc0243a18cf3f4" +"ihp-sg13g2/libs.ref/sg13g2_sram/gds/RM_IHPSG13_1P_1024x16_c2_bm_bist.gds" = "1ed0af3f9cc3a3455f4c0beb08394502a875dda3c54bc03123357e397b26e929" +"ihp-sg13g2/libs.qa/drc/devices/sg13g2_qacells.gds" = "21d8b2663ba15fa4c99752ad7a030f486043ba17950236bee8dc483e64c948b2" +"ihp-sg13g2/libs.qa/drc/special/sg13g2_stdcell_abutment.gds" = "e125bd8e6bf8976f41bf29fb761c28559064b50571289da98c148684676e1728" diff --git a/docs/development.md b/docs/development.md index da07331..493f35b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -24,6 +24,20 @@ built library without reinstalling, set `PHEON_LIBRARY` to its absolute path and run `python -m unittest discover -s tests/python` from the checkout. Unset this override when checking installed artifacts with `check_installed.py`. +## Check upstream layouts + +The [GDSII corpus](../contrib/gds-corpus.toml) pins open-source fixtures and cell +libraries. After installing `.[arrow]` and building the release CLI, run: + +```sh +python -I tools/check_gds_corpus.py --fetch +``` + +The checker uses `curl` to cache layouts and license files in `.cache/gds-corpus/` +and verifies their SHA-256 digests. Omit `--fetch` to rerun offline. It checks +full reads, scans, selected reads, and CLI self-comparisons. It writes diagnostics +to `.cache/gds-corpus/report.json` and exits with status 1 if a check fails. + ## Interfaces and reports The CFFI declarations live in [pheon/_ffi.h](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h). Keep them aligned diff --git a/tools/check_gds_corpus.py b/tools/check_gds_corpus.py new file mode 100644 index 0000000..8331145 --- /dev/null +++ b/tools/check_gds_corpus.py @@ -0,0 +1,167 @@ +"""Check a pinned open-source layout corpus against Pheon's installed readers.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tomllib +from urllib.parse import quote + +from pheon import arrow + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_input(path, url, digest, fetch): + if path.is_file(): + data = path.read_bytes() + elif fetch: + data = subprocess.check_output( + ["curl", "-fLsS", "--retry", "2", "--max-time", "60", url] + ) + else: + raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.") + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError(f"SHA-256 mismatch for {path}") + if not path.is_file(): + path.write_bytes(data) + return data + + +def inspect_layout(cli, path, data): + results = {} + reads = { + "full": lambda: arrow.read_path(path), + "scan": lambda: arrow.scan_bytes(data), + } + layouts = {} + for name, read in reads.items(): + try: + layout = read() + layouts[name] = layout + results[name] = { + "status": "ok", + "cells": len(layout.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results[name] = {"status": "failed", "message": str(error)} + + if "scan" in layouts: + cells = layouts["scan"].field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + strict=True, + ) + ) + try: + selected = arrow.read_cells_bytes(data, ranges) + if "full" in layouts and not selected.equals(layouts["full"]): + raise ValueError( + "Reading all scanned cell ranges differs from the full read" + ) + results["selected"] = { + "status": "ok", + "cells": len(selected.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results["selected"] = {"status": "failed", "message": str(error)} + else: + results["selected"] = {"status": "skipped"} + + try: + run = subprocess.run( + [cli, "diff", path, path, "--format", "json"], + capture_output=True, + timeout=60, + ) + except (subprocess.TimeoutExpired, OSError) as error: + results["cli"] = {"status": "failed", "message": str(error)} + return results + try: + report = json.loads(run.stdout) + results["cli"] = { + "status": report["status"], + "result": report["result"], + "exit_code": run.returncode, + "diagnostics": report["diagnostics"], + } + if (report["status"], report["result"], run.returncode) not in { + ("complete", "equal", 0), + ("incomplete", "unknown", 2), + ("failed", "unknown", 2), + }: + results["cli"]["status"] = "failed" + except (ValueError, KeyError, TypeError) as error: + results["cli"] = { + "status": "failed", + "message": str(error), + "exit_code": run.returncode, + "stderr": run.stderr[-4096:].decode("utf-8", errors="replace"), + } + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fetch", action="store_true", help="download missing inputs using curl" + ) + binary = "pheon.exe" if os.name == "nt" else "pheon" + parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary) + parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus") + args = parser.parse_args() + cli = args.cli.resolve() + if not cli.is_file(): + parser.error(f"{cli} is missing. Build the CLI or pass --cli.") + + for name in ("files", "licenses"): + (args.cache / name).mkdir(parents=True, exist_ok=True) + sources = tomllib.loads( + (ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8") + ) + all_results = {} + for key, source in sources.items(): + base = ( + f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}" + ) + read_input( + args.cache / "licenses" / f"{key}-LICENSE", + base + "/LICENSE", + source["license_sha256"], + args.fetch, + ) + for upstream, digest in source["layouts"].items(): + name = f"{key}-{Path(upstream).name}" + path = args.cache / "files" / name + raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch) + data = gzip.decompress(raw) if path.suffix == ".gz" else raw + results = inspect_layout(cli, path, data) + all_results[name] = results + print( + name, + *( + f"{operation}={result['status']}" + for operation, result in results.items() + ), + ) + + report = args.cache / "report.json" + report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8") + failures = sum( + any(result["status"] == "failed" for result in case.values()) + for case in all_results.values() + ) + print( + f"{len(all_results)} layouts checked, {failures} with failures. See {report}." + ) + return int(failures != 0) + + +if __name__ == "__main__": + raise SystemExit(main()) From 15e8a3b7b55112eb3cb2257225634e21c64e1a2c Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 21:49:26 -0800 Subject: [PATCH 25/41] Share GDSII reader logic and simplify review output Share record framing, name decoding, payload validation, array reference decoding, and Arrow cell-header parsing. Accept trailing NUL padding in names and retain located diagnostics for malformed records. Summarize text diffs by cell, layer, and reference target. Show comparison and omission counts before detail. Compact canonical dumps while preserving exact fields and duplicate counts for Git textconv. Simplify hierarchy output and retain per-object evidence in JSON. --- docs/cli.md | 30 +- src/basic.rs | 19 +- src/bin/pheon.rs | 2 +- src/elements.rs | 333 +++++--------- src/library.rs | 60 ++- src/record.rs | 79 +++- src/records.rs | 351 +++++++------- src/review.rs | 733 ++++++++++++++++++++---------- src/review/changes.rs | 341 ++++++++++++-- src/review/report.rs | 19 +- src/review/selection.rs | 17 + tests/golden/report-geometry.json | 2 +- tests/golden/review-canonical.txt | 11 +- tests/golden/review-diff.txt | 10 +- tests/golden/review-hierarchy.mmd | 1 - tests/golden/review-hierarchy.txt | 4 +- tests/parser_errors.rs | 136 +++++- tests/review.rs | 341 ++++++++++++-- tests/review/reports.rs | 32 +- tests/selective_reads.rs | 36 +- 20 files changed, 1723 insertions(+), 834 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 95ea2cf..8312883 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -28,10 +28,15 @@ pheon hierarchy chip.gds --format ascii --depth 4 pheon hierarchy chip.gds --format mermaid --top CHIP ``` -`dump` shows a normalized layout description. Add `--records` to inspect raw -records, including timestamps. `hierarchy` shows cell references without -expanding array placements. Shared cells are marked rather than repeated -indefinitely. Mermaid output is diagram source for a Mermaid renderer. +`dump` lists objects by cell. A prefix such as `3x` counts copies of the same +normalized object within a cell. Coordinates are in database units. Rectangles +show two corners, and longer vertex lists wrap across lines. References show +the target and any array dimensions before the coordinates. Add `--records` +to inspect the original record sequence, including timestamps. + +`hierarchy` shows direct placement counts on references. Shared cells are +marked where their contents appear earlier in the tree. Mermaid output is +diagram source for a Mermaid renderer. ## Compare revisions @@ -41,10 +46,13 @@ pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 pheon diff before.gds after.gds --format json > report.json ``` -Text reports group changes by cell. A `-` line shows an object from the old file -and a `+` line shows one from the new file. A `~` line explains a likely -replacement when Pheon can pair the two uniquely. Request a hierarchy view to -see which parent cells are affected. +Text reports start with the result and group changes by cell. Added and removed +cell headings give the object count within the comparison scope. Each layer +lists counts of rectangles, polygons, and boxes changed, moved, added, or removed. +Added or removed cells also count references by target. +Other objects use `-` for removals, `+` for additions, and `~` for likely +replacements. Use `--format json` for details of individual objects and +`--hierarchy` to see which parent cells are affected. | Exit code | Meaning | | --- | --- | @@ -86,8 +94,10 @@ pheon diff before.gds after.gds --hierarchy mermaid --depth 4 `--show-cell` and `--show-layer` filter displayed changes without changing the comparison, totals, or exit status. A layer display filter hides references -without a numeric layer. `--max-changes N` limits displayed object occurrences; -JSON includes all changed objects unless this limit is explicit. +without a numeric layer. `--max-changes N` limits displayed object occurrences. +A paired replacement uses two occurrences. The text report counts omissions +from filters and limits before listing cell changes. JSON includes all changed +objects unless the limit is explicit. The diff hierarchy marks directly changed cells, affected ancestors, and unchanged targets of modified references. It includes references from both diff --git a/src/basic.rs b/src/basic.rs index 4bda079..e4de7a0 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -98,6 +98,16 @@ pub(crate) fn decode_utf8(bytes: Vec) -> Result { .map_err(|error| ErrType::Failed(format!("Invalid UTF-8 string: {}", error.utf8_error()))) } +pub(crate) fn decode_name(mut bytes: &[u8]) -> Result<&str, String> { + while let Some(unpadded) = bytes.strip_suffix(&[0]) { + bytes = unpadded; + } + if bytes.contains(&0) { + return Err("name contains an embedded NUL".to_string()); + } + std::str::from_utf8(bytes).map_err(|error| format!("Invalid UTF-8 string: {error}")) +} + pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { let mut bits = [false; 16]; @@ -110,11 +120,12 @@ pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec> { - let length = length as usize; let (input, data) = take_bytes(input, length)?; - let true_length = length - usize::from(data.last() == Some(&0)); - let vec = data[..true_length].to_vec(); - Ok((input, vec)) + Ok((input, unpad_ascii(data).to_vec())) +} + +pub(crate) fn unpad_ascii(data: &[u8]) -> &[u8] { + data.strip_suffix(&[0]).unwrap_or(data) } diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 80145b4..9fcc786 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -348,7 +348,7 @@ fn run(args: &[OsString]) -> Result { output.push_str(&diff.text_filtered(max_changes, &show)); if format != "none" { if format == "mermaid" { - output.push_str("```mermaid\n"); + output.push_str("Hierarchy\n```mermaid\n"); } output.push_str(&diff.hierarchy(format == "mermaid", depth)); if format == "mermaid" { diff --git a/src/elements.rs b/src/elements.rs index 32fd9c8..cfee79c 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -5,14 +5,14 @@ use crate::records::{ DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, - LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, ArrayPlacement, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, ENDEL, BGNEXTN, ENDEXTN, SNAME, }; use crate::records; -use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, decode_utf8, fail, take_bytes}; +use crate::record::{Record, RecordFrame}; +use crate::basic::{ErrType, IResult, decode_name, decode_utf8, fail}; use std::string::String; use std::collections::HashMap; @@ -125,11 +125,11 @@ pub(crate) fn read_elements<'a>( let mut boundaries = Vec::::new(); let mut srefs = Vec::::new(); let mut arefs = Vec::::new(); - let (_, mut header) = RecordHeader::read(input)?; // don't consume tag - while header.tag != records::RTAG_ENDSTR { - (input, _) = match header.tag { + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { records::RTAG_SREF | records::RTAG_AREF => { - let (next_input, ref_record) = read_ref(input, header.tag, names)?; + let (next_input, ref_record) = read_ref(input, record.tag, names)?; match ref_record { RefRecord::SRef(sref) => srefs.push(sref), RefRecord::ARef(aref) => arefs.push(aref), @@ -142,18 +142,17 @@ pub(crate) fn read_elements<'a>( (next_input, ()) }, records::RTAG_PATH => read_path(input, cell_builder, layers)?, - records::RTAG_NODE => read_boxnode(input, cell_builder, header.tag, layers)?, - records::RTAG_BOX => read_boxnode(input, cell_builder, header.tag, layers)?, - records::RTAG_TEXT => read_text(input, cell_builder, layers)?, - _ => { - // don't care, skip - let result = take_bytes(input, header.data_size + 4)?; - (result.0, ()) + records::RTAG_NODE | records::RTAG_BOX => { + read_boxnode(input, cell_builder, record.tag, layers)? } + records::RTAG_TEXT => read_text(input, cell_builder, layers)?, + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; + } + _ => (remaining, ()), }; - (_, header) = RecordHeader::read(input)?; // don't consume tag - } - (input, _) = take_bytes(input, 4_usize)?; // consume endstr tag + }; let mut sref_plain = Vec::::with_capacity(srefs.len() + arefs.len() / 4); let mut sref_props = Vec::::with_capacity(srefs.len() / 16 + arefs.len() / 64); @@ -220,7 +219,7 @@ pub(crate) fn read_elements<'a>( let mut next_offset: u32 = 0; for boundary in group { - if let Some(rect) = boundary_to_rect(&boundary.vertices) { + if let Some(rect) = XY::rectangle(boundary.vertices.as_chunks::<2>().0) { rect_batch.rects.extend_from_slice(&rect); } else { boundary_batch.vertex_offsets.push(next_offset); @@ -297,11 +296,11 @@ pub(crate) fn scan_elements<'a>( ) -> IResult<'a, Vec> { let mut input = input; let mut refs = Vec::::new(); - let (_, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - (input, _) = match header.tag { + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { records::RTAG_SREF | records::RTAG_AREF => { - let (next_input, ref_record) = read_ref(input, header.tag, names)?; + let (next_input, ref_record) = read_ref(input, record.tag, names)?; refs.push(scan_ref_from_record(ref_record)); (next_input, ()) }, @@ -310,15 +309,14 @@ pub(crate) fn scan_elements<'a>( | records::RTAG_NODE | records::RTAG_BOX | records::RTAG_TEXT => skip_element(input)?, - _ => { - let result = take_bytes(input, header.data_size + 4)?; - (result.0, ()) + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; } + _ => (remaining, ()), }; - (_, header) = RecordHeader::read(input)?; - } + }; refs.sort_by_key(|scan_ref| scan_ref.target_id); - (input, _) = take_bytes(input, 4_usize)?; Ok((input, refs)) } @@ -373,12 +371,12 @@ fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { let mut input = input; loop { - let (next_input, header) = RecordHeader::read(input)?; - let (next_input, _) = take_bytes(next_input, header.data_size)?; - input = next_input; - if header.tag == ENDEL::tag() { - return Ok((input, ())); + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == ENDEL::tag() { + record.decode::()?; + return Ok((remaining, ())); } + input = remaining; } } @@ -600,33 +598,16 @@ fn read_path<'a>( let mut bgn_ext = None; let mut end_ext = None; - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PATHTYPE => { - let _path_type; - (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; - path_type = Some(_path_type); - }, - records::RTAG_WIDTH => { - let _width; - (input, _width) = WIDTH::read_data(input, header.data_size)?; - width = Some(_width); - }, - records::RTAG_BGNEXTN => { - let _bgn_ext; - (input, _bgn_ext) = BGNEXTN::read_data(input, header.data_size)?; - bgn_ext = Some(_bgn_ext); - }, - records::RTAG_ENDEXTN => { - let _end_ext; - (input, _end_ext) = ENDEXTN::read_data(input, header.data_size)?; - end_ext = Some(_end_ext); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), + records::RTAG_BGNEXTN => bgn_ext = Some(record.decode::()?), + records::RTAG_ENDEXTN => end_ext = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; } let path_type_builder = path_builder.field_builder::(1).unwrap(); path_type_builder.append_option(path_type); @@ -638,7 +619,7 @@ fn read_path<'a>( width_builder.append_option(width); let xys_builder = path_builder.field_builder::(5).unwrap(); - let (input, xy_iter) = XY::read_data(input, header.data_size)?; + let xy_iter = record.decode::()?; for xy in xy_iter { let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); xy_builder.append_value(xy); @@ -717,46 +698,27 @@ fn read_text<'a>( let (input, dtype) = TEXTTYPE::read(input)?; insert_layer(layer, dtype, layers, text_builder, 0); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { // TODO warn if repeat tags? records::RTAG_PRESENTATION => { - let _presentation; - (input, _presentation) = PRESENTATION::read_data(input, header.data_size)?; - pres_hori = Some(_presentation[14] as u8 * 2 + _presentation[15] as u8); - pres_vert = Some(_presentation[12] as u8 * 2 + _presentation[13] as u8); - pres_font = Some(_presentation[10] as u8 * 2 + _presentation[11] as u8); - }, - records::RTAG_PATHTYPE => { - let _path_type; - (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; - path_type = Some(_path_type); - }, - records::RTAG_WIDTH => { - let _width; - (input, _width) = WIDTH::read_data(input, header.data_size)?; - width = Some(_width); - }, + let presentation = record.decode::()?; + pres_hori = Some(presentation[14] as u8 * 2 + presentation[15] as u8); + pres_vert = Some(presentation[12] as u8 * 2 + presentation[13] as u8); + pres_font = Some(presentation[10] as u8 * 2 + presentation[11] as u8); + } + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), records::RTAG_STRANS => { - let strans; - (input, strans) = STRANS::read_data(input, header.data_size)?; + let strans = record.decode::()?; invert_y = Some(strans[0]); - }, - records::RTAG_MAG => { - let _mag; - (input, _mag) = MAG::read_data(input, header.data_size)?; - mag = Some(_mag); - }, - records::RTAG_ANGLE => { - let _angle_deg; - (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; - angle_deg = Some(_angle_deg); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), + } + records::RTAG_MAG => mag = Some(record.decode::()?), + records::RTAG_ANGLE => angle_deg = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), } - (input, header) = RecordHeader::read(input)?; + (input, record) = RecordFrame::read(input)?; } let pres_hori_builder = text_builder.field_builder::(1).unwrap(); @@ -776,10 +738,11 @@ fn read_text<'a>( let angle_builder = text_builder.field_builder::(8).unwrap(); angle_builder.append_option(angle_deg); - let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; - let xx = xy_iter.next().unwrap(); - let yy = xy_iter.next().unwrap(); - let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let mut xy_iter = record.decode::()?; + let (Some(xx), Some(yy)) = (xy_iter.next(), xy_iter.next()) else { + return fail(record.data, "TEXT needs an XY coordinate pair".to_string()); + }; + let xy = pack_point([xx, yy]); let xy_builder = text_builder.field_builder::(9).unwrap(); xy_builder.append_value(xy); @@ -796,48 +759,11 @@ fn read_text<'a>( } -fn boundary_to_rect(vertices: &[i32]) -> Option<[i32; 4]> { - if vertices.len() != 8 { - return None; - } - - let xs = [vertices[0], vertices[2], vertices[4], vertices[6]]; - let ys = [vertices[1], vertices[3], vertices[5], vertices[7]]; - - let min_x = *xs.iter().min().unwrap(); - let max_x = *xs.iter().max().unwrap(); - let min_y = *ys.iter().min().unwrap(); - let max_y = *ys.iter().max().unwrap(); - if min_x >= max_x || min_y >= max_y { - return None; - } - - let mut seen_corners: u8 = 0; - for ii in 0..4 { - let jj = (ii + 1) % 4; - let same_x = xs[jj] == xs[ii]; - let same_y = ys[jj] == ys[ii]; - if same_x == same_y { - return None; - } - - let x = xs[ii]; - let y = ys[ii]; - if (x != min_x && x != max_x) || (y != min_y && y != max_y) { - return None; - } - let code = ((x == max_x) as u8) << 1 | ((y == max_y) as u8); - seen_corners |= 1 << code; - } - - if seen_corners != 0b1111 { - return None; - } - Some([min_x, min_y, max_x, max_y]) +fn pack_point([x, y]: [i32; 2]) -> u64 { + ((x as u32 as u64) << 32) | (y as u32 as u64) } - fn read_ref<'a>( input: &'a [u8], tag: u16, @@ -856,65 +782,46 @@ fn read_ref<'a>( let mut colrow = None; let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; - let struct_name = decode_utf8(struct_name_bytes).map_err(|error| (input, error))?; + let struct_name = decode_name(&struct_name_bytes) + .map_err(|error| (input, ErrType::Failed(error)))?; let next_id = names.len(); - let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); + let id = names.entry(struct_name.to_owned()).or_insert(next_id.try_into().unwrap()); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { records::RTAG_STRANS => { - let strans; - (input, strans) = STRANS::read_data(input, header.data_size)?; + let strans = record.decode::()?; invert_y = strans[0]; - }, - records::RTAG_MAG => { - let _mag; - (input, _mag) = MAG::read_data(input, header.data_size)?; - scale = _mag; - }, + } + records::RTAG_MAG => scale = record.decode::()?, records::RTAG_ANGLE => { - let _angle_deg; - (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; - angle_rad = _angle_deg.to_radians(); - }, + angle_rad = record.decode::()?.to_radians(); + } records::RTAG_COLROW => { - if header.data_size != 4 { - return fail(input, "COLROW must contain two counts".to_string()); - } - let mut _colrow; - (input, _colrow) = COLROW::read_data(input, header.data_size)?; - colrow = Some((_colrow.next().unwrap(), _colrow.next().unwrap())); if !is_aref { - return fail(input, "Got a COLROW record inside an SREF".to_string()); + return fail(record.data, "Got a COLROW record inside an SREF".to_string()); } - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; + COLROW::check_size(record.data.len() as u16) + .map_err(|error| (record.data, ErrType::Failed(error)))?; + colrow = Some(record.data); + } + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; } - let expected_xy_size = if is_aref { 24 } else { 8 }; - if header.data_size != expected_xy_size { - return fail(input, format!("Expected {expected_xy_size} bytes of reference coordinates, got {}", header.data_size)); - } - let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; - let xx = xy_iter.next().unwrap(); - let yy = xy_iter.next().unwrap(); - let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); let (input, properties) = read_properties_vec(input)?; if is_aref { - if colrow.is_none() { - return fail(input, "AREF without COLROW before XY".to_string()) - } - let (count0, count1) = colrow.unwrap(); - if count0 <= 0 || count1 <= 0 { - return fail(input, "AREF column and row counts must be positive".to_string()); - } + let Some(colrow) = colrow else { + return fail(input, "AREF without COLROW before XY".to_string()); + }; + let placement = ArrayPlacement::decode(colrow, record.data)?; + let [columns, rows] = placement.counts; + let [origin, column_end, row_end] = placement.points; - let pitch = |end: i32, origin: i32, count: i16| { + let pitch = |end: i32, origin: i32, count: u16| { let delta = i64::from(end) - i64::from(origin); let divisor = i64::from(count); if delta % divisor != 0 { @@ -927,32 +834,35 @@ fn read_ref<'a>( "AREF pitch exceeds signed 32-bit coordinate range".to_string() ))) }; - let x0a = pitch(xy_iter.next().unwrap(), xx, count0)?; - let y0a = pitch(xy_iter.next().unwrap(), yy, count0)?; - let x1a = pitch(xy_iter.next().unwrap(), xx, count1)?; - let y1a = pitch(xy_iter.next().unwrap(), yy, count1)?; - - let xy0a = ((x0a as u32 as u64) << 32) | (y0a as u32 as u64); - let xy1a = ((x1a as u32 as u64) << 32) | (y1a as u32 as u64); - let counts = ((count0 as u16 as u32) << 16) | (count1 as u16 as u32); + let axis_pitch = |end: [i32; 2], count| -> Result<_, _> { + Ok(pack_point([ + pitch(end[0], origin[0], count)?, + pitch(end[1], origin[1], count)?, + ])) + }; + let counts = (u32::from(columns) << 16) | u32::from(rows); Ok((input, RefRecord::ARef(ARefRecord { target_id: *id, invert_y, scale, angle_rad, - xy, - xy0: xy0a, - xy1: xy1a, + xy: pack_point(origin), + xy0: axis_pitch(column_end, columns)?, + xy1: axis_pitch(row_end, rows)?, counts, properties, }))) } else { + if record.data.len() != 8 { + return fail(record.data, "SREF needs exactly one XY coordinate pair".to_string()); + } + let origin = XY::points(record.data).next().unwrap(); Ok((input, RefRecord::SRef(SRefRecord { target_id: *id, invert_y, scale, angle_rad, - xy, + xy: pack_point(origin), properties, }))) } @@ -976,22 +886,25 @@ fn walk_properties<'a>( input: &'a [u8], mut on_property: impl FnMut(PropertyRecord), ) -> IResult<'a, ()> { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != ENDEL::tag() { - if header.tag == PROPATTR::tag() { - let key; - let value_bytes; - (input, key) = PROPATTR::read_data(input, header.data_size)?; - (input, value_bytes) = PROPVALUE::read(input)?; - - on_property(PropertyRecord { - key, - value: decode_utf8(value_bytes).map_err(|error| (input, error))?, - }); + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { + records::RTAG_ENDEL => { + record.decode::()?; + return Ok((remaining, ())); + } + records::RTAG_PROPATTR => { + let key = record.decode::()?; + let value_bytes; + (input, value_bytes) = PROPVALUE::read(remaining)?; + let value = decode_utf8(value_bytes).map_err(|error| (input, error))?; + on_property(PropertyRecord { key, value }); + } + _ if record.data.is_empty() => input = remaining, + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), } - (input, header) = RecordHeader::read(input)?; } - Ok((input, ())) } diff --git a/src/library.rs b/src/library.rs index d4db8f2..2e0ac81 100644 --- a/src/library.rs +++ b/src/library.rs @@ -4,10 +4,10 @@ use std::io::Write; -use crate::record::{RecordHeader, Record}; +use crate::record::{Record, RecordFrame}; use crate::records; use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -use crate::basic::{ErrType, IResult, OResult, decode_utf8, take_bytes, fail}; +use crate::basic::{ErrType, IResult, OResult, decode_name, fail}; use std::string::String; use std::collections::HashMap; @@ -325,7 +325,7 @@ fn append_file_header( lib_builder: &mut StructBuilder, header: &FileHeader, ) -> Result<(), ErrType> { - let name = decode_utf8(header.name.clone())?; + let name = decode_name(&header.name).map_err(ErrType::Failed)?; let dbu_builder = lib_builder.field_builder::(0).unwrap(); dbu_builder.append_value(header.meters_per_db_unit); let uu_builder = lib_builder.field_builder::(1).unwrap(); @@ -409,6 +409,24 @@ fn intern_name(names: &mut HashMap, name: String) -> u32 { } +fn read_cell_header<'a>(begin: &RecordFrame<'a>, input: &'a [u8]) -> IResult<'a, String> { + if begin.tag != records::RTAG_BGNSTR { + return fail( + begin.data, + format!( + "Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", + begin.tag, + records::RTAG_BGNSTR + ), + ); + } + begin.decode::()?; + let (input, name_bytes) = records::STRNAME::read(input)?; + let name = decode_name(&name_bytes).map_err(|error| (input, ErrType::Failed(error)))?; + Ok((input, name.to_owned())) +} + + fn append_full_cell<'a>( cells_builder: &mut DListBuilder, names: &mut HashMap, @@ -440,23 +458,23 @@ where F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, { let input_size = input.len(); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDLIB { - let record_start = base_offset + input_size - input.len() - 4; - if header.tag == records::RTAG_BGNSTR { - (input, _) = records::BGNSTR::read_data(input, header.data_size)?; - let name_bytes; - (input, name_bytes) = records::STRNAME::read(input)?; - let name = decode_utf8(name_bytes).map_err(|error| (input, error))?; - let id = intern_name(names, name); - let (next_input, ()) = on_struct(input, record_start, id, names)?; - input = next_input; - } else { - (input, _) = take_bytes(input, header.data_size)?; + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { + records::RTAG_BGNSTR => { + let record_start = base_offset + input_size - input.len(); + let (after_name, name) = read_cell_header(&record, remaining)?; + let id = intern_name(names, name); + (input, ()) = on_struct(after_name, record_start, id, names)?; + } + records::RTAG_ENDLIB => { + record.decode::()?; + return Ok((remaining, ())); + } + _ => input = remaining, } - (input, header) = RecordHeader::read(input)?; } - Ok((input, ())) } @@ -559,10 +577,8 @@ pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IRes let start = usize::try_from(*start_u64).unwrap(); let end = usize::try_from(*end_u64).unwrap(); - let struct_input = &input[start..end]; - let (struct_input, _) = records::BGNSTR::read(struct_input)?; - let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; - let name = decode_utf8(name_bytes).map_err(|error| (struct_input, error))?; + let (struct_input, begin) = RecordFrame::read(&input[start..end])?; + let (struct_input, name) = read_cell_header(&begin, struct_input)?; let file_offset = end - struct_input.len(); let id = intern_name(&mut names, name); diff --git a/src/record.rs b/src/record.rs index 6de51d1..5ad0762 100644 --- a/src/record.rs +++ b/src/record.rs @@ -6,7 +6,7 @@ use std::io::{self, Write}; use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; +use crate::basic::{ErrType, IResult, OResult, fail, incomplete, take_bytes}; use crate::records; @@ -19,12 +19,15 @@ pub struct RecordHeader { impl RecordHeader { pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { - let (input, size) = parse_u16(input)?; - let (input, tag) = parse_u16(input)?; + let Some((&[size_hi, size_lo, tag_hi, tag_lo], payload)) = input.split_first_chunk::<4>() else { + return incomplete(input, Some(4)); + }; + let size = u16::from_be_bytes([size_hi, size_lo]); let Some(data_size) = size.checked_sub(4) else { return fail(input, format!("Record size {size} is smaller than its header")); }; - Ok((input, RecordHeader{tag, data_size})) + let tag = u16::from_be_bytes([tag_hi, tag_lo]); + Ok((payload, RecordHeader { tag, data_size })) } pub fn pack_into(&self) -> io::Result<[u8; 4]> { @@ -43,6 +46,39 @@ impl RecordHeader { } +/// A complete record borrowing its payload from the input. +pub(crate) struct RecordFrame<'a> { + pub tag: u16, + pub data: &'a [u8], +} + +impl<'a> RecordFrame<'a> { + pub fn read(input: &'a [u8]) -> IResult<'a, Self> { + let (_, header) = RecordHeader::read(input)?; + let (remaining, data) = read_payload(input, header.data_size)?; + Ok((remaining, Self { tag: header.tag, data })) + } + + /// Decode the payload after the caller has matched the record tag. + pub fn decode + ?Sized, D: RecordData>( + &self, + ) -> Result, (&'a [u8], ErrType)> { + let (_, value) = R::read_data(self.data, self.data.len() as u16)?; + Ok(value) + } +} + +fn read_payload(input: &[u8], size: u16) -> IResult<'_, &[u8]> { + if !size.is_multiple_of(2) { + return fail(input, "odd record length".to_string()); + } + let Some((record, remaining)) = input.split_at_checked(4 + usize::from(size)) else { + return fail(input, "truncated record".to_string()); + }; + Ok((remaining, &record[4..])) +} + + pub trait RecordData { type BareData<'a>; type InData : ?Sized; @@ -91,25 +127,28 @@ pub trait Record { /// fn skip_past(input: &[u8]) -> IResult<'_, bool> { let original_input = input; - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_ENDLIB { - return Ok((original_input, false)) + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, true)); } - (input, header) = RecordHeader::read(input)?; + if record.tag == records::RTAG_ENDLIB { + return Ok((original_input, false)); + } + input = remaining; } - (input, _) = take_bytes(input, header.data_size)?; - Ok((input, true)) } fn skip_and_read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - (input, header) = RecordHeader::read(input)?; + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, record.decode::()?)); + } + input = remaining; } - Self::read_data(input, header.data_size) } fn expect_header(input: &[u8]) -> IResult<'_, u16> { @@ -122,8 +161,10 @@ pub trait Record { } fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { - let (input, size) = Self::expect_header(input)?; - Self::read_data(input, size) + let (_, size) = Self::expect_header(input)?; + let (remaining, data) = read_payload(input, size)?; + let (_, value) = Self::read_data(data, size)?; + Ok((remaining, value)) } fn write(ww: &mut W, data: &RData::InData) -> OResult { diff --git a/src/records.rs b/src/records.rs index 5481e2b..705652e 100644 --- a/src/records.rs +++ b/src/records.rs @@ -2,7 +2,11 @@ /// Record type and tag definitions /// -use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; +use crate::basic::{ErrType, IResult, fail}; +use crate::record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, +}; // record tags pub const RTAG_HEADER: u16 = 0x0002; @@ -77,16 +81,64 @@ pub const RTAG_CONTACT: u16 = 0x4500; -pub struct HEADER; -impl Record for HEADER { - fn tag() -> u16 { RTAG_HEADER } - fn expected_size() -> Option { Some(2) } +// Define fixed-size records once for typed decoding and raw-record validation. +macro_rules! fixed_size_records { + ($($(#[$attr:meta])* $name:ident => $tag:ident, $data:ty, $size:expr;)+) => { + $( + $(#[$attr])* + pub struct $name; + impl Record<$data> for $name { + fn tag() -> u16 { $tag } + fn expected_size() -> Option { Some($size) } + } + )+ + + pub(crate) fn check_payload_size(tag: u16, size: u16) -> Result<(), String> { + match tag { + $($tag => $name::check_size(size),)+ + RTAG_UNITS => UNITS::check_size(size), + RTAG_XY => XY::check_size(size), + _ => Ok(()), + } + } + }; } -pub struct BGNLIB; -impl Record for BGNLIB { - fn tag() -> u16 { RTAG_BGNLIB } - fn expected_size() -> Option { Some(2 * 2 * 6) } +fixed_size_records! { + HEADER => RTAG_HEADER, Int2, 2; + BGNLIB => RTAG_BGNLIB, DateTimePair, 2 * 2 * 6; + ENDLIB => RTAG_ENDLIB, Empty, 0; + BGNSTR => RTAG_BGNSTR, DateTimePair, 2 * 6 * 2; + ENDSTR => RTAG_ENDSTR, Empty, 0; + BOUNDARY => RTAG_BOUNDARY, Empty, 0; + PATH => RTAG_PATH, Empty, 0; + SREF => RTAG_SREF, Empty, 0; + AREF => RTAG_AREF, Empty, 0; + TEXT => RTAG_TEXT, Empty, 0; + LAYER => RTAG_LAYER, Int2, 2; + DATATYPE => RTAG_DATATYPE, Int2, 2; + WIDTH => RTAG_WIDTH, Int4, 4; + ENDEL => RTAG_ENDEL, Empty, 0; + COLROW => RTAG_COLROW, Int2Array, 4; + NODE => RTAG_NODE, Empty, 0; + TEXTTYPE => RTAG_TEXTTYPE, Int2, 2; + PRESENTATION => RTAG_PRESENTATION, BitArray, 2; + STRANS => RTAG_STRANS, BitArray, 2; + MAG => RTAG_MAG, Real8, 8; + ANGLE => RTAG_ANGLE, Real8, 8; + PATHTYPE => RTAG_PATHTYPE, Int2, 2; + GENERATIONS => RTAG_GENERATIONS, Int2, 2; + ELFLAGS => RTAG_ELFLAGS, BitArray, 2; + NODETYPE => RTAG_NODETYPE, Int2, 2; + PROPATTR => RTAG_PROPATTR, Int2, 2; + BOX => RTAG_BOX, Empty, 0; + BOXTYPE => RTAG_BOXTYPE, Int2, 2; + PLEX => RTAG_PLEX, Int4, 4; + BGNEXTN => RTAG_BGNEXTN, Int4, 4; + ENDEXTN => RTAG_ENDEXTN, Int4, 4; + STRCLASS => RTAG_STRCLASS, Int2, 2; + /// End of MASKS records + ENDMASKS => RTAG_ENDMASKS, Empty, 0; } pub struct LIBNAME; @@ -100,18 +152,14 @@ impl Record for UNITS { // (user_units_per_db_unit, db_units_per_meter) fn tag() -> u16 { RTAG_UNITS } fn expected_size() -> Option { Some(2 * 8) } -} -pub struct ENDLIB; -impl Record for ENDLIB { - fn tag() -> u16 { RTAG_ENDLIB } - fn expected_size() -> Option { Some(0) } -} - -pub struct BGNSTR; -impl Record for BGNSTR { - fn tag() -> u16 { RTAG_BGNSTR } - fn expected_size() -> Option { Some(2 * 6 * 2) } + fn read_data(input: &[u8], size: u16) -> IResult<'_, (f64, f64)> { + let (remaining, (user, meters)) = Real8Pair::read(input, size)?; + if !user.is_finite() || user <= 0.0 || !meters.is_finite() || meters <= 0.0 { + return fail(input, "UNITS must be positive finite values".to_string()); + } + Ok((remaining, (user, meters))) + } } pub struct STRNAME; @@ -120,70 +168,102 @@ impl Record for STRNAME { fn expected_size() -> Option { None } } -pub struct ENDSTR; -impl Record for ENDSTR { - fn tag() -> u16 { RTAG_ENDSTR } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOUNDARY; -impl Record for BOUNDARY { - fn tag() -> u16 { RTAG_BOUNDARY } - fn expected_size() -> Option { Some(0) } -} - -pub struct PATH; -impl Record for PATH { - fn tag() -> u16 { RTAG_PATH } - fn expected_size() -> Option { Some(0) } -} - -pub struct SREF; -impl Record for SREF { - fn tag() -> u16 { RTAG_SREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct AREF; -impl Record for AREF { - fn tag() -> u16 { RTAG_AREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXT; -impl Record for TEXT { - fn tag() -> u16 { RTAG_TEXT } - fn expected_size() -> Option { Some(0) } -} - -pub struct LAYER; -impl Record for LAYER { - fn tag() -> u16 { RTAG_LAYER } - fn expected_size() -> Option { Some(2) } -} - -pub struct DATATYPE; -impl Record for DATATYPE { - fn tag() -> u16 { RTAG_DATATYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct WIDTH; -impl Record for WIDTH { - fn tag() -> u16 { RTAG_WIDTH } - fn expected_size() -> Option { Some(4) } -} - pub struct XY; impl Record for XY { fn tag() -> u16 { RTAG_XY } fn expected_size() -> Option { None } + + fn check_size(size: u16) -> Result<(), String> { + if !size.is_multiple_of(8) { + return Err("XY payload must contain whole coordinate pairs".to_string()); + } + Ok(()) + } } -pub struct ENDEL; -impl Record for ENDEL { - fn tag() -> u16 { RTAG_ENDEL } - fn expected_size() -> Option { Some(0) } +impl XY { + /// Iterate over a validated XY payload. + pub(crate) fn points(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + }) + } + + /// Bounds of a rectangle encoded as a closed XY ring. + pub(crate) fn closed_rectangle(data: &[u8]) -> Option<[i32; 4]> { + if data.len() != 40 { + return None; + } + let mut points = Self::points(data); + let corners: [[i32; 2]; 4] = std::array::from_fn(|_| points.next().unwrap()); + if points.next() != Some(corners[0]) { + return None; + } + Self::rectangle(&corners) + } + + /// Bounds of four corners in ring order, with the closing point omitted. + pub(crate) fn rectangle(corners: &[[i32; 2]]) -> Option<[i32; 4]> { + let corners: &[[i32; 2]; 4] = corners.try_into().ok()?; + let min_x = corners.iter().map(|point| point[0]).min().unwrap(); + let max_x = corners.iter().map(|point| point[0]).max().unwrap(); + let min_y = corners.iter().map(|point| point[1]).min().unwrap(); + let max_y = corners.iter().map(|point| point[1]).max().unwrap(); + if min_x == max_x || min_y == max_y { + return None; + } + + let mut seen = 0_u8; + for (index, &[x, y]) in corners.iter().enumerate() { + let [next_x, next_y] = corners[(index + 1) % corners.len()]; + if (x == next_x) == (y == next_y) { + return None; + } + if (x != min_x && x != max_x) || (y != min_y && y != max_y) { + return None; + } + let corner = u8::from(x == max_x) << 1 | u8::from(y == max_y); + seen |= 1 << corner; + } + (seen == 0b1111).then_some([min_x, min_y, max_x, max_y]) + } +} + +/// The array dimensions and its three coordinates in GDSII record order. +pub(crate) struct ArrayPlacement { + pub counts: [u16; 2], + pub points: [[i32; 2]; 3], +} + +impl ArrayPlacement { + pub fn decode<'a>(colrow: &'a [u8], xy: &'a [u8]) -> Result { + let (_, mut counts) = COLROW::read_data(colrow, colrow.len() as u16)?; + let columns = counts.next().unwrap(); + let rows = counts.next().unwrap(); + if columns <= 0 || rows <= 0 { + return Err(( + colrow, + ErrType::Failed("AREF column and row counts must be positive".to_string()), + )); + } + if xy.len() != 24 { + return Err(( + xy, + ErrType::Failed("AREF needs exactly three XY coordinate pairs".to_string()), + )); + } + let mut points = XY::points(xy); + Ok(Self { + counts: [columns as u16, rows as u16], + points: std::array::from_fn(|_| points.next().unwrap()), + }) + } } pub struct SNAME; @@ -192,30 +272,6 @@ impl Record for SNAME { fn expected_size() -> Option { None } } -pub struct COLROW; -impl Record for COLROW { - fn tag() -> u16 { RTAG_COLROW } - fn expected_size() -> Option { Some(4) } -} - -pub struct NODE; -impl Record for NODE { - fn tag() -> u16 { RTAG_NODE } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXTTYPE; -impl Record for TEXTTYPE { - fn tag() -> u16 { RTAG_TEXTTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PRESENTATION; -impl Record for PRESENTATION { - fn tag() -> u16 { RTAG_PRESENTATION } - fn expected_size() -> Option { Some(2) } -} - pub struct SPACING; impl Record for SPACING { fn tag() -> u16 { RTAG_SPACING } @@ -228,24 +284,6 @@ impl Record for STRING { fn expected_size() -> Option { None } } -pub struct STRANS; -impl Record for STRANS { - fn tag() -> u16 { RTAG_STRANS } - fn expected_size() -> Option { Some(2) } -} - -pub struct MAG; -impl Record for MAG { - fn tag() -> u16 { RTAG_MAG } - fn expected_size() -> Option { Some(8) } -} - -pub struct ANGLE; -impl Record for ANGLE { - fn tag() -> u16 { RTAG_ANGLE } - fn expected_size() -> Option { Some(8) } -} - pub struct UINTEGER; impl Record for UINTEGER { fn tag() -> u16 { RTAG_UINTEGER } @@ -286,18 +324,6 @@ impl Record for FONTS { } } -pub struct PATHTYPE; -impl Record for PATHTYPE { - fn tag() -> u16 { RTAG_PATHTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct GENERATIONS; -impl Record for GENERATIONS { - fn tag() -> u16 { RTAG_GENERATIONS } - fn expected_size() -> Option { Some(2) } -} - pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } @@ -324,12 +350,6 @@ impl Record for STRTYPE { fn expected_size() -> Option { None } } -pub struct ELFLAGS; -impl Record for ELFLAGS { - fn tag() -> u16 { RTAG_ELFLAGS } - fn expected_size() -> Option { Some(2) } -} - pub struct ELKEY; impl Record for ELKEY { fn tag() -> u16 { RTAG_ELKEY } @@ -348,54 +368,12 @@ impl Record for LINKKEYS { fn expected_size() -> Option { Some(2) } } -pub struct NODETYPE; -impl Record for NODETYPE { - fn tag() -> u16 { RTAG_NODETYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PROPATTR; -impl Record for PROPATTR { - fn tag() -> u16 { RTAG_PROPATTR } - fn expected_size() -> Option { Some(2) } -} - pub struct PROPVALUE; impl Record for PROPVALUE { fn tag() -> u16 { RTAG_PROPVALUE } fn expected_size() -> Option { None } } -pub struct BOX; -impl Record for BOX { - fn tag() -> u16 { RTAG_BOX } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOXTYPE; -impl Record for BOXTYPE { - fn tag() -> u16 { RTAG_BOXTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PLEX; -impl Record for PLEX { - fn tag() -> u16 { RTAG_PLEX } - fn expected_size() -> Option { Some(4) } -} - -pub struct BGNEXTN; -impl Record for BGNEXTN { - fn tag() -> u16 { RTAG_BGNEXTN } - fn expected_size() -> Option { Some(4) } -} - -pub struct ENDEXTN; -impl Record for ENDEXTN { - fn tag() -> u16 { RTAG_ENDEXTN } - fn expected_size() -> Option { Some(4) } -} - pub struct TAPENUM; impl Record for TAPENUM { fn tag() -> u16 { RTAG_TAPENUM } @@ -408,12 +386,6 @@ impl Record for TAPECODE { fn expected_size() -> Option { Some(2 * 6) } } -pub struct STRCLASS; -impl Record for STRCLASS { - fn tag() -> u16 { RTAG_STRCLASS } - fn expected_size() -> Option { Some(2) } -} - pub struct RESERVED; impl Record for RESERVED { fn tag() -> u16 { RTAG_RESERVED } @@ -432,13 +404,6 @@ impl Record for MASK { fn expected_size() -> Option { None } } -/// End of MASKS records -pub struct ENDMASKS; -impl Record for ENDMASKS { - fn tag() -> u16 { RTAG_ENDMASKS } - fn expected_size() -> Option { Some(0) } -} - pub struct LIBDIRSIZE; impl Record for LIBDIRSIZE { fn tag() -> u16 { RTAG_LIBDIRSIZE } diff --git a/src/review.rs b/src/review.rs index 9bb17a8..9f18daa 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1,7 +1,11 @@ //! Deterministic GDS object review without the normalization performed by the //! Arrow reader. Raw payloads borrow the file; only polygon keys are copied. -use crate::{basic::decode_real8, record::RecordHeader, records::*}; +use crate::{ + basic::{ErrType, decode_name, decode_real8, encode_real8, unpad_ascii}, + record::{Record as _, RecordFrame}, + records::*, +}; use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet}, @@ -162,12 +166,56 @@ impl Object<'_> { fn format_object<'a>( out: &mut fmt::Formatter<'_>, kind: u16, - fields: impl IntoIterator, + layer: Option, + fields: impl Iterator + Clone, properties: impl IntoIterator, ) -> fmt::Result { - out.write_str(kind_name(kind))?; + let field = |tag| { + fields + .clone() + .find_map(|(t, data)| (t == tag).then_some(data)) + }; + let rectangle = if is_polygon(kind) { + field(RTAG_XY).and_then(XY::closed_rectangle) + } else { + None + }; + let target = if matches!(kind, RTAG_SREF | RTAG_AREF) { + field(RTAG_SNAME) + } else { + None + }; + let dimensions = if kind == RTAG_AREF { + field(RTAG_COLROW).and_then(array_dimensions) + } else { + None + }; + if let Some((layer, datatype)) = layer { + write!(out, "{layer}/{datatype}: ")?; + } + out.write_str(object_name(kind, rectangle.is_some()))?; + if let Some(target) = target { + write!(out, " {}", quoted(target))?; + } + if let Some([columns, rows]) = dimensions { + write!(out, " ({columns} × {rows})")?; + } for (tag, value) in fields { - write!(out, " {}={}", tag_name(tag), format_data(tag, value))?; + if (layer.is_some() && matches_layer_field(kind, tag)) + || (tag == RTAG_SNAME && target.is_some()) + || (tag == RTAG_COLROW && dimensions.is_some()) + { + continue; + } + if tag == RTAG_XY && value.len().is_multiple_of(8) { + if let Some([xmin, ymin, xmax, ymax]) = rectangle { + write!(out, " ({xmin}, {ymin}) to ({xmax}, {ymax})")?; + } else { + write_points(out, value)?; + } + } else { + write!(out, " {}={}", tag_name(tag), format_exact_value(tag, value))?; + } } for (attribute, value) in properties { write!(out, " property[{attribute}]={}", quoted(value))?; @@ -175,11 +223,36 @@ fn format_object<'a>( Ok(()) } +fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> { + let [column_hi, column_lo, row_hi, row_lo]: [u8; 4] = data.try_into().ok()?; + Some([ + u16::from_be_bytes([column_hi, column_lo]), + u16::from_be_bytes([row_hi, row_lo]), + ]) +} + +fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result { + let mut points = XY::points(data); + if points.len() == 1 { + let [x, y] = points.next().unwrap(); + return write!(out, " xy=({x}, {y})"); + } + out.write_str(" xy=[")?; + for (index, [x, y]) in points.enumerate() { + if index != 0 { + out.write_str(if index % 4 == 0 { ",\n " } else { ", " })?; + } + write!(out, "({x}, {y})")?; + } + out.write_str("]") +} + impl fmt::Display for Object<'_> { fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { format_object( out, self.kind, + self.layer, self.key .fields .iter() @@ -217,10 +290,6 @@ fn error(offset: usize, message: impl std::fmt::Display) -> String { format!("at byte {offset}: {message}") } -fn string(data: &[u8], offset: usize) -> Result { - decoded_string(unpad(data), offset) -} - fn decoded_string(data: &[u8], offset: usize) -> Result { if data.contains(&0) { return Err(error(offset, "name or metadata contains an embedded NUL")); @@ -230,29 +299,16 @@ fn decoded_string(data: &[u8], offset: usize) -> Result { .map_err(|_| error(offset, "name or metadata is not UTF-8")) } -fn unpad(data: &[u8]) -> &[u8] { - if data.last() == Some(&0) { - &data[..data.len() - 1] - } else { - data - } -} - fn integer(data: &[u8]) -> u16 { u16::from_be_bytes(data.try_into().unwrap()) } -fn points(data: &[u8]) -> Vec<[i32; 2]> { - data.as_chunks::<8>() - .0 - .iter() - .map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| { - [ - i32::from_be_bytes([x0, x1, x2, x3]), - i32::from_be_bytes([y0, y1, y2, y3]), - ] - }) - .collect() +fn coordinate_bounds(points: impl IntoIterator) -> Bounds { + let mut points = points.into_iter(); + let [x, y] = points.next().unwrap(); + points.fold([x, y, x, y], |[xmin, ymin, xmax, ymax], [x, y]| { + [xmin.min(x), ymin.min(y), xmax.max(x), ymax.max(y)] + }) } // Booth's least-rotation algorithm avoids quadratic work on repeated vertices. @@ -282,8 +338,7 @@ fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { (0..n).map(|k| points[(start + k) % n]).collect() } -fn canonical_ring(data: &[u8]) -> Vec { - let mut ring = points(data); +fn canonical_ring(mut ring: Vec<[i32; 2]>) -> Vec { ring.pop(); // The caller has checked closure. let forward = least_rotation(&ring); ring.reverse(); @@ -297,37 +352,30 @@ fn canonical_ring(data: &[u8]) -> Vec { } fn validate_payload(record: &Record<'_>) -> Result<(), String> { - let size = record.data.len(); - let valid = match record.tag { - RTAG_BGNLIB | RTAG_BGNSTR => size == 24, - RTAG_UNITS => size == 16, - RTAG_HEADER | RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE - | RTAG_PROPATTR | RTAG_PATHTYPE | RTAG_STRANS | RTAG_PRESENTATION | RTAG_ELFLAGS - | RTAG_STRCLASS | RTAG_GENERATIONS => size == 2, - RTAG_WIDTH | RTAG_BGNEXTN | RTAG_ENDEXTN | RTAG_PLEX | RTAG_COLROW => size == 4, - RTAG_MAG | RTAG_ANGLE => size == 8, - RTAG_XY => size.is_multiple_of(8), - RTAG_ENDLIB | RTAG_ENDSTR | RTAG_ENDEL | RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF - | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX | RTAG_ENDMASKS => size == 0, - _ => true, - }; - if !valid { - return Err(error( + let size = record.data.len() as u16; + check_payload_size(record.tag, size).map_err(|message| { + error( record.offset, - format!("invalid payload size {size} for {}", tag_name(record.tag)), - )); - } - if record.tag == RTAG_UNITS - && record.data.as_chunks::<8>().0.iter().any(|&bytes| { - let value = decode_real8(u64::from_be_bytes(bytes)); - !value.is_finite() || value <= 0.0 - }) - { - return Err(error(record.offset, "UNITS must be positive finite values")); + format!( + "invalid payload size {size} for {}: {message}", + tag_name(record.tag) + ), + ) + })?; + if record.tag == RTAG_UNITS { + UNITS::read_data(record.data, size) + .map_err(|(_, reason)| payload_error(record.offset, reason))?; } Ok(()) } +fn payload_error(offset: usize, reason: ErrType) -> String { + match reason { + ErrType::Failed(message) => error(offset, message), + ErrType::Incomplete(_) => error(offset, "truncated record payload"), + } +} + fn is_element(tag: u16) -> bool { matches!( tag, @@ -335,6 +383,23 @@ fn is_element(tag: u16) -> bool { ) } +fn is_polygon(kind: u16) -> bool { + matches!(kind, RTAG_BOUNDARY | RTAG_BOX) +} + +fn layer_type_tag(kind: u16) -> u16 { + match kind { + RTAG_TEXT => RTAG_TEXTTYPE, + RTAG_NODE => RTAG_NODETYPE, + RTAG_BOX => RTAG_BOXTYPE, + _ => RTAG_DATATYPE, + } +} + +fn matches_layer_field(kind: u16, tag: u16) -> bool { + tag == RTAG_LAYER || tag == layer_type_tag(kind) +} + fn allowed_field(kind: u16, tag: u16) -> bool { if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) { return true; @@ -372,6 +437,7 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result = XY::points(record.data).collect(); if p.len() < 4 || p.first() != p.last() { return Err(error( record.offset, "polygon must have at least three vertices and be closed", )); } - data = Cow::Owned(canonical_ring(record.data)); + data = Cow::Owned(canonical_ring(p)); } fields.push((record.tag, data)); i += 1; @@ -424,7 +494,7 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result p.len() == 1, - RTAG_AREF => p.len() == 3, - RTAG_PATH => p.len() >= 2, - RTAG_NODE => !p.is_empty(), - RTAG_BOX => p.len() == 5, - _ => true, - }; - if !valid_count { - return Err(error(first.offset, "invalid coordinate count")); - } - object.bounds = Some([ - p.iter().map(|p| p[0]).min().unwrap(), - p.iter().map(|p| p[1]).min().unwrap(), - p.iter().map(|p| p[0]).max().unwrap(), - p.iter().map(|p| p[1]).max().unwrap(), - ]); + object.bounds = Some(if first.tag == RTAG_AREF { + let counts = object + .field(RTAG_COLROW) + .ok_or_else(|| error(first.offset, "array has no COLROW"))?; + let placement = ArrayPlacement::decode(counts, xy) + .map_err(|(_, reason)| payload_error(first.offset, reason))?; + let [columns, rows] = placement.counts; + object.multiplicity = u64::from(columns) * u64::from(rows); + coordinate_bounds(placement.points) + } else { + let points = XY::points(xy); + let valid_count = match first.tag { + RTAG_TEXT | RTAG_SREF => points.len() == 1, + RTAG_PATH => points.len() >= 2, + RTAG_NODE => points.len() > 0, + RTAG_BOX => points.len() == 5, + _ => true, + }; + if !valid_count { + return Err(error(first.offset, "invalid coordinate count")); + } + coordinate_bounds(points) + }); if matches!(first.tag, RTAG_SREF | RTAG_AREF) { - object.target = Some(decoded_string( - object - .field(RTAG_SNAME) - .ok_or_else(|| error(first.offset, "reference has no SNAME"))?, - first.offset, - )?); - if first.tag == RTAG_AREF { - let counts = object - .field(RTAG_COLROW) - .ok_or_else(|| error(first.offset, "array has no COLROW"))?; - let a = i16::from_be_bytes(counts[..2].try_into().unwrap()); - let b = i16::from_be_bytes(counts[2..].try_into().unwrap()); - if a <= 0 || b <= 0 { - return Err(error(first.offset, "array counts must be positive")); - } - object.multiplicity = a as u64 * b as u64; + if object.target.is_none() { + return Err(error(first.offset, "reference has no SNAME")); } } else { - let dtype = match first.tag { - RTAG_TEXT => RTAG_TEXTTYPE, - RTAG_BOX => RTAG_BOXTYPE, - RTAG_NODE => RTAG_NODETYPE, - _ => RTAG_DATATYPE, - }; + let dtype = layer_type_tag(first.tag); object.layer = Some(( integer( object @@ -511,28 +567,25 @@ fn read_records(input: &[u8]) -> Result>, String> { } let mut records = Vec::new(); - let mut offset = 0; - while offset < input.len() { - let (payload, header) = RecordHeader::read(&input[offset..]) - .map_err(|(_, e)| error(offset, format!("invalid record header: {e:?}")))?; - let size = header.data_size as usize; - if !(size + 4).is_multiple_of(2) { - return Err(error(offset, "odd record length")); - } - let data = payload - .get(..size) - .ok_or_else(|| error(offset, "truncated record"))?; - let record = Record { + let mut remaining = input; + while !remaining.is_empty() { + let offset = input.len() - remaining.len(); + let (next, frame) = RecordFrame::read(remaining).map_err(|(_, reason)| match reason { + ErrType::Incomplete(_) => error(offset, "truncated record header"), + ErrType::Failed(message) => error(offset, message), + })?; + records.push(Record { offset, - tag: header.tag, - data, - }; - offset += size + 4; - let ended = record.tag == RTAG_ENDLIB; - records.push(record); - if ended { - if input[offset..].iter().any(|b| *b != 0) { - return Err(error(offset, "nonzero data after ENDLIB")); + tag: frame.tag, + data: frame.data, + }); + remaining = next; + if frame.tag == RTAG_ENDLIB { + if remaining.iter().any(|b| *b != 0) { + return Err(error( + input.len() - remaining.len(), + "nonzero data after ENDLIB", + )); } break; } @@ -602,8 +655,12 @@ impl<'a> Document<'a> { } doc.header.push(( record.tag, - if record.tag & 0xff == 6 { - unpad(record.data) + if record.tag == RTAG_LIBNAME { + decode_name(record.data) + .map_err(|message| error(record.offset, message))? + .as_bytes() + } else if record.tag & 0xff == 6 { + unpad_ascii(record.data) } else { record.data }, @@ -622,7 +679,9 @@ impl<'a> Document<'a> { .get(i) .filter(|r| r.tag == RTAG_STRNAME) .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; - let name = string(name_record.data, name_record.offset)?; + let name = decode_name(name_record.data) + .map_err(|message| error(name_record.offset, message))? + .to_owned(); if name.is_empty() { return Err(error(name_record.offset, "empty cell name")); } @@ -779,23 +838,58 @@ impl<'a> Document<'a> { } pub fn canonical_dump(&self) -> String { + if !self.present { + return "library absent\n".to_string(); + } + let version_index = self + .header + .iter() + .position(|(tag, _)| *tag == RTAG_HEADER) + .unwrap(); + let header = |tag| self.header.iter().find(|(t, _)| *t == tag).unwrap().1; + let (user, meters) = header(RTAG_UNITS).split_at(8); + let version = i16::from_be_bytes(self.header[version_index].1.try_into().unwrap()); let mut out = format!( - "profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n", - self.scope.description() + "library {} (GDSII {version}, {} user units/dbu, {} m/dbu)\n", + quoted(header(RTAG_LIBNAME)), + format_exact_value(RTAG_UNITS, user), + format_exact_value(RTAG_UNITS, meters) ); - for (tag, data) in &self.header { - writeln!(out, "{}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + self.scope.write_restrictions(&mut out); + for (index, (tag, data)) in self.header.iter().enumerate() { + if index != version_index && !matches!(*tag, RTAG_LIBNAME | RTAG_UNITS) { + writeln!(out, "{}={}", tag_name(*tag), format_exact_value(*tag, data)).unwrap(); + } } for ((layer, dtype), name) in &self.layer_names { - writeln!(out, "layer {layer}/{dtype} name={name:?}").unwrap(); + writeln!(out, "{layer}/{dtype}: name {name:?}").unwrap(); } for (name, cell) in &self.cells { writeln!(out, "cell {name:?}").unwrap(); for (tag, data) in &cell.attributes { - writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + writeln!( + out, + " {}={}", + tag_name(*tag), + format_exact_value(*tag, data) + ) + .unwrap(); } - for object in cell.objects.iter().filter(|o| !o.metadata) { - writeln!(out, " {object}").unwrap(); + let mut objects = cell + .objects + .iter() + .filter(|object| !object.metadata) + .peekable(); + while let Some(object) = objects.next() { + let mut count = 1; + while objects.next_if(|next| next.key == object.key).is_some() { + count += 1; + } + if count == 1 { + writeln!(out, " {object}").unwrap(); + } else { + writeln!(out, " {count}x {object}").unwrap(); + } } } for diagnostic in &self.diagnostics { @@ -827,7 +921,7 @@ fn format_records(records: &[Record<'_>]) -> String { format_data( record.tag, if record.tag & 0xff == 6 { - unpad(record.data) + unpad_ascii(record.data) } else { record.data } @@ -876,6 +970,7 @@ impl fmt::Display for ObjectChange { format_object( out, self.kind, + self.layer, self.fields .iter() .map(|field| (field.tag, field.data.as_slice())), @@ -895,21 +990,17 @@ impl From<&Object<'_>> for ObjectChange { layer: object.layer, bounds: object.bounds, target: object.target.clone(), - origin: object.field(RTAG_XY).map(|data| { - [ - i32::from_be_bytes(data[..4].try_into().unwrap()), - i32::from_be_bytes(data[4..8].try_into().unwrap()), - ] - }), + origin: object + .field(RTAG_XY) + .and_then(|data| XY::points(data).next()), width: object .field(RTAG_WIDTH) .map(|data| i32::from_be_bytes(data.try_into().unwrap())), - counts: object.target.as_ref().map(|_| { + counts: object.target.as_ref().and_then(|_| { if object.kind == RTAG_AREF { - let data = object.field(RTAG_COLROW).unwrap(); - [integer(&data[..2]), integer(&data[2..])] + object.field(RTAG_COLROW).and_then(array_dimensions) } else { - [1, 1] + Some([1, 1]) } }), fields: owned_fields( @@ -986,19 +1077,34 @@ pub enum LibraryChange { impl fmt::Display for LibraryChange { fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Presence { old, new } => { - let label = |present| if present { "present" } else { "absent" }; - write!(out, "presence: {:?} -> {:?}", [label(*old)], [label(*new)]) - } + Self::Presence { old, new } => out.write_str(match (old, new) { + (false, true) => "added", + (true, false) => "removed", + (true, true) => "present", + (false, false) => "absent", + }), Self::Record { tag, old, new } => { - write!(out, "{}: ", tag_name(*tag))?; - out.debug_list() - .entries(old.iter().map(|data| format_data(*tag, data))) - .finish()?; - out.write_str(" -> ")?; - out.debug_list() - .entries(new.iter().map(|data| format_data(*tag, data))) - .finish() + let values = if old.len() <= 1 && new.len() <= 1 { + format_field_change( + *tag, + old.first().map(Vec::as_slice), + new.first().map(Vec::as_slice), + ) + } else { + let list = |items: &[Vec]| { + if items.is_empty() { + return "absent".to_string(); + } + let values = items + .iter() + .map(|data| format_exact_value(*tag, data)) + .collect::>() + .join(", "); + format!("[{values}]") + }; + format!("{} -> {}", list(old), list(new)) + }; + write!(out, "{} {values}", tag_name(*tag)) } } } @@ -1199,10 +1305,50 @@ impl Comparison { } pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { - let mut out = format!("profile {PROFILE}\n{}\n", self.old.scope.description()); - if show.cell.is_some() || show.layer.is_some() { - writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap(); + let (removed, added) = self.cells.iter().fold((0, 0), |(removed, added), cell| { + (removed + cell.removed.len(), added + cell.added.len()) + }); + let total = removed + added; + let mut out = match self.outcome() { + Outcome::Equal => "Equal", + Outcome::Different => "Different", + Outcome::Incomplete { .. } => "Incomplete", + Outcome::Failed => "Failed", } + .to_string(); + if !self.cells.is_empty() { + let cells = if self.cells.len() == 1 { + "cell" + } else { + "cells" + }; + write!(out, " · {} changed {cells}", self.cells.len()).unwrap(); + } + if total != 0 { + write!(out, " · {removed} removed, {added} added").unwrap(); + } + for (count, label) in [ + (self.library_changes.len(), "library change"), + (self.layer_names.len(), "layer name change"), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural}").unwrap(); + } + } + out.push('\n'); + self.old.scope.write_restrictions(&mut out); + if show.cell.is_some() || show.layer.is_some() { + out.push_str("Display filter"); + if let Some(cell) = &show.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = show.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.push('\n'); + } + let notice_start = out.len(); for change in &self.library_changes { writeln!(out, "library {change}").unwrap(); } @@ -1215,111 +1361,141 @@ impl Comparison { } ) }) { - out.push_str("UNITS changed; coordinates remain in each file's database units\n"); + out.push_str("Coordinates use each file's database units\n"); } for change in &self.layer_names { + let name = |value: Option<&str>| { + value.map_or_else(|| "absent".to_string(), |value| format!("{value:?}")) + }; writeln!( out, - "layer {}/{} name: {:?} -> {:?}", - change.layer.0, change.layer.1, change.old, change.new + "{}/{}: name {} -> {}", + change.layer.0, + change.layer.1, + name(change.old.as_deref()), + name(change.new.as_deref()) ) .unwrap(); } - let changed_layers: BTreeSet<_> = self - .cells - .iter() - .filter(|cell| show.cell.as_ref().is_none_or(|name| name == &cell.name)) - .flat_map(|cell| cell.removed.iter().chain(&cell.added)) - .filter_map(|object| object.layer) - .filter(|layer| show.layer.is_none_or(|selected| selected == *layer)) - .collect(); - for layer in changed_layers { - if let Some(name) = self.layer_legend.get(&layer) { - writeln!(out, "layer {}/{} name={name:?}", layer.0, layer.1).unwrap(); - } - } - let total: usize = self - .cells - .iter() - .map(|c| c.removed.len() + c.added.len()) - .sum(); + let body_start = out.len(); + let mut shown_layers = BTreeSet::new(); + let mut selected = 0; let mut shown: usize = 0; + let mut hidden_cell_changes = 0; + let visible = + |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); + let presence = |action, count| { + let objects = if count == 1 { "object" } else { "objects" }; + Cow::Owned(format!(" ({action}, {count} {objects})")) + }; for cell in &self.cells { + let note = match (&cell.old, &cell.new) { + (None, Some(_)) => presence("added", cell.added.len()), + (Some(_), None) => presence("removed", cell.removed.len()), + _ if cell.attributes_changed() => Cow::Borrowed(" (attributes changed)"), + _ => Cow::Borrowed(""), + }; if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + hidden_cell_changes += usize::from(!note.is_empty()); continue; } - writeln!( - out, - "cell {:?} old@{:?} new@{:?}{}", - cell.name, - cell.old.as_ref().map(|revision| revision.offset), - cell.new.as_ref().map(|revision| revision.offset), - if cell.attributes_changed() { - " [cell/header changed]" - } else { - "" - } - ) - .unwrap(); + let cell_start = out.len(); + let previously_shown = shown; + writeln!(out, "cell {:?}{note}", cell.name).unwrap(); + let whole_cell = cell.old.is_none() || cell.new.is_none(); + let mut polygons = changes::PolygonChanges::default(); let mut paired_old = BTreeSet::new(); let mut paired_new = BTreeSet::new(); for replacement in &cell.replacements { let a = &cell.removed[replacement.removed]; let b = &cell.added[replacement.added]; - if shown.saturating_add(2) <= max_objects - && show - .layer - .is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer)) - { - writeln!(out, " ~ {}", changes::explain(a, b)).unwrap(); - writeln!(out, " - @{} bbox={:?} {}", a.offset, a.bounds, a).unwrap(); - writeln!(out, " + @{} bbox={:?} {}", b.offset, b.bounds, b).unwrap(); + if max_objects - shown >= 2 && visible(a) && visible(b) { + if is_polygon(a.kind) { + polygons.replacement(a, b); + } else { + writeln!(out, " ~ {}", changes::replacement_summary(a, b)).unwrap(); + } + shown_layers.extend(a.layer); paired_old.insert(replacement.removed); paired_new.insert(replacement.added); shown += 2; } } - for (sign, objects, paired) in [ - ("-", &cell.removed, &paired_old), - ("+", &cell.added, &paired_new), + for (removed, objects, paired) in [ + (true, &cell.removed, &paired_old), + (false, &cell.added, &paired_new), ] { + let sign = if removed { "-" } else { "+" }; + let mut references = BTreeMap::<&str, changes::ReferenceCounts>::new(); for (index, object) in objects.iter().enumerate() { - if paired.contains(&index) { + if !visible(object) { continue; } - if show.layer.is_some_and(|layer| object.layer != Some(layer)) { - continue; - } - if shown < max_objects { - writeln!( - out, - " {sign} @{} bbox={:?} {}", - object.offset, object.bounds, object - ) - .unwrap(); + selected += 1; + if !paired.contains(&index) && shown < max_objects { + if is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if whole_cell && let Some(target) = object.target.as_deref() { + references.entry(target).or_default().add(object); + } else { + writeln!(out, " {sign} {object}").unwrap(); + } + shown_layers.extend(object.layer); shown += 1; } } + for (target, counts) in references { + counts.write(&mut out, sign, target); + } + } + polygons.write(&mut out); + if shown == previously_shown && note.is_empty() { + out.truncate(cell_start); } } - writeln!( - out, - "{} changed cells; {shown} of {total} object changes shown; {} layer-name changes", - self.cells.len(), - self.layer_names.len() - ) - .unwrap(); - // General comparison errors precede revision-specific errors in text. - for diagnostic in self - .diagnostics - .iter() - .filter(|d| d.side.is_none()) - .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) - { - writeln!(out, "INCOMPLETE: {diagnostic}").unwrap(); + let mut legend = String::new(); + for layer in shown_layers { + if let Some(name) = self.layer_legend.get(&layer) { + writeln!(legend, "{}/{}: name {name:?}", layer.0, layer.1).unwrap(); + } } - if self.outcome().is_equal() { - out.push_str("equal under this profile\n"); + out.insert_str(body_start, &legend); + let mut notice = String::new(); + if shown < total { + let mut hidden = Vec::new(); + if total > selected { + hidden.push(format!("{} filtered out", total - selected)); + } + if selected > shown { + hidden.push(format!("{} over the limit", selected - shown)); + } + writeln!( + notice, + "Showing {shown} of {total} object changes ({})", + hidden.join(", ") + ) + .unwrap(); + } + if hidden_cell_changes != 0 { + let plural = if hidden_cell_changes == 1 { "" } else { "s" }; + writeln!( + notice, + "{hidden_cell_changes} cell-level change{plural} filtered out" + ) + .unwrap(); + } + out.insert_str(notice_start, ¬ice); + // General comparison errors precede revision-specific errors in text. + if !self.diagnostics.is_empty() { + out.push_str("Diagnostics\n"); + for diagnostic in self + .diagnostics + .iter() + .filter(|d| d.side.is_none()) + .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) + { + writeln!(out, " {diagnostic}").unwrap(); + } } out } @@ -1499,7 +1675,10 @@ fn render_hierarchy( .enumerate() .map(|(i, n)| (n, i)) .collect(); - let mut out = format!("flowchart TD\n %% {}\n", new.scope.description()); + let mut out = "flowchart TD\n".to_string(); + if changes.is_none() && new.scope != Scope::default() { + writeln!(out, " %% {}", new.scope.description()).unwrap(); + } for (name, id) in &ids { let label = format!("{name}{}", status(name)); let label = label @@ -1520,15 +1699,18 @@ fn render_hierarchy( } out } else { - let mut out = format!("{}\n", new.scope.description()); + let mut out = "Hierarchy\n".to_string(); + if changes.is_none() { + new.scope.write_restrictions(&mut out); + } for (name, depth, parent, seen, limited) in rows { - let count = parent - .map(|p| edge_label(&p, &name)) - .unwrap_or_else(|| "x1".to_string()); + write!(out, "{}{name:?}", " ".repeat(depth)).unwrap(); + if let Some(parent) = parent { + write!(out, " {}", edge_label(&parent, &name)).unwrap(); + } writeln!( out, - "{}{name:?} {count}{}{}{}", - " ".repeat(depth), + "{}{}{}", status(&name), if seen { " [shown above]" } else { "" }, if limited { " [children omitted]" } else { "" } @@ -1553,6 +1735,57 @@ fn quoted(data: &[u8]) -> String { out } +fn real8_words(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .copied() + .map(u64::from_be_bytes) +} + +fn format_real8(data: &[u8], show_bytes: bool) -> String { + let mut values = real8_words(data).map(decode_real8); + let mut out = if values.len() == 1 { + format!("{:?}", values.next().unwrap()) + } else { + format!("{:?}", values.collect::>()) + }; + if show_bytes { + write!(out, " raw={}", hex(data)).unwrap(); + } + out +} + +fn is_real8(tag: u16, data: &[u8]) -> bool { + tag & 0xff == 5 && data.len().is_multiple_of(8) +} + +fn format_exact_value(tag: u16, data: &[u8]) -> String { + if is_real8(tag, data) { + let show_bytes = + real8_words(data).any(|bits| encode_real8(decode_real8(bits)).ok() != Some(bits)); + format_real8(data, show_bytes) + } else { + format_data(tag, data) + } +} + +fn format_field_change(tag: u16, before: Option<&[u8]>, after: Option<&[u8]>) -> String { + let show_bytes = before.zip(after).is_some_and(|(a, b)| { + is_real8(tag, a) + && is_real8(tag, b) + && real8_words(a) + .zip(real8_words(b)) + .any(|(a, b)| a != b && decode_real8(a).to_bits() == decode_real8(b).to_bits()) + }); + let format_value = |data: Option<&[u8]>| match data { + Some(data) if is_real8(tag, data) => format_real8(data, show_bytes), + Some(data) => format_data(tag, data), + None => "absent".to_string(), + }; + format!("{} -> {}", format_value(before), format_value(after)) +} + fn format_data(tag: u16, data: &[u8]) -> String { if matches!( tag, @@ -1583,11 +1816,7 @@ fn format_data(tag: u16, data: &[u8]) -> String { ), 5 if data.len().is_multiple_of(8) => format!( "{:?} raw={}", - data.as_chunks::<8>() - .0 - .iter() - .map(|&b| decode_real8(u64::from_be_bytes(b))) - .collect::>(), + real8_words(data).map(decode_real8).collect::>(), hex(data) ), 6 => quoted(data), @@ -1595,6 +1824,20 @@ fn format_data(tag: u16, data: &[u8]) -> String { } } +fn object_name(kind: u16, rectangular: bool) -> &'static str { + match kind { + RTAG_BOUNDARY if rectangular => "rectangle", + RTAG_BOUNDARY => "polygon", + RTAG_BOX => "box", + RTAG_PATH => "path", + RTAG_SREF => "reference", + RTAG_AREF => "array reference", + RTAG_TEXT => "label", + RTAG_NODE => "node", + _ => kind_name(kind), + } +} + fn hex(data: &[u8]) -> String { let mut out = "0x".to_string(); for byte in data { diff --git a/src/review/changes.rs b/src/review/changes.rs index 7c8d3b8..56c9484 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -74,54 +74,305 @@ pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec String { - let mut parts = vec![format!( - "{} modified{}", - kind_name(old.kind), - old.target - .as_ref() - .map(|t| format!(" target={t:?}")) - .unwrap_or_default() - )]; - if old.target.is_some() - && old.origin != new.origin - && let (Some(a), Some(b)) = (old.origin, new.origin) - { - parts.push(format!( - "origin {a:?} -> {b:?}, delta=({}, {}) dbu", - i64::from(b[0]) - i64::from(a[0]), - i64::from(b[1]) - i64::from(a[1]) - )); +pub(super) fn field_changes<'a>( + old: &'a ObjectChange, + new: &'a ObjectChange, +) -> impl Iterator, Option<&'a [u8]>)> { + let mut fields = BTreeMap::, Option<&[u8]>)>::new(); + for field in &old.fields { + fields.entry(field.tag).or_default().0 = Some(&field.data); } - if old.counts != new.counts { - let counts = |c: Option<[u16; 2]>| { - c.map(|[a, b]| format!("{a} x {b}")) - .unwrap_or_else(|| "absent".to_string()) - }; - parts.push(format!( - "columns/rows {} -> {}", - counts(old.counts), - counts(new.counts) - )); + for field in &new.fields { + fields.entry(field.tag).or_default().1 = Some(&field.data); } - if old.width != new.width { - let width = |n: Option| { - n.map(|n| n.to_string()) - .unwrap_or_else(|| "absent".to_string()) + fields + .into_iter() + .filter_map(|(tag, (before, after))| (before != after).then_some((tag, before, after))) +} + +fn polygon_heading(object: &ObjectChange, rectangular: bool) -> String { + object_heading(object, object_name(object.kind, rectangular)) +} + +fn object_heading(object: &ObjectChange, kind: &str) -> String { + match object.layer { + Some((layer, datatype)) => format!("{layer}/{datatype}: {kind}"), + None => kind.to_string(), + } +} + +fn polygon_xy(object: &ObjectChange) -> &[u8] { + object + .fields + .iter() + .find(|field| field.tag == RTAG_XY) + .map_or(&[], |field| field.data.as_slice()) +} + +fn vertex_count(xy: &[u8]) -> usize { + XY::points(xy).len().saturating_sub(1) +} + +fn rectangle_size(xy: &[u8]) -> Option<[i64; 2]> { + let [xmin, ymin, xmax, ymax] = XY::closed_rectangle(xy)?; + Some([ + i64::from(xmax) - i64::from(xmin), + i64::from(ymax) - i64::from(ymin), + ]) +} + +fn translation(before: &[u8], after: &[u8]) -> Option<[i64; 2]> { + let mut old = XY::points(before); + let mut new = XY::points(after); + if old.len() != new.len() { + return None; + } + let (Some(a), Some(b)) = (old.next(), new.next()) else { + return None; + }; + let displacement = |[x0, y0]: [i32; 2], [x1, y1]: [i32; 2]| { + [i64::from(x1) - i64::from(x0), i64::from(y1) - i64::from(y0)] + }; + let shift = displacement(a, b); + (shift != [0, 0] && old.zip(new).all(|(a, b)| displacement(a, b) == shift)).then_some(shift) +} + +#[derive(Default)] +struct ShapeCounts { + rectangles: usize, + polygons: usize, + boxes: usize, +} + +impl ShapeCounts { + fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) { + let is_rectangle = + |object: &ObjectChange| XY::closed_rectangle(polygon_xy(object)).is_some(); + match object.kind { + RTAG_BOX => self.boxes += 1, + _ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => { + self.rectangles += 1 + } + _ => self.polygons += 1, + } + } +} + +#[derive(Default)] +struct LayerPolygonChanges { + changed: ShapeCounts, + moved: ShapeCounts, + removed: ShapeCounts, + added: ShapeCounts, +} + +#[derive(Default)] +pub(super) struct PolygonChanges(BTreeMap, LayerPolygonChanges>); + +impl PolygonChanges { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) { + let counts = self.0.entry(old.layer).or_default(); + let moved = old.properties == new.properties + && translation(polygon_xy(old), polygon_xy(new)).is_some() + && field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY); + let shapes = if moved { + &mut counts.moved + } else { + &mut counts.changed }; - parts.push(format!( - "width {} -> {} dbu", - width(old.width), - width(new.width) - )); - } - if old.bounds == new.bounds { - parts.push("coordinate bounds unchanged".to_string()); + shapes.add(old, Some(new)); + } + + pub fn unmatched(&mut self, object: &ObjectChange, removed: bool) { + let counts = self.0.entry(object.layer).or_default(); + let shapes = if removed { + &mut counts.removed + } else { + &mut counts.added + }; + shapes.add(object, None); + } + + pub fn write(&self, out: &mut String) { + for (layer, counts) in &self.0 { + if let Some((layer, datatype)) = layer { + write!(out, " {layer}/{datatype}: ").unwrap(); + } else { + out.push_str(" unknown layer: "); + } + let mut separator = ""; + for (action, shapes) in [ + ("changed", &counts.changed), + ("moved", &counts.moved), + ("removed", &counts.removed), + ("added", &counts.added), + ] { + let terms = [ + (shapes.rectangles, "rectangle", "rectangles"), + (shapes.polygons, "polygon", "polygons"), + (shapes.boxes, "box", "boxes"), + ] + .into_iter() + .filter(|(count, _, _)| *count != 0) + .map(|(count, singular, plural)| { + let shape = if count == 1 { singular } else { plural }; + format!("{count} {shape}") + }) + .collect::>(); + if !terms.is_empty() { + write!(out, "{separator}{} {action}", terms.join(", ")).unwrap(); + separator = " · "; + } + } + out.push('\n'); + } + } +} + +#[derive(Default)] +pub(super) struct ReferenceCounts { + singles: usize, + arrays: usize, + placements: u128, +} + +impl ReferenceCounts { + pub fn add(&mut self, object: &ObjectChange) { + if object.kind == RTAG_AREF { + self.arrays += 1; + if let Some([columns, rows]) = object.counts { + self.placements += u128::from(columns) * u128::from(rows); + } + } else { + self.singles += 1; + self.placements += 1; + } + } + + pub fn write(&self, out: &mut String, sign: &str, target: &str) { + write!(out, " {sign} ").unwrap(); + if self.singles > 0 { + let suffix = if self.singles == 1 { "" } else { "s" }; + write!(out, "{} reference{suffix}", self.singles).unwrap(); + } + if self.arrays > 0 { + if self.singles > 0 { + out.push_str(", "); + } + let suffix = if self.arrays == 1 { "" } else { "s" }; + write!(out, "{} array reference{suffix}", self.arrays).unwrap(); + } + write!(out, " to {target:?}").unwrap(); + if self.arrays > 0 { + let suffix = if self.placements == 1 { "" } else { "s" }; + write!(out, " ({} placement{suffix})", self.placements).unwrap(); + } + out.push('\n'); + } +} + +fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { + let before = polygon_xy(old); + let after = polygon_xy(new); + let old_size = rectangle_size(before); + let new_size = rectangle_size(after); + let mut out = polygon_heading(old, old_size.is_some() && new_size.is_some()); + if before == after { + out.push_str(" modified"); + } else { + match (old_size, new_size) { + (Some(a), Some(b)) if a == b => out.push_str(" moved"), + (Some([w0, h0]), Some([w1, h1])) => { + write!(out, " dimensions changed ({w0} x {h0} -> {w1} x {h1} dbu)").unwrap(); + } + _ if translation(before, after).is_some() => out.push_str(" moved"), + _ => { + out.push_str(" vertices changed"); + let (a, b) = (vertex_count(before), vertex_count(after)); + if a != b { + write!(out, " ({a} -> {b} vertices)").unwrap(); + } + } + } + } + out +} + +pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { + let polygon = is_polygon(old.kind); + let mut out = if polygon { + polygon_replacement(old, new) } else { - parts.push(format!( - "coordinate bounds {:?} -> {:?}", - old.bounds, new.bounds - )); + let mut heading = object_heading(old, object_name(old.kind, false)); + if let Some(target) = &old.target { + write!(heading, " {target:?}").unwrap(); + } + heading + }; + + let mut separator = if polygon { "\n " } else { " " }; + for (tag, before, after) in field_changes(old, new) { + if polygon && tag == RTAG_XY { + continue; + } + let change = match (tag, before, after) { + (RTAG_XY, Some(a), Some(b)) => { + if let Some([dx, dy]) = translation(a, b) { + format!("moved by ({dx}, {dy}) dbu") + } else if matches!(old.kind, RTAG_PATH | RTAG_NODE) { + let (a, b) = (XY::points(a).len(), XY::points(b).len()); + if a == b { + "vertices changed".to_string() + } else { + format!("vertices changed ({a} -> {b} vertices)") + } + } else { + let points = |data: &[u8]| XY::points(data).collect::>(); + format!("coordinates {:?} -> {:?} dbu", points(a), points(b)) + } + } + (RTAG_COLROW, _, _) if old.kind == RTAG_AREF => { + let counts = |dimensions: Option<[u16; 2]>| { + dimensions.map_or_else(|| "absent".to_string(), |[a, b]| format!("{a} x {b}")) + }; + format!( + "columns/rows {} -> {}", + counts(old.counts), + counts(new.counts) + ) + } + (RTAG_WIDTH, _, _) => { + let width = + |n: Option| n.map_or_else(|| "absent".to_string(), |n| n.to_string()); + format!("width {} -> {} dbu", width(old.width), width(new.width)) + } + _ => { + let name = if tag == RTAG_STRING { + "text".to_string() + } else { + tag_name(tag) + }; + format!("{name} {}", format_field_change(tag, before, after)) + } + }; + write!(out, "{separator}{change}").unwrap(); + separator = "\n "; + } + if old.properties != new.properties { + let properties = |items: &[(u16, Vec)]| { + items + .iter() + .map(|(attribute, value)| format!("{attribute}={}", quoted(value))) + .collect::>() + .join(", ") + }; + write!( + out, + "{separator}properties [{}] -> [{}]", + properties(&old.properties), + properties(&new.properties) + ) + .unwrap(); } - parts.join("; ") + out } diff --git a/src/review/report.rs b/src/review/report.rs index aa1a0c1..228ac12 100644 --- a/src/review/report.rs +++ b/src/review/report.rs @@ -97,7 +97,7 @@ fn diagnostic(value: &Diagnostic) -> Value { fn field(tag: u16, data: &[u8]) -> Value { let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { - ("xy", json!(points(data))) + ("xy", json!(XY::points(data).collect::>())) } else if matches!( tag, RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE @@ -185,19 +185,12 @@ fn object(value: &ObjectChange, id: &str, side: &str) -> Value { } fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { - let a: BTreeMap<_, _> = old.fields.iter().map(|f| (f.tag, &f.data)).collect(); - let b: BTreeMap<_, _> = new.fields.iter().map(|f| (f.tag, &f.data)).collect(); - a.keys() - .chain(b.keys()) - .copied() - .collect::>() - .into_iter() - .filter(|tag| a.get(tag) != b.get(tag)) - .map(|tag| { + changes::field_changes(old, new) + .map(|(tag, before, after)| { json!({ "tag": tag, - "before": a.get(&tag).map(|data| field(tag, data)), - "after": b.get(&tag).map(|data| field(tag, data)) + "before": before.map(|data| field(tag, data)), + "after": after.map(|data| field(tag, data)) }) }) .collect() @@ -292,7 +285,7 @@ impl Comparison { let b = format!("c{cell_index}:new:{}", r.added); (shown.contains(&a) && shown.contains(&b)).then(|| json!({ "removed": a, "added": b, "match_rule": r.match_rule, - "explanation": changes::explain(&cell.removed[r.removed], &cell.added[r.added]), + "explanation": changes::replacement_summary(&cell.removed[r.removed], &cell.added[r.added]), "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties })) diff --git a/src/review/selection.rs b/src/review/selection.rs index 820352c..db52a59 100644 --- a/src/review/selection.rs +++ b/src/review/selection.rs @@ -17,6 +17,23 @@ pub struct DisplayFilter { } impl Scope { + pub(super) fn write_restrictions(&self, out: &mut String) { + if self == &Self::default() { + return; + } + out.push_str("Scope"); + if let Some(top) = &self.top { + write!(out, " top={top:?}").unwrap(); + } + if let Some(cell) = &self.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = self.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.push('\n'); + } + pub fn description(&self) -> String { let top = self .top diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json index ba0797f..c5c5fe7 100644 --- a/tests/golden/report-geometry.json +++ b/tests/golden/report-geometry.json @@ -160,7 +160,7 @@ "replacements": [ { "added": "c0:new:0", - "explanation": "BOUNDARY modified; coordinate bounds unchanged", + "explanation": "1/0: polygon vertices changed (4 -> 3 vertices)", "field_changes": [ { "after": { diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt index 9ef3ac6..8546367 100644 --- a/tests/golden/review-canonical.txt +++ b/tests/golden/review-canonical.txt @@ -1,9 +1,4 @@ -profile pheon-structural-v1 -scope top=all cell=all layer=all -coordinates: integer database units; bounds: coordinate envelopes -HEADER=[600] -LIBNAME="test" -UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54 -layer 1/0 name="WG" +library "test" (GDSII 600, 0.001 user units/dbu, 1e-9 m/dbu) +1/0: name "WG" cell "CHIP" - BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] + 1/0: rectangle (0, 0) to (10, 10) diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt index bb10311..eeb7497 100644 --- a/tests/golden/review-diff.txt +++ b/tests/golden/review-diff.txt @@ -1,7 +1,3 @@ -profile pheon-structural-v1 -scope top=all cell=all layer=all -cell "CHIP" old@Some(62) new@Some(62) - ~ BOUNDARY modified; coordinate bounds unchanged - - @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] - + @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 0, 0, 0] -1 changed cells; 2 of 2 object changes shown; 0 layer-name changes +Different · 1 changed cell · 1 removed, 1 added +cell "CHIP" + 1/0: 1 polygon changed diff --git a/tests/golden/review-hierarchy.mmd b/tests/golden/review-hierarchy.mmd index dcc963b..86cffd6 100644 --- a/tests/golden/review-hierarchy.mmd +++ b/tests/golden/review-hierarchy.mmd @@ -1,5 +1,4 @@ flowchart TD - %% scope top=all cell=all layer=all n0["CHIP [affected]"] n1["leaf [cell changed]"] n2["left [affected]"] diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt index fa40d79..cdab505 100644 --- a/tests/golden/review-hierarchy.txt +++ b/tests/golden/review-hierarchy.txt @@ -1,5 +1,5 @@ -scope top=all cell=all layer=all -"CHIP" x1 [affected] +Hierarchy +"CHIP" [affected] "left" x1 [affected] "leaf" x2 [cell changed] "right" x1 [affected] diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs index eca1116..be3ede4 100644 --- a/tests/parser_errors.rs +++ b/tests/parser_errors.rs @@ -2,12 +2,14 @@ mod common; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use pheon::{ - basic::ErrType, + basic::{ErrType, encode_real8}, last_error_message, library::{read_library, read_selected_library, scan_library}, read_bytes, read_cells_bytes, record::RecordHeader, - records, scan_bytes, + records, + review::{Comparison, Document, dump_records}, + scan_bytes, }; use std::ffi::CStr; @@ -126,6 +128,136 @@ fn invalid_utf8_returns_located_parse_errors_in_each_reader() { } } +#[test] +fn extra_name_padding_is_ignored_by_comparison_and_arrow_readers() { + let (mut bytes, (start, end)) = fixture(); + let target = payload_range(&bytes, records::RTAG_SNAME); + bytes[target].copy_from_slice(b"leaf"); + bytes.truncate(end as usize); + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let mut padded = bytes.clone(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let payload = payload_range(&padded, tag); + let mut data = padded[payload.clone()].to_vec(); + data.extend_from_slice(&[0; 2]); + let mut record = Vec::new(); + common::record(&mut record, tag, &data); + padded.splice(payload.start - 4..payload.end, record); + } + let baseline = Document::parse(&bytes).unwrap(); + let document = Document::parse(&padded).unwrap(); + assert_eq!(document.cells["töp"].objects.len(), 2); + assert!( + Comparison::between(&baseline, &document) + .outcome() + .is_equal() + ); + + let full = read_library(&padded).unwrap().1; + let scan = scan_library(&padded).unwrap().1; + let ranges = [(start + 2, end + 6), (end + 6, padded.len() as u64 - 4)]; + let selected = read_selected_library(&padded, &ranges).unwrap().1; + let expected = read_library(&bytes).unwrap().1; + for actual in [&full, &scan, &selected] { + for field in ["lib_name", "cell_names"] { + assert_eq!(actual.column_by_name(field), expected.column_by_name(field)); + } + } + assert_eq!(&full, &selected); +} + +#[test] +fn interior_nuls_in_names_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0; + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + let message = "name contains an embedded NUL"; + check_read_errors(&malformed, range, offset, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", payload.start - 4) + ); + } +} + +#[test] +fn malformed_record_frames_are_rejected_at_the_same_offset() { + let (bytes, range) = fixture(); + // STRING also exercises framing in the Arrow scan's geometry skip path. + for tag in [ + records::RTAG_STRNAME, + records::RTAG_XY, + records::RTAG_STRING, + ] { + let start = payload_range(&bytes, tag).start - 4; + for (size, message) in [ + (2_u16, "Record size 2 is smaller than its header"), + (5, "odd record length"), + (u16::MAX - 1, "truncated record"), + ] { + let mut malformed = bytes.clone(); + malformed[start..start + 2].copy_from_slice(&size.to_be_bytes()); + check_read_errors(&malformed, range, start, message, true); + assert_eq!( + dump_records(&malformed).unwrap_err(), + format!("at byte {start}: {message}") + ); + } + } +} + +#[test] +fn nonpositive_unit_scales_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + let units = payload_range(&bytes, records::RTAG_UNITS); + let message = "UNITS must be positive finite values"; + for field in [units.start, units.start + 8] { + for scale in [0.0, -1.0] { + let mut malformed = bytes.clone(); + malformed[field..field + 8] + .copy_from_slice(&encode_real8(scale).unwrap().to_be_bytes()); + check_read_errors(&malformed, range, units.start, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", units.start - 4) + ); + } + } +} + +#[test] +fn coordinate_payloads_require_whole_pairs() { + let (mut bytes, (start, end)) = fixture(); + let after_text = payload_range(&bytes, records::RTAG_TEXT).end; + let xy = payload_range(&bytes[after_text..], records::RTAG_XY); + let payload_start = after_text + xy.start; + let mut replacement = Vec::new(); + common::record(&mut replacement, records::RTAG_XY, &[0; 12]); + bytes.splice(payload_start - 4..after_text + xy.end, replacement); + + let message = "XY payload must contain whole coordinate pairs"; + check_read_errors(&bytes, (start, end + 4), payload_start, message, false); + assert!(Document::parse(&bytes).unwrap_err().contains(message)); +} + #[test] fn large_invalid_utf8_strings_do_not_dump_their_contents() { let (mut bytes, range) = fixture(); diff --git a/tests/review.rs b/tests/review.rs index 979c391..02ffb3b 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -52,10 +52,14 @@ fn gzip(data: &[u8]) -> Vec { } fn polygon(points: &[[i32; 2]]) -> Vec { + ring(RTAG_BOUNDARY, RTAG_DATATYPE, 1, points) +} + +fn ring(kind: u16, type_tag: u16, layer: u16, points: &[[i32; 2]]) -> Vec { let mut bytes = Vec::new(); - record(&mut bytes, RTAG_BOUNDARY, &[]); - record(&mut bytes, RTAG_LAYER, &1_u16.to_be_bytes()); - record(&mut bytes, RTAG_DATATYPE, &0_u16.to_be_bytes()); + record(&mut bytes, kind, &[]); + record(&mut bytes, RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, type_tag, &0_u16.to_be_bytes()); xy(&mut bytes, points); record(&mut bytes, RTAG_ENDEL, &[]); bytes @@ -312,9 +316,7 @@ fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { let doc = Document::parse(&bytes).unwrap(); let dump = doc.canonical_dump(); assert!( - dump.ends_with( - "cell \"CHIP\"\n BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0]\n" - ), + dump.ends_with("cell \"CHIP\"\n 1/0: rectangle (0, 0) to (10, 10)\n"), "{dump}" ); assert!(!dump.contains("BGNSTR")); @@ -323,6 +325,44 @@ fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { .contains("0000000000 0002 HEADER size=6 [600]") ); assert!(doc.record_dump().contains("BGNSTR")); + let notched = polygon(&[[0, 0], [0, 3], [2, 3], [2, 2], [3, 2], [3, 0], [0, 0]]); + let bytes = layout(&[("CHIP", vec![notched])]); + let dump = Document::parse(&bytes).unwrap().canonical_dump(); + assert!( + dump.ends_with(concat!( + " 1/0: polygon xy=[(0, 0), (0, 3), (2, 3), (2, 2),\n", + " (3, 2), (3, 0), (0, 0)]\n", + )), + "{dump}" + ); +} + +#[test] +fn canonical_dump_counts_duplicates_by_normalized_object_within_each_cell() { + let test_layout = |copies: usize| { + let mut objects = vec![square(); copies - 1]; + objects.push(polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])); + objects.push(with_property(square(), "marked")); + layout(&[("A", objects), ("B", vec![square()])]) + }; + let a = test_layout(3); + let b = test_layout(2); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let dump = a.canonical_dump(); + assert!( + dump.ends_with(concat!( + "cell \"A\"\n", + " 3x 1/0: rectangle (0, 0) to (10, 10)\n", + " 1/0: rectangle (0, 0) to (10, 10) property[17]=\"marked\"\n", + "cell \"B\"\n", + " 1/0: rectangle (0, 0) to (10, 10)\n", + )), + "{dump}" + ); + assert!(b.canonical_dump().contains(" 2x 1/0: rectangle")); + assert_ne!(dump, b.canonical_dump()); + assert_eq!(Comparison::between(&a, &b).outcome().exit_code(), 1); } #[test] @@ -442,7 +482,9 @@ fn unknown_records_never_produce_a_successful_equality_claim() { assert!(!diff.outcome().is_equal()); assert_eq!(diff.outcome().exit_code(), 2); assert!(doc.record_dump().contains("7f00")); - assert!(diff.text(0).contains("INCOMPLETE")); + let text = diff.text(0); + assert!(text.starts_with("Incomplete\nDiagnostics\n")); + assert!(text.contains("unsupported record 0x7f00")); } #[test] @@ -455,8 +497,8 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { assert_eq!( diff.hierarchy(false, 8), concat!( - "scope top=all cell=all layer=all\n", - "\"CHIP\" x1 [affected]\n", + "Hierarchy\n", + "\"CHIP\" [affected]\n", " \"left\" x1 [affected]\n", " \"leaf\" x2 [cell changed]\n", " \"right\" x1 [affected]\n", @@ -510,7 +552,7 @@ fn same_bounds_changes_and_truncated_reports_remain_differences() { diff.cells[0].removed[0].bounds, diff.cells[0].added[0].bounds ); - assert!(diff.text(0).contains("0 of 2 object changes shown")); + assert!(diff.text(0).contains("Showing 0 of 2 object changes")); assert_eq!(diff.outcome().exit_code(), 1); assert!(diff.cells[0].removed[0].offset > 0); } @@ -539,7 +581,10 @@ fn unit_changes_report_values_without_claiming_geometry_equivalence() { .. }] )); - assert!(diff.text(0).contains("UNITS changed")); + let text = diff.text(0); + assert!(text.contains("library UNITS [0.001, 1e-9] -> [0.001, 2e-9]")); + assert!(text.contains("Coordinates use each file's database units")); + assert!(!text.contains("raw=")); assert!(diff.cells.is_empty()); } @@ -669,6 +714,7 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); let equal = command().arg("diff").arg(&old).arg(&old).output().unwrap(); assert_eq!(equal.status.code(), Some(0)); + assert_eq!(equal.stdout, b"Equal\n"); let diff = command() .arg("diff") .arg(&old) @@ -678,8 +724,9 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { .unwrap(); assert_eq!(diff.status.code(), Some(1)); let text = String::from_utf8(diff.stdout).unwrap(); - assert!(text.contains("0 of 2 object changes shown")); - assert!(text.contains("```mermaid\nflowchart TD")); + assert!(text.contains("Showing 0 of 2 object changes")); + assert!(text.contains("Hierarchy\n```mermaid\nflowchart TD")); + assert!(!text.contains("scope top=all")); let dump = command() .arg("dump") .arg(&old) @@ -921,7 +968,7 @@ fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { ["CHIP", "leaf"] ); assert_eq!(Comparison::between(&old, &new).cells.len(), 1); - assert!(old.canonical_dump().contains("scope top=\"CHIP\"")); + assert!(old.canonical_dump().contains("Scope top=\"CHIP\"\n")); let cell = Scope { cell: Some("CHIP".to_string()), ..Scope::default() @@ -1058,11 +1105,99 @@ fn display_filters_never_change_equality_or_totals() { layer: Some((42, 0)), }, ); - assert!(text.contains("display-only")); - assert!(text.contains("0 of 2 object changes shown")); + assert!(text.starts_with("Different · 1 changed cell · 1 removed, 1 added\n")); + assert!(text.contains("Display filter layer=42/0\n")); + assert!(text.contains("Showing 0 of 2 object changes (2 filtered out)")); + assert!(!text.contains("cell \"CHIP\"")); assert_eq!(diff.outcome().exit_code(), 1); } +#[test] +fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { + use pheon::review::selection::DisplayFilter; + + let mut attributes = Vec::new(); + record(&mut attributes, RTAG_STRCLASS, &[0, 1]); + let a = layout(&[ + ("changed", vec![square()]), + ("other", vec![square()]), + ("gone", vec![square(), triangle()]), + ("header", vec![]), + ]); + let b = layout(&[ + ("changed", vec![triangle()]), + ("other", vec![triangle()]), + ("new", vec![square()]), + ("empty", vec![]), + ("header", vec![attributes]), + ]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text(0); + assert!(text.contains("cell \"gone\" (removed, 2 objects)\n")); + assert!(text.contains("cell \"new\" (added, 1 object)\n")); + assert!(text.contains("cell \"empty\" (added, 0 objects)\n")); + assert!(text.contains("cell \"header\" (attributes changed)\n")); + assert!(!text.contains("cell \"changed\"")); + assert!(text.contains("Showing 0 of 7 object changes (7 over the limit)")); + + let text = diff.text_filtered( + 1, + &DisplayFilter { + cell: Some("changed".to_string()), + layer: Some((1, 0)), + }, + ); + assert!(text.contains(concat!( + "Display filter cell=\"changed\" layer=1/0\n", + "Showing 1 of 7 object changes (5 filtered out, 1 over the limit)\n", + "4 cell-level changes filtered out\n", + "cell \"changed\"\n 1/0: 1 rectangle removed", + ))); + assert!(!text.contains("cell \"other\"")); + assert!(!text.contains("cell \"new\"")); +} + +#[test] +fn reference_summaries_group_whole_cells_and_keep_full_placements_in_dumps() { + let empty = layout(&[("leaf", vec![]), ("other", vec![])]); + let with_chip = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 0, 1), + reference("leaf", 5, 1), + reference("leaf", 2, 2), + reference("other", 10, 1), + ], + ), + ("leaf", vec![]), + ("other", vec![]), + ]); + let existing_chip = layout(&[("CHIP", vec![]), ("leaf", vec![]), ("other", vec![])]); + let empty = Document::parse(&empty).unwrap(); + let with_chip = Document::parse(&with_chip).unwrap(); + let existing_chip = Document::parse(&existing_chip).unwrap(); + + for (old, new, sign) in [(&empty, &with_chip, "+"), (&with_chip, &empty, "-")] { + let diff = Comparison::between(old, new); + let text = diff.text(100); + assert!(text.contains(&format!( + " {sign} 2 references, 1 array reference to \"leaf\" (4 placements)\n" + ))); + assert!(text.contains(&format!(" {sign} 1 reference to \"other\"\n"))); + assert!(!text.contains("xy=")); + let limited = diff.text(2); + assert!(limited.contains(&format!(" {sign} 2 references to \"leaf\"\n"))); + assert!(limited.contains("Showing 2 of 4 object changes (2 over the limit)")); + } + + let text = Comparison::between(&existing_chip, &with_chip).text(100); + assert!(text.contains(" + reference \"leaf\" xy=(0, 0)\n")); + let dump = with_chip.canonical_dump(); + assert!(dump.contains(" reference \"leaf\" xy=(5, 0)\n")); + assert!(dump.contains(" array reference \"leaf\" (2 × 1) xy=[(2, 0), (102, 0), (2, 0)]\n")); +} + #[test] fn git_external_driver_handles_changes_additions_deletions_and_errors() { let directory = std::env::temp_dir().join(format!( @@ -1130,7 +1265,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!( String::from_utf8(changed.stdout) .unwrap() - .contains("2 of 2 object changes shown") + .contains("Different · 1 changed cell · 1 removed, 1 added") ); fs::write(&path, b"invalid").unwrap(); let invalid = Command::new("git") @@ -1169,7 +1304,10 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let result = git(&["diff", "--cached", "--find-renames"]); let text = String::from_utf8(result.stdout).unwrap(); assert!(text.contains("renamed layout.gds"), "{text}"); - assert!(text.contains("2 of 2 object changes shown"), "{text}"); + assert!( + text.contains("Different · 1 changed cell · 1 removed, 1 added"), + "{text}" + ); git(&["commit", "-qm", "rename fixture"]); fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); fs::write(&renamed, shared(square())).unwrap(); @@ -1177,7 +1315,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!( String::from_utf8(result.stdout) .unwrap() - .contains("- BOUNDARY") + .contains("- 1/0: polygon") ); fs::remove_dir_all(directory).unwrap(); } @@ -1197,21 +1335,130 @@ fn replacement_explanations_are_unique_and_keep_full_evidence() { let diff = Comparison::between(&old, &new); assert_eq!(diff.cells[0].replacements.len(), 1); let text = diff.text(100); - assert!(text.contains(" ~ ")); - assert!(text.contains(" - @")); - assert!(text.contains(" + @")); match pair.name { - "moved reference" => assert!(text.contains("delta=(5, 0) dbu")), - "array count" => assert!(text.contains("columns/rows 2 x 1 -> 3 x 1")), + "moved reference" => { + assert!(text.contains(" ~ reference \"leaf\" moved by (5, 0) dbu\n")) + } + "array count" => { + assert!(text.contains(" ~ array reference \"leaf\" columns/rows 2 x 1 -> 3 x 1\n")) + } _ => assert_eq!(text, include_str!("golden/review-diff.txt")), } + assert!(!text.contains(" - ")); + assert!(!text.contains(" + ")); + assert!(!text.contains("bbox=")); assert!(!diff.text(0).contains(" ~ ")); - assert!(diff.text(1).contains("1 of 2 object changes shown")); + assert!(diff.text(1).contains("Showing 1 of 2 object changes")); } let a = layout(&[("CHIP", vec![square(), square()])]); let b = layout(&[("CHIP", vec![triangle(), triangle()])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); assert!(diff.cells[0].replacements.is_empty()); + let text = diff.text(100); + assert!(text.contains(" 1/0: 2 rectangles removed · 2 polygons added\n")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); +} + +#[test] +fn polygon_diff_reports_moves_and_other_changes_without_individual_geometry() { + let before = layout(&[("CHIP", vec![with_property(square(), "old")])]); + let before = Document::parse(&before).unwrap(); + for (shape, label) in [ + ( + polygon(&[[-5, 2], [5, 2], [5, 12], [-5, 12], [-5, 2]]), + "1 rectangle changed", + ), + ( + polygon(&[[0, 0], [20, 0], [20, 10], [0, 10], [0, 0]]), + "1 rectangle changed", + ), + ( + polygon(&[[5, 0], [10, 5], [5, 10], [0, 5], [5, 0]]), + "1 polygon changed", + ), + (square(), "1 rectangle changed"), + ] { + let after = layout(&[("CHIP", vec![with_property(shape, "new")])]); + let after = Document::parse(&after).unwrap(); + let diff = Comparison::between(&before, &after); + let text = diff.text(100); + assert!(text.contains(&format!(" 1/0: {label}\n")), "{text}"); + assert!(!text.contains("dbu")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); + } + + for (shape, shifted, name) in [ + ( + triangle(), + polygon(&[[2, -1], [12, -1], [2, 9], [2, -1]]), + "polygon", + ), + ( + square(), + polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]), + "rectangle", + ), + ] { + let before = layout(&[("CHIP", vec![shape])]); + let after = layout(&[("CHIP", vec![shifted])]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!(diff.text(100).contains(&format!(" 1/0: 1 {name} moved\n"))); + } +} + +#[test] +fn polygon_diff_groups_shapes_by_layer_across_dimensions() { + let shapes = [ + square(), + polygon(&[[30, 0], [50, 0], [50, 5], [30, 5], [30, 0]]), + polygon(&[[70, 0], [80, 0], [71, 9], [70, 0]]), + ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + ), + ]; + let layout_with_property = |value| { + let objects = shapes + .iter() + .map(|shape| with_property(shape.clone(), value)) + .collect(); + layout(&[("CHIP", objects)]) + }; + let before = layout_with_property("old"); + let after = layout_with_property("new"); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different · 1 changed cell · 4 removed, 4 added\n", + "cell \"CHIP\"\n", + " 1/0: 2 rectangles, 1 polygon changed\n", + " 2/0: 1 polygon changed\n", + ) + ); +} + +#[test] +fn polygon_diff_distinguishes_boxes_from_rectangular_boundaries() { + let before = layout(&[("CHIP", vec![square()])]); + let points = [[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]; + let after = layout(&[("CHIP", vec![ring(RTAG_BOX, RTAG_BOXTYPE, 1, &points)])]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + let text = diff.text(100); + assert!(text.contains(" 1/0: 1 rectangle removed · 1 box added\n")); } #[test] @@ -1231,7 +1478,21 @@ fn path_width_explanation_preserves_absent_and_zero() { let a = path(None); let b = path(Some(0)); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); - assert!(diff.text(100).contains("width absent -> 0 dbu")); + assert!( + diff.text(100) + .contains(" ~ 1/0: path width absent -> 0 dbu\n") + ); +} + +#[test] +fn label_replacements_show_only_changed_fields() { + let a = layout(&[("CHIP", vec![label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert_eq!( + diff.text(100), + "Different · 1 changed cell · 1 removed, 1 added\ncell \"CHIP\"\n ~ 2/0: label text \"old\" -> \"new\"\n" + ); } #[test] @@ -1250,8 +1511,20 @@ fn unchanged_layer_names_explain_geometry_changes_once() { let b = named(triangle()); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); let text = diff.text(100); - assert_eq!(text.matches("name=\"WG\"").count(), 1); + assert_eq!(text.matches("1/0: name \"WG\"").count(), 1); assert!(diff.layer_names.is_empty()); + + let names = corpus() + .into_iter() + .find(|case| case.name == "layer name only") + .unwrap(); + let old = Document::parse(&names.old).unwrap(); + let new = Document::parse(&names.new).unwrap(); + assert!( + Comparison::between(&old, &new) + .text(0) + .contains("1/0: name \"WG\" -> \"CORE:WG\"") + ); } #[test] @@ -1261,19 +1534,11 @@ fn library_presence_changes_distinguish_empty_and_absent_revisions() { let absent = doc.empty_like(); let added = Comparison::between(&absent, &doc); assert_eq!(added.outcome().exit_code(), 1); - assert!( - added - .text(0) - .contains("library presence: [\"absent\"] -> [\"present\"]\n") - ); - assert!(!added.text(0).contains("equal under")); + assert!(added.text(0).contains("library added\n")); + assert!(added.text(0).starts_with("Different · 1 library change\n")); let removed = Comparison::between(&doc, &absent); assert_eq!(removed.outcome().exit_code(), 1); - assert!( - removed - .text(0) - .contains("library presence: [\"present\"] -> [\"absent\"]\n") - ); + assert!(removed.text(0).contains("library removed\n")); } #[test] diff --git a/tests/review/reports.rs b/tests/review/reports.rs index bad49d9..26e796a 100644 --- a/tests/review/reports.rs +++ b/tests/review/reports.rs @@ -75,7 +75,7 @@ fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { assert_eq!(limited["hierarchy"], actual["hierarchy"]); assert_eq!(limited["summary"], actual["summary"]); assert_eq!(limited["exit_code"], 1); - assert!(comparison.text(0).contains("0 of 2 object changes shown")); + assert!(comparison.text(0).contains("Showing 0 of 2 object changes")); } #[test] @@ -118,9 +118,10 @@ fn comparison_diagnostics_capture_both_sides_and_scope_mismatch_once() { assert_eq!(report["diagnostics"][1]["side"], "new"); assert_eq!(report["diagnostics"][2]["code"], "scope_mismatch"); let text = comparison.text(0); - let scope = text.find("INCOMPLETE: comparison scopes differ").unwrap(); - let old = text.find("INCOMPLETE: old:").unwrap(); - let new = text.find("INCOMPLETE: new:").unwrap(); + assert!(text.starts_with("Incomplete\nScope cell=\"CHIP\"\nDiagnostics\n")); + let scope = text.find("\n comparison scopes differ").unwrap(); + let old = text.find("\n old:").unwrap(); + let new = text.find("\n new:").unwrap(); assert!(scope < old && old < new); } @@ -259,7 +260,7 @@ fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { let diff = Comparison::between(&doc, &doc); assert_eq!(diff.outcome().exit_code(), 2); assert!(!diff.outcome().is_equal()); - assert!(diff.text(0).contains("INCOMPLETE")); + assert!(diff.text(0).starts_with("Incomplete\n")); let json = diff.json_report(&inputs(), &ReportOptions::default()); assert_eq!(json["status"], "incomplete"); assert_eq!(json["result"], "unknown"); @@ -348,6 +349,21 @@ fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { b[within..within + 8].copy_from_slice(&0x4201000000000000_u64.to_be_bytes()); let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + let before = Document::parse(&a).unwrap(); + let after = Document::parse(&b).unwrap(); + assert!(before.canonical_dump().contains("mag=1.0 property")); + assert!( + after + .canonical_dump() + .contains("mag=1.0 raw=0x4201000000000000") + ); + let text = Comparison::between(&before, &after).text(100); + assert!(text.contains("mag 1.0 raw=0x4110000000000000 -> 1.0 raw=0x4201000000000000")); + let omitted = layout(&[("CHIP", vec![reference("leaf", 0, 1)]), ("leaf", vec![])]); + let omitted = Document::parse(&omitted).unwrap(); + let text = Comparison::between(&before, &omitted).text(100); + assert!(text.contains("mag 1.0 -> absent")); + assert!(!text.contains("raw=")); let r = report(&a, &b, &ReportOptions::default()); let replacement = &r["cell_changes"][0]["replacements"][0]; assert_eq!(replacement["match_rule"], "unique_kind_target"); @@ -587,11 +603,7 @@ fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { .output() .unwrap(); assert_eq!(output.status.code(), Some(0)); - assert!( - String::from_utf8(output.stdout) - .unwrap() - .starts_with("profile ") - ); + assert_eq!(output.stdout, b"Equal\n"); } #[test] diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs index aaef2ab..cd06348 100644 --- a/tests/selective_reads.rs +++ b/tests/selective_reads.rs @@ -122,11 +122,41 @@ fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { #[test] fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { + let check_array_error = |bytes: &[u8], message: &str| { + let ranges = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for result in [ + read_library(bytes), + scan_library(bytes), + read_selected_library(bytes, &ranges), + ] { + let (_, error) = result.unwrap_err(); + assert!(matches!(error, pheon::basic::ErrType::Failed(reason) + if reason.contains(message))); + } + assert!( + pheon::review::Document::parse(bytes) + .unwrap_err() + .contains(message) + ); + }; for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); - assert!(read_library(&bytes).is_err()); - assert!(scan_library(&bytes).is_err()); + check_array_error(&bytes, "AREF column and row counts must be positive"); } + + let mut bytes = aref_fixture([0, 0], [10, 0], [0, 10], [1, 2]); + let xy_offset = pheon::review::Document::parse(&bytes) + .unwrap() + .records + .iter() + .find(|record| record.tag == records::RTAG_XY) + .unwrap() + .offset; + let mut replacement = Vec::new(); + record(&mut replacement, records::RTAG_XY, &[0; 16]); + bytes.splice(xy_offset..xy_offset + 28, replacement); + check_array_error(&bytes, "AREF needs exactly three XY coordinate pairs"); + let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); assert!(read_library(&bytes).is_err()); assert!(scan_library(&bytes).is_err()); @@ -472,7 +502,7 @@ fn selections_support_subsets_reordering_and_empty_results() { #[test] fn id_tables_preserve_encounter_order_and_selection_local_ids() { let mut bytes = library_header(); - for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha", 9, 4)] { + for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha\0", 9, 4)] { record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); if name == "zeta" { From 8efeb04aabd37d3b4eadbc2b94ec6a0c336c2e7c Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 10:58:19 -0800 Subject: [PATCH 26/41] Simplify and speed up GDSII diffs Summarize added and removed cells, group references, and keep replacement pairs together at text display limits. Trim hierarchy labels and summarize long unmatched paths. Stream JSON object evidence and reduce parsing and comparison allocations. Parse large revisions concurrently and adaptively reuse matching cell bodies when most of the layout is shared. --- Cargo.lock | 1 + Cargo.toml | 1 + docs/cli.md | 35 +-- docs/git.md | 7 +- docs/native-api.md | 17 +- src/bin/pheon.rs | 126 +++++++--- src/records.rs | 15 +- src/review.rs | 403 ++++++++++++++++++++++-------- src/review/changes.rs | 38 ++- src/review/pair.rs | 218 ++++++++++++++++ src/review/report.rs | 176 ++++++++++--- tests/golden/review-diff.txt | 2 +- tests/golden/review-hierarchy.txt | 4 +- tests/review.rs | 230 ++++++++++++----- tests/review/reports.rs | 123 ++++++++- 15 files changed, 1136 insertions(+), 260 deletions(-) create mode 100644 src/review/pair.rs diff --git a/Cargo.lock b/Cargo.lock index 4220cfd..3c11424 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,6 +376,7 @@ dependencies = [ "arrow-array", "arrow-schema", "flate2", + "serde", "serde_json", ] diff --git a/Cargo.toml b/Cargo.toml index c40323a..11a6afe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ crate-type = ["cdylib", "rlib"] arrow-array = {version = "^54", features = ["ffi"]} arrow-schema = "^54" flate2 = "1" +serde = "1" serde_json = "1" diff --git a/docs/cli.md b/docs/cli.md index 8312883..ffee2fc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -46,13 +46,18 @@ pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 pheon diff before.gds after.gds --format json > report.json ``` -Text reports start with the result and group changes by cell. Added and removed -cell headings give the object count within the comparison scope. Each layer -lists counts of rectangles, polygons, and boxes changed, moved, added, or removed. -Added or removed cells also count references by target. -Other objects use `-` for removals, `+` for additions, and `~` for likely -replacements. Use `--format json` for details of individual objects and -`--hierarchy` to see which parent cells are affected. +Text reports start with the result and counts of modified, added, and removed +cells. Added and removed cells show their object counts and group references by +target. Modified cells show counts of rectangles, polygons, and boxes changed, +moved, added, or removed on each layer. Repeated reference additions or removals +to the same target also appear as a count. Object details use `-` for removals, +`+` for additions, and `~` for likely replacements. Unmatched paths with five +or more vertices show their width, vertex count, and vertex bounds in one line. +Use `--format json` for details of individual objects and `--hierarchy` to see +which parent cells are affected. + +For large unscoped diffs with substantial byte overlap, Pheon reuses parsed +contents of matching cells. Cell timestamps and file offsets can differ. | Exit code | Meaning | | --- | --- | @@ -95,14 +100,16 @@ pheon diff before.gds after.gds --hierarchy mermaid --depth 4 `--show-cell` and `--show-layer` filter displayed changes without changing the comparison, totals, or exit status. A layer display filter hides references without a numeric layer. `--max-changes N` limits displayed object occurrences. -A paired replacement uses two occurrences. The text report counts omissions -from filters and limits before listing cell changes. JSON includes all changed -objects unless the limit is explicit. +Text shows a replacement only when both objects fit within the limit. +A single omission line lists changes hidden by filters or the limit. JSON +includes all changed objects unless the limit is explicit. -The diff hierarchy marks directly changed cells, affected ancestors, and -unchanged targets of modified references. It includes references from both -revisions so removed branches remain visible. Counts represent direct placements; -arrays remain symbolic. `--depth` limits text diagrams. For a complete graph in +The diff hierarchy shows directly changed cells, affected ancestors, and targets +of modified references. It includes references from both revisions so removed +branches remain visible. Counts represent direct placements; +arrays remain symbolic. ASCII omits the section when there are no cell changes +and marks reference changes whose placement count stays the same. +`--depth` limits text diagrams. For a complete graph in JSON, use `--format json --hierarchy graph`; text depth and display filters do not trim that graph. diff --git a/docs/git.md b/docs/git.md index e1d0c17..d17aadb 100644 --- a/docs/git.md +++ b/docs/git.md @@ -23,9 +23,10 @@ Add this to `.git/config`, or use the same section in your user Git config: ``` Then run `git diff` as usual. The driver reads Git's temporary old and new files -and handles additions, deletions, and renames. Keep the final `--`: Git appends -its path and revision arguments after it. Put Pheon options before that marker, -for example `--top CHIP` or `--show-layer 1/0`. +and handles additions, deletions, and renames. The heading names the file and +indicates whether it was added or deleted. Keep the final `--` because Git appends +its path and revision arguments after it. Put Pheon options such as `--top CHIP` +or `--show-layer 1/0` before that marker. For an existing `diff=layout` attribute, put the configuration under `[diff "layout"]` instead. Attribute names only select a Git driver. diff --git a/docs/native-api.md b/docs/native-api.md index 5e9f9ab..f953d71 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -26,10 +26,19 @@ Layout-review APIs live in `pheon::review`, with layer-name helpers in `Comparison::between(&old, &new)` captures an owned result with revision metadata, changes, and structured diagnostics. Use `outcome()` for completeness, equality, -and exit status. Its `text`, `json_report`, and `hierarchy` renderers work after -the source documents and byte buffers are dropped. Standalone hierarchy inspection -uses `Document::hierarchy`. Reference counts are captured with the result; -union-graph and ancestor analysis run only when a hierarchy is requested. +and exit status. Its renderers work after the source documents and byte buffers +are dropped. `json_report` returns a JSON value, and `write_json_report` writes +object evidence incrementally to a writer. `Document::hierarchy` inspects a +single revision. A comparison captures reference counts and analyzes the union +graph and ancestors when a hierarchy is requested. + +`TextComparison::between` captures the data needed for text and hierarchy +reports. Layered objects in added and removed cells contribute to the cell's +object count. + +Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison +that can reuse matching cells. Its `compare` and `compare_text` methods return +the corresponding owned comparison. The `record`, `records`, and `basic` modules also provide low-level GDSII record writing. There is no high-level writer for the Arrow layout. REAL8 writing is diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 9fcc786..33a891e 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,7 +1,7 @@ use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ - Comparison, Diagnostic, Document, Outcome, Side, dump_records, + Comparison, Diagnostic, Document, Outcome, RevisionPair, Side, TextComparison, dump_records, names::{parse_layer, parse_layer_name, write_layer_names}, selection::{DisplayFilter, Scope}, }; @@ -12,6 +12,7 @@ use std::{ io::{self, Read, Write}, path::PathBuf, process::ExitCode, + thread, }; const HELP: &str = "Usage: @@ -116,17 +117,36 @@ fn stdout(output: &str) -> Result<(), String> { .map_err(|e| e.to_string()) } -fn json_stdout(report: &serde_json::Value) -> Result<(), String> { - let mut output = io::stdout().lock(); - serde_json::to_writer_pretty(&mut output, report).map_err(|e| e.to_string())?; - output.write_all(b"\n").map_err(|e| e.to_string()) +fn json_stdout( + write: impl FnOnce(&mut io::BufWriter>) -> serde_json::Result<()>, +) -> Result<(), String> { + let mut output = io::BufWriter::new(io::stdout().lock()); + write(&mut output).map_err(|e| e.to_string())?; + output + .write_all(b"\n") + .and_then(|()| output.flush()) + .map_err(|e| e.to_string()) } -fn prepare_comparison( +fn parse_revision(bytes: Option<&[u8]>, side: Side) -> Result>, Diagnostic> { + bytes + .map(|bytes| { + let mut document = Document::parse(bytes) + .map_err(|message| Diagnostic::new("invalid_input", message, Some(side)))?; + // Comparison uses the parsed cells and their source offsets. + document.records = Vec::new(); + Ok(document) + }) + .transpose() +} + +fn prepare_comparison( old_path: Option<&OsStr>, new_path: Option<&OsStr>, scope: &Scope, -) -> Result { + compare: impl FnOnce(&Document<'_>, &Document<'_>) -> T, + compare_pair: impl FnOnce(RevisionPair<'_>) -> T, +) -> Result { let read = |path: Option<&OsStr>, side| { path.map(|path| { read_input(path).map_err(|mut diagnostic| { @@ -138,16 +158,34 @@ fn prepare_comparison( }; let old_bytes = read(old_path, Side::Old)?; let new_bytes = read(new_path, Side::New)?; - let old = old_bytes + const PARALLEL_PARSE_MIN_BYTES: usize = 2 * 1024 * 1024; + let large_pair = old_bytes .as_deref() - .map(Document::parse) - .transpose() - .map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::Old)))?; - let new = new_bytes - .as_deref() - .map(Document::parse) - .transpose() - .map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::New)))?; + .zip(new_bytes.as_deref()) + .filter(|(old, new)| old.len().min(new.len()) >= PARALLEL_PARSE_MIN_BYTES); + if scope == &Scope::default() + && let Some((old, new)) = large_pair + { + return RevisionPair::parse(old, new).map(compare_pair); + } + let parse_old = || parse_revision(old_bytes.as_deref(), Side::Old); + let parse_new = || parse_revision(new_bytes.as_deref(), Side::New); + let parallel = if large_pair.is_some() + && thread::available_parallelism().is_ok_and(|count| count.get() > 1) + { + thread::scope(|scope| { + let worker = thread::Builder::new().spawn_scoped(scope, parse_old).ok()?; + let new = parse_new(); + Some((worker.join().unwrap(), new)) + }) + } else { + None + }; + // Report an old-side parse error first if both inputs are malformed. + let (old, new) = match parallel { + Some((old, new)) => (old?, new?), + None => (parse_old()?, parse_new()?), + }; let (old, new) = match (old, new) { (Some(old), Some(new)) => (old, new), (None, Some(new)) => (new.empty_like(), new), @@ -163,6 +201,9 @@ fn prepare_comparison( )); } }; + if scope == &Scope::default() { + return Ok(compare(&old, &new)); + } scope .validate_pair(&old, &new) .map_err(|message| Diagnostic::new("invalid_scope", message, None))?; @@ -172,7 +213,7 @@ fn prepare_comparison( let new = new .selected(scope, true) .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::New)))?; - Ok(Comparison::between(&old, &new)) + Ok(compare(&old, &new)) } fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { @@ -180,14 +221,26 @@ fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Resu old: Some(PathBuf::from(&paths[0])), new: Some(PathBuf::from(&paths[1])), }; - let (report, outcome) = match prepare_comparison(Some(&paths[0]), Some(&paths[1]), scope) { - Ok(diff) => (diff.json_report(&inputs, options), diff.outcome()), - Err(error) => ( - failure_report(&inputs, scope, options, &error), - Outcome::Failed, + let (result, outcome) = match prepare_comparison( + Some(&paths[0]), + Some(&paths[1]), + scope, + Comparison::between, + |pair| pair.compare(), + ) { + Ok(diff) => ( + json_stdout(|output| diff.write_json_report(output, &inputs, options)), + diff.outcome(), ), + Err(error) => { + let report = failure_report(&inputs, scope, options, &error); + ( + json_stdout(|output| serde_json::to_writer_pretty(output, &report)), + Outcome::Failed, + ) + } }; - if let Err(error) = json_stdout(&report) { + if let Err(error) = result { // Once output starts, never append a second report on a write failure. eprintln!("pheon: {error}"); return Ok(2); @@ -327,25 +380,30 @@ fn run(args: &[OsString]) -> Result { (!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()), (!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()), &scope, + TextComparison::between, + |pair| pair.compare_text(), ) .map_err(|error| error.message)?; let mut output = String::new(); if git { + let action = match (old_missing, new_missing) { + (true, false) => " added", + (false, true) => " deleted", + _ => "", + }; if paths.len() == 9 { - output.push_str(&format!("layout {:?} -> {:?}\n", paths[0], paths[7])); + output.push_str(&format!( + "layout {:?} -> {:?}{action}\n", + paths[0], paths[7] + )); } else { - output.push_str(&format!("layout {:?}\n", paths[0])); + output.push_str(&format!("layout {:?}{action}\n", paths[0])); } - if old_missing { - output.push_str("layout added\n"); - } else if new_missing { - output.push_str("layout deleted\n"); - } - if paths[3] != paths[6] { + if !old_missing && !new_missing && paths[3] != paths[6] { output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); } } - output.push_str(&diff.text_filtered(max_changes, &show)); + output.push_str(&diff.render_text(max_changes, &show, !old_missing && !new_missing)); if format != "none" { if format == "mermaid" { output.push_str("Hierarchy\n```mermaid\n"); @@ -402,7 +460,9 @@ fn main() -> ExitCode { &ReportOptions::default(), &Diagnostic::new("command_error", message, None), ); - if let Err(error) = json_stdout(&report) { + if let Err(error) = + json_stdout(|output| serde_json::to_writer_pretty(output, &report)) + { eprintln!("pheon: {error}"); } } else { diff --git a/src/records.rs b/src/records.rs index 705652e..3f6ba58 100644 --- a/src/records.rs +++ b/src/records.rs @@ -182,17 +182,20 @@ impl Record for XY { } impl XY { + pub(crate) fn point([x0, x1, x2, x3, y0, y1, y2, y3]: [u8; 8]) -> [i32; 2] { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + } + /// Iterate over a validated XY payload. pub(crate) fn points(data: &[u8]) -> impl ExactSizeIterator + '_ { data.as_chunks::<8>() .0 .iter() - .map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| { - [ - i32::from_be_bytes([x0, x1, x2, x3]), - i32::from_be_bytes([y0, y1, y2, y3]), - ] - }) + .copied() + .map(Self::point) } /// Bounds of a rectangle encoded as a closed XY ring. diff --git a/src/review.rs b/src/review.rs index 9f18daa..7c2ee16 100644 --- a/src/review.rs +++ b/src/review.rs @@ -14,10 +14,13 @@ use std::{ pub mod changes; pub mod names; +mod pair; pub mod report; pub mod selection; use changes::Replacement; +use pair::CellReuse; +pub use pair::RevisionPair; use selection::{DisplayFilter, Scope}; /// Input revision associated with a diagnostic. @@ -139,7 +142,7 @@ struct Key<'a> { properties: Vec<(u16, &'a [u8])>, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Object<'a> { pub offset: usize, pub end_offset: usize, @@ -231,6 +234,8 @@ fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> { ]) } +const POINTS_PER_LINE: usize = 4; + fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result { let mut points = XY::points(data); if points.len() == 1 { @@ -240,7 +245,11 @@ fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result { out.write_str(" xy=[")?; for (index, [x, y]) in points.enumerate() { if index != 0 { - out.write_str(if index % 4 == 0 { ",\n " } else { ", " })?; + out.write_str(if index % POINTS_PER_LINE == 0 { + ",\n " + } else { + ", " + })?; } write!(out, "({x}, {y})")?; } @@ -312,11 +321,10 @@ fn coordinate_bounds(points: impl IntoIterator) -> Bounds { } // Booth's least-rotation algorithm avoids quadratic work on repeated vertices. -fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { - let n = points.len(); +fn least_rotation(n: usize, point: impl Fn(usize) -> [i32; 2]) -> usize { let (mut i, mut j, mut k) = (0, 1, 0); while i < n && j < n && k < n { - match points[(i + k) % n].cmp(&points[(j + k) % n]) { + match point((i + k) % n).cmp(&point((j + k) % n)) { std::cmp::Ordering::Equal => k += 1, std::cmp::Ordering::Greater => { i += k + 1; @@ -334,21 +342,37 @@ fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { } } } - let start = i.min(j); - (0..n).map(|k| points[(start + k) % n]).collect() + i.min(j) } -fn canonical_ring(mut ring: Vec<[i32; 2]>) -> Vec { - ring.pop(); // The caller has checked closure. - let forward = least_rotation(&ring); - ring.reverse(); - let backward = least_rotation(&ring); - let mut ring = forward.min(backward); - ring.push(ring[0]); - ring.into_iter() - .flatten() - .flat_map(i32::to_be_bytes) - .collect() +fn canonical_ring(data: &[u8]) -> Option> { + let (points, _) = data.as_chunks::<8>(); + let (last, ring) = points.split_last()?; + if ring.len() < 3 || ring.first() != Some(last) { + return None; + } + let n = ring.len(); + let first = least_rotation(n, |index| XY::point(ring[index])); + let split = n - least_rotation(n, |index| XY::point(ring[n - 1 - index])); + let forward = ring[first..].iter().chain(&ring[..first]); + let backward = ring[..split].iter().rev().chain(ring[split..].iter().rev()); + let decode = |point: &[u8; 8]| XY::point(*point); + let use_forward = forward + .clone() + .map(decode) + .cmp(backward.clone().map(decode)) + .is_le(); + if use_forward && first == 0 { + return Some(Cow::Borrowed(data)); + } + let mut result = Vec::with_capacity(data.len()); + if use_forward { + result.extend(forward.flatten().copied()); + } else { + result.extend(backward.flatten().copied()); + } + result.extend_from_within(..8); + Some(Cow::Owned(result)) } fn validate_payload(record: &Record<'_>) -> Result<(), String> { @@ -474,14 +498,12 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result = XY::points(record.data).collect(); - if p.len() < 4 || p.first() != p.last() { - return Err(error( + data = canonical_ring(record.data).ok_or_else(|| { + error( record.offset, "polygon must have at least three vertices and be closed", - )); - } - data = Cow::Owned(canonical_ring(p)); + ) + })?; } fields.push((record.tag, data)); i += 1; @@ -556,6 +578,13 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result Result>, String> { + read_records_with(input, |_, _| {}) +} + +fn read_records_with<'a>( + input: &'a [u8], + mut observe: impl FnMut(usize, &Record<'a>), +) -> Result>, String> { if input.starts_with(b"\x1f\x8b") { return Err(error( 0, @@ -574,11 +603,13 @@ fn read_records(input: &[u8]) -> Result>, String> { ErrType::Incomplete(_) => error(offset, "truncated record header"), ErrType::Failed(message) => error(offset, message), })?; - records.push(Record { + let record = Record { offset, tag: frame.tag, data: frame.data, - }); + }; + observe(records.len(), &record); + records.push(record); remaining = next; if frame.tag == RTAG_ENDLIB { if remaining.iter().any(|b| *b != 0) { @@ -596,7 +627,16 @@ fn read_records(input: &[u8]) -> Result>, String> { impl<'a> Document<'a> { pub fn parse(input: &'a [u8]) -> Result { let records = read_records(input)?; - for record in &records { + let mut doc = Self::parse_records(&records, None)?; + doc.records = records; + Ok(doc) + } + + fn parse_records( + records: &[Record<'a>], + reuse: Option>, + ) -> Result { + for record in records { validate_payload(record)?; } if records.first().map(|r| r.tag) != Some(RTAG_HEADER) @@ -606,7 +646,7 @@ impl<'a> Document<'a> { return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string()); } let mut doc = Self { - records, + records: Vec::new(), cells: BTreeMap::new(), layer_names: BTreeMap::new(), diagnostics: Vec::new(), @@ -619,9 +659,9 @@ impl<'a> Document<'a> { let mut i = 2; let mut global_seen = BTreeSet::new(); let mut cells_started = false; - doc.header.push((RTAG_HEADER, doc.records[0].data)); - while i + 1 < doc.records.len() { - let record = &doc.records[i]; + doc.header.push((RTAG_HEADER, records[0].data)); + while i + 1 < records.len() { + let record = &records[i]; if record.tag != RTAG_BGNSTR { if cells_started { return Err(error( @@ -674,8 +714,7 @@ impl<'a> Document<'a> { cells_started = true; let offset = record.offset; i += 1; - let name_record = doc - .records + let name_record = records .get(i) .filter(|r| r.tag == RTAG_STRNAME) .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; @@ -685,6 +724,15 @@ impl<'a> Document<'a> { if name.is_empty() { return Err(error(name_record.offset, "empty cell name")); } + if let Some(reuse) = &reuse + && let Some((cell, end)) = reuse.cell(&name, offset, records) + { + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + i = end + 1; + continue; + } i += 1; let mut cell = Cell { offset, @@ -694,8 +742,7 @@ impl<'a> Document<'a> { issues: Vec::new(), }; loop { - let record = doc - .records + let record = records .get(i) .ok_or_else(|| error(offset, "missing ENDSTR"))?; if record.tag == RTAG_ENDSTR { @@ -706,19 +753,19 @@ impl<'a> Document<'a> { if is_element(record.tag) { let start = i; i += 1; - while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { - if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) - || is_element(doc.records[i].tag) + while records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { + if matches!(records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) + || is_element(records[i].tag) { - return Err(error(doc.records[i].offset, "missing ENDEL")); + return Err(error(records[i].offset, "missing ENDEL")); } i += 1; } - if i == doc.records.len() { + if i == records.len() { return Err(error(offset, "missing ENDEL")); } let mut issues = Vec::new(); - let mut parsed = object(&doc.records[start..=i], &mut issues)?; + let mut parsed = object(&records[start..=i], &mut issues)?; parsed.issues = issues; cell.objects.push(parsed); i += 1; @@ -820,16 +867,21 @@ impl<'a> Document<'a> { } pub fn edges(&self) -> BTreeMap<(String, String), u64> { - let mut edges = BTreeMap::new(); - for (name, cell) in &self.cells { - for object in &cell.objects { - if let Some(target) = &object.target { - let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64); - *count = count.saturating_add(object.multiplicity); + self.cells + .iter() + .flat_map(|(name, cell)| { + let mut counts = BTreeMap::new(); + for object in &cell.objects { + if let Some(target) = object.target.as_deref() { + let count = counts.entry(target).or_insert(0_u64); + *count = count.saturating_add(object.multiplicity); + } } - } - } - edges + counts + .into_iter() + .map(|(target, count)| ((name.clone(), target.to_owned()), count)) + }) + .collect() } pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { @@ -1150,8 +1202,62 @@ pub struct Comparison { pub diagnostics: Vec, } +/// A comparison for text and hierarchy reports. Layered objects in whole-cell +/// changes contribute to the cell's object count. +#[derive(Debug)] +pub struct TextComparison { + comparison: Comparison, + whole_cell_objects: BTreeMap, +} + +impl TextComparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new()) + } + + fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self { + let mut whole_cell_objects = BTreeMap::new(); + let comparison = Comparison::collect(old, new, reused_cells, Some(&mut whole_cell_objects)); + Self { + comparison, + whole_cell_objects, + } + } + + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.comparison.render_text_with_counts( + max_objects, + show, + show_library_presence, + &self.whole_cell_objects, + ) + } + + pub fn outcome(&self) -> Outcome { + self.comparison.outcome() + } + + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + self.comparison.hierarchy(mermaid, max_depth) + } +} + impl Comparison { pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new(), None) + } + + fn collect( + old: &Document<'_>, + new: &Document<'_>, + reused_cells: &BTreeSet<&str>, + mut whole_cell_objects: Option<&mut BTreeMap>, + ) -> Self { let mut result = Self { old: Revision::from_document(old), new: Revision::from_document(new), @@ -1231,6 +1337,9 @@ impl Comparison { } let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect(); for name in names { + if reused_cells.contains(name.as_str()) { + continue; + } let a = old.cells.get(name); let b = new.cells.get(name); let mut change = CellChange { @@ -1241,6 +1350,30 @@ impl Comparison { added: Vec::new(), replacements: Vec::new(), }; + if let Some(counts) = whole_cell_objects.as_deref_mut() + && (a.is_none() || b.is_none()) + { + let details = if a.is_some() { + &mut change.removed + } else { + &mut change.added + }; + let mut count = 0; + for object in a.or(b).into_iter().flat_map(|cell| &cell.objects) { + if object.metadata { + continue; + } + count += 1; + if object.layer.is_none() || object.target.is_some() { + details.push(object.into()); + } + } + if count != details.len() { + counts.insert(name.clone(), count); + } + result.cells.push(change); + continue; + } let a: Vec<_> = a .into_iter() .flat_map(|c| &c.objects) @@ -1295,6 +1428,7 @@ impl Comparison { } } + /// Render the affected hierarchy. ASCII output is empty when no cells changed. pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) } @@ -1305,10 +1439,35 @@ impl Comparison { } pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { - let (removed, added) = self.cells.iter().fold((0, 0), |(removed, added), cell| { - (removed + cell.removed.len(), added + cell.added.len()) - }); - let total = removed + added; + self.render_text(max_objects, show, true) + } + + /// Render a text report. Set `show_library_presence` to false when a file + /// header already announces the library addition or deletion. + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.render_text_with_counts(max_objects, show, show_library_presence, &BTreeMap::new()) + } + + fn render_text_with_counts( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + whole_cell_objects: &BTreeMap, + ) -> String { + let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize); + for cell in &self.cells { + match (&cell.old, &cell.new) { + (None, Some(_)) => added += 1, + (Some(_), None) => removed += 1, + _ => modified += 1, + } + } let mut out = match self.outcome() { Outcome::Equal => "Equal", Outcome::Different => "Different", @@ -1316,19 +1475,25 @@ impl Comparison { Outcome::Failed => "Failed", } .to_string(); - if !self.cells.is_empty() { - let cells = if self.cells.len() == 1 { - "cell" - } else { - "cells" - }; - write!(out, " · {} changed {cells}", self.cells.len()).unwrap(); - } - if total != 0 { - write!(out, " · {removed} removed, {added} added").unwrap(); + let mut cell_counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = cell_counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + write!(out, " · {count} {noun} {action}").unwrap(); + for (count, action) in cell_counts { + write!(out, ", {count} {action}").unwrap(); + } } + let library_changes = self.library_changes.iter().filter(|change| { + show_library_presence || !matches!(change, LibraryChange::Presence { .. }) + }); for (count, label) in [ - (self.library_changes.len(), "library change"), + (library_changes.clone().count(), "library change"), (self.layer_names.len(), "layer name change"), ] { if count != 0 { @@ -1349,7 +1514,7 @@ impl Comparison { out.push('\n'); } let notice_start = out.len(); - for change in &self.library_changes { + for change in library_changes { writeln!(out, "library {change}").unwrap(); } if self.library_changes.iter().any(|change| { @@ -1379,22 +1544,38 @@ impl Comparison { } let body_start = out.len(); let mut shown_layers = BTreeSet::new(); + let mut detail_total = 0; let mut selected = 0; let mut shown: usize = 0; let mut hidden_cell_changes = 0; - let visible = - |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); let presence = |action, count| { let objects = if count == 1 { "object" } else { "objects" }; Cow::Owned(format!(" ({action}, {count} {objects})")) }; for cell in &self.cells { + let object_count = |details| { + whole_cell_objects + .get(&cell.name) + .copied() + .unwrap_or(details) + }; let note = match (&cell.old, &cell.new) { - (None, Some(_)) => presence("added", cell.added.len()), - (Some(_), None) => presence("removed", cell.removed.len()), + (None, Some(_)) => presence("added", object_count(cell.added.len())), + (Some(_), None) => presence("removed", object_count(cell.removed.len())), _ if cell.attributes_changed() => Cow::Borrowed(" (attributes changed)"), _ => Cow::Borrowed(""), }; + let whole_cell = cell.old.is_none() || cell.new.is_none(); + // The heading accounts for layered objects in a whole-cell change. + detail_total += if whole_cell { + cell.removed + .iter() + .chain(&cell.added) + .filter(|object| object.layer.is_none()) + .count() + } else { + cell.removed.len() + cell.added.len() + }; if show.cell.as_ref().is_some_and(|name| name != &cell.name) { hidden_cell_changes += usize::from(!note.is_empty()); continue; @@ -1402,11 +1583,16 @@ impl Comparison { let cell_start = out.len(); let previously_shown = shown; writeln!(out, "cell {:?}{note}", cell.name).unwrap(); - let whole_cell = cell.old.is_none() || cell.new.is_none(); + let visible = |object: &ObjectChange| { + (!whole_cell || object.layer.is_none()) + && show.layer.is_none_or(|layer| object.layer == Some(layer)) + }; let mut polygons = changes::PolygonChanges::default(); let mut paired_old = BTreeSet::new(); let mut paired_new = BTreeSet::new(); for replacement in &cell.replacements { + paired_old.insert(replacement.removed); + paired_new.insert(replacement.added); let a = &cell.removed[replacement.removed]; let b = &cell.added[replacement.added]; if max_objects - shown >= 2 && visible(a) && visible(b) { @@ -1416,8 +1602,6 @@ impl Comparison { writeln!(out, " ~ {}", changes::replacement_summary(a, b)).unwrap(); } shown_layers.extend(a.layer); - paired_old.insert(replacement.removed); - paired_new.insert(replacement.added); shown += 2; } } @@ -1426,7 +1610,7 @@ impl Comparison { (false, &cell.added, &paired_new), ] { let sign = if removed { "-" } else { "+" }; - let mut references = BTreeMap::<&str, changes::ReferenceCounts>::new(); + let mut references = BTreeMap::new(); for (index, object) in objects.iter().enumerate() { if !visible(object) { continue; @@ -1435,17 +1619,24 @@ impl Comparison { if !paired.contains(&index) && shown < max_objects { if is_polygon(object.kind) { polygons.unmatched(object, removed); - } else if whole_cell && let Some(target) = object.target.as_deref() { - references.entry(target).or_default().add(object); + } else if let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); } else { - writeln!(out, " {sign} {object}").unwrap(); + changes::write_unmatched(&mut out, sign, object); } shown_layers.extend(object.layer); shown += 1; } } - for (target, counts) in references { - counts.write(&mut out, sign, target); + for (target, (first, counts)) in references { + if whole_cell || counts.len() > 1 { + counts.write(&mut out, sign, target); + } else { + writeln!(out, " {sign} {first}").unwrap(); + } } } polygons.write(&mut out); @@ -1460,31 +1651,23 @@ impl Comparison { } } out.insert_str(body_start, &legend); - let mut notice = String::new(); - if shown < total { - let mut hidden = Vec::new(); - if total > selected { - hidden.push(format!("{} filtered out", total - selected)); + let mut omissions = Vec::new(); + for (count, label, reason) in [ + (detail_total - selected, "object change", "filtered out"), + (selected - shown, "object change", "over the limit"), + (hidden_cell_changes, "cell-level change", "filtered out"), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + omissions.push(format!("{count} {label}{plural} {reason}")); } - if selected > shown { - hidden.push(format!("{} over the limit", selected - shown)); - } - writeln!( - notice, - "Showing {shown} of {total} object changes ({})", - hidden.join(", ") - ) - .unwrap(); } - if hidden_cell_changes != 0 { - let plural = if hidden_cell_changes == 1 { "" } else { "s" }; - writeln!( - notice, - "{hidden_cell_changes} cell-level change{plural} filtered out" - ) - .unwrap(); + if !omissions.is_empty() { + out.insert_str( + notice_start, + &format!("Omitted · {}\n", omissions.join(" · ")), + ); } - out.insert_str(notice_start, ¬ice); // General comparison errors precede revision-specific errors in text. if !self.diagnostics.is_empty() { out.push_str("Diagnostics\n"); @@ -1563,6 +1746,10 @@ fn render_hierarchy( mermaid: bool, max_depth: usize, ) -> String { + if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { + return String::new(); + } + let HierarchyAnalysis { old_edges, new_edges, @@ -1639,10 +1826,18 @@ fn render_hierarchy( " [unresolved]" } } else if direct.contains(name) { - " [cell changed]" + if mermaid { + " [cell changed]" + } else if !old.known_cells.contains(name) { + " [added]" + } else if !new.known_cells.contains(name) { + " [removed]" + } else { + " [changed]" + } } else if affected.contains(name) { " [affected]" - } else if changes.is_some() { + } else if mermaid && changes.is_some() { " [cell unchanged]" } else { "" @@ -1659,7 +1854,7 @@ fn render_hierarchy( }; format!( "{count}{}", - if changed_refs.contains(&key) { + if (mermaid || a == b) && changed_refs.contains(&key) { " [reference changed]" } else { "" diff --git a/src/review/changes.rs b/src/review/changes.rs index 56c9484..8131a66 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -28,6 +28,9 @@ fn group(object: &ObjectChange, bounds: bool) -> Group { /// Pair only unique candidates in the remaining multisets. Ambiguous groups /// remain additions/removals; no identity or geometric equivalence is inferred. pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + if old.is_empty() || new.is_empty() { + return Vec::new(); + } let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new()); let mut result = Vec::new(); for bounds in [true, false] { @@ -101,7 +104,7 @@ fn object_heading(object: &ObjectChange, kind: &str) -> String { } } -fn polygon_xy(object: &ObjectChange) -> &[u8] { +fn object_xy(object: &ObjectChange) -> &[u8] { object .fields .iter() @@ -109,6 +112,27 @@ fn polygon_xy(object: &ObjectChange) -> &[u8] { .map_or(&[], |field| field.data.as_slice()) } +pub(super) fn write_unmatched(out: &mut String, sign: &str, object: &ObjectChange) { + if object.kind == RTAG_PATH + && let Some([xmin, ymin, xmax, ymax]) = object.bounds + { + let vertices = XY::points(object_xy(object)).len(); + if vertices > POINTS_PER_LINE { + let heading = object_heading(object, "path"); + let width = object + .width + .map_or_else(|| "absent".to_string(), |width| format!("{width} dbu")); + writeln!( + out, + " {sign} {heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" + ) + .unwrap(); + return; + } + } + writeln!(out, " {sign} {object}").unwrap(); +} + fn vertex_count(xy: &[u8]) -> usize { XY::points(xy).len().saturating_sub(1) } @@ -147,7 +171,7 @@ struct ShapeCounts { impl ShapeCounts { fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) { let is_rectangle = - |object: &ObjectChange| XY::closed_rectangle(polygon_xy(object)).is_some(); + |object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some(); match object.kind { RTAG_BOX => self.boxes += 1, _ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => { @@ -173,7 +197,7 @@ impl PolygonChanges { pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) { let counts = self.0.entry(old.layer).or_default(); let moved = old.properties == new.properties - && translation(polygon_xy(old), polygon_xy(new)).is_some() + && translation(object_xy(old), object_xy(new)).is_some() && field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY); let shapes = if moved { &mut counts.moved @@ -237,6 +261,10 @@ pub(super) struct ReferenceCounts { } impl ReferenceCounts { + pub fn len(&self) -> usize { + self.singles + self.arrays + } + pub fn add(&mut self, object: &ObjectChange) { if object.kind == RTAG_AREF { self.arrays += 1; @@ -272,8 +300,8 @@ impl ReferenceCounts { } fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { - let before = polygon_xy(old); - let after = polygon_xy(new); + let before = object_xy(old); + let after = object_xy(new); let old_size = rectangle_size(before); let new_size = rectangle_size(after); let mut out = polygon_heading(old, old_size.is_some() && new_size.is_some()); diff --git a/src/review/pair.rs b/src/review/pair.rs new file mode 100644 index 0000000..a40128c --- /dev/null +++ b/src/review/pair.rs @@ -0,0 +1,218 @@ +//! Adaptive parsing for unscoped comparisons of two revisions. + +use super::*; +use std::{sync::Mutex, thread}; + +struct CellSpan<'a> { + body: &'a [u8], + end_record: usize, +} + +struct FramedRevision<'a> { + records: Vec>, + spans: Option>>, +} + +impl<'a> FramedRevision<'a> { + fn read(input: &'a [u8]) -> Result { + let mut markers = Vec::new(); + let records = read_records_with(input, |index, record| { + if matches!(record.tag, RTAG_BGNSTR | RTAG_ENDSTR) { + markers.push(index); + } + })?; + let spans = index_cells(input, &records, markers); + Ok(Self { records, spans }) + } +} + +// Failure to index a layout leaves error reporting to the regular parser. +fn index_cells<'a>( + input: &'a [u8], + records: &[Record<'a>], + markers: Vec, +) -> Option>> { + let mut cells = BTreeMap::new(); + let mut start = None; + for index in markers { + let record = &records[index]; + match record.tag { + RTAG_BGNSTR => { + if start.is_some() { + return None; + } + let name_record = records.get(index + 1).filter(|r| r.tag == RTAG_STRNAME)?; + let name = decode_name(name_record.data) + .ok() + .filter(|name| !name.is_empty())?; + start = Some((name, name_record.offset)); + } + RTAG_ENDSTR => { + let (name, begin) = start.take()?; + let body = &input[begin..record.offset + 4]; + let span = CellSpan { + body, + end_record: index, + }; + if cells.insert(name, span).is_some() { + return None; + } + } + _ => (), + } + } + start.is_none().then_some(cells) +} + +fn matching_cells<'a>( + old: &FramedRevision<'a>, + new: &FramedRevision<'a>, + input_size: usize, +) -> BTreeMap<&'a str, usize> { + let mut matches = BTreeMap::new(); + if let (Some(old), Some(new)) = (&old.spans, &new.spans) { + let mut bytes = 0; + for (&name, cell) in new { + if old.get(name).is_some_and(|old| old.body == cell.body) { + bytes += cell.body.len(); + matches.insert(name, cell.end_record); + } + } + // Reuse serializes some parsing work, so require substantial overlap. + if bytes < input_size - input_size / 10 { + matches.clear(); + } + } + matches +} + +pub(super) struct CellReuse<'a, 'b> { + old: &'b Document<'a>, + matches: &'b BTreeMap<&'a str, usize>, +} + +impl<'a> CellReuse<'a, '_> { + pub(super) fn cell( + &self, + name: &str, + offset: usize, + records: &[Record<'a>], + ) -> Option<(Cell<'a>, usize)> { + let end = *self.matches.get(name)?; + let previous = &self.old.cells[name]; + // Re-run library metadata and reference checks with the new offsets. + let objects = previous + .objects + .iter() + .filter(|object| object.target.is_some() || object.kind == RTAG_TEXT) + .cloned() + .map(|mut object| { + object.offset = offset + (object.offset - previous.offset); + object.end_offset = offset + (object.end_offset - previous.offset); + object + }) + .collect(); + Some(( + Cell { + offset, + end_offset: records[end].offset, + objects, + attributes: previous.attributes.clone(), + issues: Vec::new(), + }, + end, + )) + } +} + +/// Two unscoped GDSII revisions prepared for comparison. +#[derive(Debug)] +pub struct RevisionPair<'a> { + old: Document<'a>, + new: Document<'a>, + reused_cells: BTreeSet<&'a str>, +} + +impl<'a> RevisionPair<'a> { + pub fn parse(old: &'a [u8], new: &'a [u8]) -> Result { + let invalid = |message, side| Diagnostic::new("invalid_input", message, Some(side)); + let input_size = old.len().max(new.len()); + let parallel = thread::available_parallelism().is_ok_and(|count| count.get() > 1); + let (old, new) = run_together(parallel, old, new, FramedRevision::read); + let old = old.map_err(|message| invalid(message, Side::Old))?; + let new = match new { + Ok(new) => new, + Err(message) => { + Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + return Err(invalid(message, Side::New)); + } + }; + let mut matches = matching_cells(&old, &new, input_size); + let (old, new) = if matches.is_empty() { + let (old, new) = run_together(parallel, old, new, |revision| { + Document::parse_records(&revision.records, None) + }); + (old.map_err(|message| invalid(message, Side::Old))?, new) + } else { + let old_doc = Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + drop(old); + // The regular parser gives unsupported records their own offsets + // and messages, so parse those cells independently on each side. + matches.retain(|name, _| { + old_doc.cells.get(*name).is_some_and(|cell| { + cell.issues.is_empty() + && cell.objects.iter().all(|object| object.issues.is_empty()) + }) + }); + let reuse = CellReuse { + old: &old_doc, + matches: &matches, + }; + let new = Document::parse_records(&new.records, Some(reuse)); + (old_doc, new) + }; + Ok(Self { + old, + new: new.map_err(|message| invalid(message, Side::New))?, + reused_cells: matches.into_keys().collect(), + }) + } + + pub fn compare(self) -> Comparison { + Comparison::collect(&self.old, &self.new, &self.reused_cells, None) + } + + pub fn compare_text(self) -> TextComparison { + TextComparison::collect(&self.old, &self.new, &self.reused_cells) + } +} + +fn run_together( + parallel: bool, + old: I, + new: I, + process: impl Fn(I) -> T + Copy + Send, +) -> (T, T) { + if !parallel { + return (process(old), process(new)); + } + // Transfer ownership only once a worker starts, so a spawn failure can + // process the same input on the calling thread. + let input = Mutex::new(Some(old)); + let run_old = { + let input = &input; + move || { + let old = input.lock().unwrap().take().unwrap(); + process(old) + } + }; + thread::scope(|scope| { + if let Ok(worker) = thread::Builder::new().spawn_scoped(scope, run_old) { + let new = process(new); + return (worker.join().unwrap(), new); + } + (run_old(), process(new)) + }) +} diff --git a/src/review/report.rs b/src/review/report.rs index 228ac12..6450755 100644 --- a/src/review/report.rs +++ b/src/review/report.rs @@ -1,6 +1,11 @@ //! Versioned machine-readable diff reports. No consumer needs to parse prose. use super::*; +use serde::{ + Serialize, Serializer, + ser::{SerializeMap, SerializeSeq}, +}; use serde_json::{Value, json}; +use std::io; use std::path::{Path, PathBuf}; pub const SCHEMA_VERSION: u32 = 1; @@ -196,10 +201,127 @@ fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { .collect() } +struct SelectedObjects<'a> { + // Up to the last emitted object. A layer filter can leave gaps in this prefix. + objects: &'a [ObjectChange], + layer: Option, + cell_index: usize, + side: &'static str, +} + +impl SelectedObjects<'_> { + fn id(&self, index: usize) -> String { + format!("c{}:{}:{index}", self.cell_index, self.side) + } + + fn contains(&self, index: usize) -> bool { + self.objects + .get(index) + .is_some_and(|object| self.layer.is_none_or(|layer| object.layer == Some(layer))) + } +} + +impl Serialize for SelectedObjects<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(None)?; + for (index, change) in self.objects.iter().enumerate() { + if self.layer.is_none_or(|layer| change.layer == Some(layer)) { + sequence.serialize_element(&object(change, &self.id(index), self.side))?; + } + } + sequence.end() + } +} + +struct SelectedCell<'a> { + cell: &'a CellChange, + removed: SelectedObjects<'a>, + added: SelectedObjects<'a>, +} + +impl Serialize for SelectedCell<'_> { + fn serialize(&self, serializer: S) -> Result { + let revision = |revision: &CellRevision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)}); + let mut map = serializer.serialize_map(Some(7))?; + map.serialize_entry("added", &self.added)?; + map.serialize_entry("after", &self.cell.new.as_ref().map(revision))?; + map.serialize_entry("attributes_changed", &self.cell.attributes_changed())?; + map.serialize_entry("before", &self.cell.old.as_ref().map(revision))?; + map.serialize_entry("name", &self.cell.name)?; + map.serialize_entry("removed", &self.removed)?; + map.serialize_entry("replacements", &SelectedReplacements(self))?; + map.end() + } +} + +struct SelectedReplacements<'a>(&'a SelectedCell<'a>); + +impl Serialize for SelectedReplacements<'_> { + fn serialize(&self, serializer: S) -> Result { + let selection = self.0; + let mut sequence = serializer.serialize_seq(None)?; + for replacement in &selection.cell.replacements { + if selection.removed.contains(replacement.removed) + && selection.added.contains(replacement.added) + { + let old = &selection.cell.removed[replacement.removed]; + let new = &selection.cell.added[replacement.added]; + sequence.serialize_element(&json!({ + "removed": selection.removed.id(replacement.removed), + "added": selection.added.id(replacement.added), + "match_rule": replacement.match_rule, + "explanation": changes::replacement_summary(old, new), + "field_changes": field_changes(old, new), + "properties_changed": old.properties != new.properties + }))?; + } + } + sequence.end() + } +} + +struct JsonReport<'a> { + metadata: Value, + cells: Vec>, +} + +impl Serialize for JsonReport<'_> { + fn serialize(&self, serializer: S) -> Result { + let metadata = self + .metadata + .as_object() + .expect("report metadata is an object"); + let mut map = serializer.serialize_map(Some(metadata.len()))?; + for (key, value) in metadata { + if key == "cell_changes" { + map.serialize_entry(key, &self.cells)?; + } else { + map.serialize_entry(key, value)?; + } + } + map.end() + } +} + impl Comparison { /// Render the captured comparison evidence without retaining its source documents. /// IDs are local to this report, not persistent object identities. pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value { + serde_json::to_value(self.prepare_report(inputs, options)) + .expect("serialize comparison report") + } + + /// Stream a pretty-printed report to a writer. + pub fn write_json_report( + &self, + writer: impl io::Write, + inputs: &ReportInputs, + options: &ReportOptions, + ) -> serde_json::Result<()> { + serde_json::to_writer_pretty(writer, &self.prepare_report(inputs, options)) + } + + fn prepare_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> JsonReport<'_> { let outcome = self.outcome(); let mut report = envelope( inputs, @@ -257,12 +379,8 @@ impl Comparison { filtered_objects += cell.removed.len() + cell.added.len(); continue; } - let (mut removed, mut added) = (Vec::new(), Vec::new()); - let mut shown = BTreeSet::new(); - for (side, objects, output) in [ - ("old", &cell.removed, &mut removed), - ("new", &cell.added, &mut added), - ] { + let mut select = |objects: &[ObjectChange]| { + let mut end = 0; for (index, change) in objects.iter().enumerate() { if options .filter @@ -273,32 +391,29 @@ impl Comparison { } else if options.max_objects.is_some_and(|max| emitted >= max) { limited += 1; } else { - let id = format!("c{cell_index}:{side}:{index}"); - output.push(object(change, &id, side)); - shown.insert(id); + end = index + 1; emitted += 1; } } - } - let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| { - let a = format!("c{cell_index}:old:{}", r.removed); - let b = format!("c{cell_index}:new:{}", r.added); - (shown.contains(&a) && shown.contains(&b)).then(|| json!({ - "removed": a, "added": b, "match_rule": r.match_rule, - "explanation": changes::replacement_summary(&cell.removed[r.removed], &cell.added[r.added]), - "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), - "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties - })) - }).collect(); - cells.push(json!({ - "name": cell.name, - "before": cell.old.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})), - "after": cell.new.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})), - "attributes_changed": cell.attributes_changed(), - "removed": removed, "added": added, "replacements": replacements - })); + end + }; + let layer = options.filter.layer; + cells.push(SelectedCell { + cell, + removed: SelectedObjects { + objects: &cell.removed[..select(&cell.removed)], + layer, + cell_index, + side: "old", + }, + added: SelectedObjects { + objects: &cell.added[..select(&cell.added)], + layer, + cell_index, + side: "new", + }, + }); } - report["cell_changes"] = json!(cells); report["report_selection"]["truncated"] = json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0); for (key, count) in [ @@ -313,7 +428,10 @@ impl Comparison { if options.hierarchy { report["hierarchy"] = graph(self); } - report + JsonReport { + metadata: report, + cells, + } } } diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt index eeb7497..d493fa4 100644 --- a/tests/golden/review-diff.txt +++ b/tests/golden/review-diff.txt @@ -1,3 +1,3 @@ -Different · 1 changed cell · 1 removed, 1 added +Different · 1 cell modified cell "CHIP" 1/0: 1 polygon changed diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt index cdab505..facae9e 100644 --- a/tests/golden/review-hierarchy.txt +++ b/tests/golden/review-hierarchy.txt @@ -1,6 +1,6 @@ Hierarchy "CHIP" [affected] "left" x1 [affected] - "leaf" x2 [cell changed] + "leaf" x2 [changed] "right" x1 [affected] - "leaf" x3 [cell changed] [shown above] + "leaf" x3 [changed] [shown above] diff --git a/tests/review.rs b/tests/review.rs index 02ffb3b..3c312e6 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -5,7 +5,9 @@ mod reports; use common::{library_header, record}; use pheon::{ records::*, - review::{Comparison, Document, names::LAYER_SENTINEL}, + review::{ + Comparison, Document, TextComparison, names::LAYER_SENTINEL, selection::DisplayFilter, + }, }; use std::{fs, path::PathBuf, process::Command}; @@ -103,6 +105,19 @@ fn label(carrier: (u16, u16), text: &str) -> Vec { bytes } +fn path(width: Option, points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_PATH, &[]); + record(&mut bytes, RTAG_LAYER, &[0, 1]); + record(&mut bytes, RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, RTAG_WIDTH, &width.to_be_bytes()); + } + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + fn with_property(mut object: Vec, value: &str) -> Vec { object.truncate(object.len() - 4); record(&mut object, RTAG_PROPATTR, &17_u16.to_be_bytes()); @@ -500,9 +515,9 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { "Hierarchy\n", "\"CHIP\" [affected]\n", " \"left\" x1 [affected]\n", - " \"leaf\" x2 [cell changed]\n", + " \"leaf\" x2 [changed]\n", " \"right\" x1 [affected]\n", - " \"leaf\" x3 [cell changed] [shown above]\n", + " \"leaf\" x3 [changed] [shown above]\n", ) ); let graph = diff.hierarchy(true, 8); @@ -524,11 +539,12 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let new = Document::parse(&pair.new).unwrap(); let diff = Comparison::between(&old, &new); let tree = diff.hierarchy(false, 8); - assert!(tree.contains("[cell unchanged]"), "{tree}"); - assert!(tree.contains("[reference changed]"), "{tree}"); - if pair.name == "array count" { - assert!(tree.contains("x2 -> x3"), "{tree}"); - } + let edge = if pair.name == "array count" { + " \"leaf\" x2 -> x3\n" + } else { + " \"leaf\" x1 [reference changed]\n" + }; + assert!(tree.ends_with(edge), "{tree}"); } let a = layout(&[ ("CHIP", vec![reference("leaf", 0, 1)]), @@ -537,14 +553,19 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let b = layout(&[("CHIP", vec![])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); - let tree = Comparison::between(&a, &b).hierarchy(false, 8); - assert!(tree.contains("\"leaf\" x1 -> x0"), "{tree}"); + for (old, new, edge) in [ + (&a, &b, " \"leaf\" x1 -> x0 [removed]\n"), + (&b, &a, " \"leaf\" x0 -> x1 [added]\n"), + ] { + let tree = Comparison::between(old, new).hierarchy(false, 8); + assert!(tree.ends_with(edge), "{tree}"); + } } #[test] fn same_bounds_changes_and_truncated_reports_remain_differences() { let a = layout(&[("CHIP", vec![square()])]); - let b = layout(&[("CHIP", vec![triangle()])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new label")])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); @@ -552,7 +573,10 @@ fn same_bounds_changes_and_truncated_reports_remain_differences() { diff.cells[0].removed[0].bounds, diff.cells[0].added[0].bounds ); - assert!(diff.text(0).contains("Showing 0 of 2 object changes")); + let limited = diff.text(1); + assert!(limited.contains("Omitted · 2 object changes over the limit")); + assert!(limited.contains("+ 2/0: label")); + assert!(!limited.contains("1/0:")); assert_eq!(diff.outcome().exit_code(), 1); assert!(diff.cells[0].removed[0].offset > 0); } @@ -586,6 +610,7 @@ fn unit_changes_report_values_without_claiming_geometry_equivalence() { assert!(text.contains("Coordinates use each file's database units")); assert!(!text.contains("raw=")); assert!(diff.cells.is_empty()); + assert!(diff.hierarchy(false, 8).is_empty()); } #[test] @@ -712,7 +737,13 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { fs::write(&old, shared(square())).unwrap(); fs::write(&new, shared(triangle())).unwrap(); let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); - let equal = command().arg("diff").arg(&old).arg(&old).output().unwrap(); + let equal = command() + .arg("diff") + .arg(&old) + .arg(&old) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); assert_eq!(equal.status.code(), Some(0)); assert_eq!(equal.stdout, b"Equal\n"); let diff = command() @@ -724,7 +755,7 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { .unwrap(); assert_eq!(diff.status.code(), Some(1)); let text = String::from_utf8(diff.stdout).unwrap(); - assert!(text.contains("Showing 0 of 2 object changes")); + assert!(text.contains("Omitted · 2 object changes over the limit")); assert!(text.contains("Hierarchy\n```mermaid\nflowchart TD")); assert!(!text.contains("scope top=all")); let dump = command() @@ -1105,9 +1136,9 @@ fn display_filters_never_change_equality_or_totals() { layer: Some((42, 0)), }, ); - assert!(text.starts_with("Different · 1 changed cell · 1 removed, 1 added\n")); + assert!(text.starts_with("Different · 1 cell modified\n")); assert!(text.contains("Display filter layer=42/0\n")); - assert!(text.contains("Showing 0 of 2 object changes (2 filtered out)")); + assert!(text.contains("Omitted · 2 object changes filtered out")); assert!(!text.contains("cell \"CHIP\"")); assert_eq!(diff.outcome().exit_code(), 1); } @@ -1133,12 +1164,13 @@ fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { ]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); let text = diff.text(0); + assert!(text.starts_with("Different · 3 cells modified, 2 added, 1 removed\n")); assert!(text.contains("cell \"gone\" (removed, 2 objects)\n")); assert!(text.contains("cell \"new\" (added, 1 object)\n")); assert!(text.contains("cell \"empty\" (added, 0 objects)\n")); assert!(text.contains("cell \"header\" (attributes changed)\n")); assert!(!text.contains("cell \"changed\"")); - assert!(text.contains("Showing 0 of 7 object changes (7 over the limit)")); + assert!(text.contains("Omitted · 4 object changes over the limit")); let text = diff.text_filtered( 1, @@ -1149,21 +1181,25 @@ fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { ); assert!(text.contains(concat!( "Display filter cell=\"changed\" layer=1/0\n", - "Showing 1 of 7 object changes (5 filtered out, 1 over the limit)\n", - "4 cell-level changes filtered out\n", - "cell \"changed\"\n 1/0: 1 rectangle removed", + "Omitted · 2 object changes filtered out · 2 object changes over the limit", + " · 4 cell-level changes filtered out\n", ))); + assert!(!text.contains("cell \"changed\"")); assert!(!text.contains("cell \"other\"")); assert!(!text.contains("cell \"new\"")); } #[test] -fn reference_summaries_group_whole_cells_and_keep_full_placements_in_dumps() { +fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps() { let empty = layout(&[("leaf", vec![]), ("other", vec![])]); let with_chip = layout(&[ ( "CHIP", vec![ + square(), + label((2, 0), "chip label"), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "3:0:WG"), reference("leaf", 0, 1), reference("leaf", 5, 1), reference("leaf", 2, 2), @@ -1178,21 +1214,60 @@ fn reference_summaries_group_whole_cells_and_keep_full_placements_in_dumps() { let with_chip = Document::parse(&with_chip).unwrap(); let existing_chip = Document::parse(&existing_chip).unwrap(); - for (old, new, sign) in [(&empty, &with_chip, "+"), (&with_chip, &empty, "-")] { + for (old, new, action, sign) in [ + (&empty, &with_chip, "added", "+"), + (&with_chip, &empty, "removed", "-"), + ] { let diff = Comparison::between(old, new); + let text_diff = TextComparison::between(old, new); let text = diff.text(100); + assert!(text.contains(&format!("cell \"CHIP\" ({action}, 6 objects)\n"))); assert!(text.contains(&format!( " {sign} 2 references, 1 array reference to \"leaf\" (4 placements)\n" ))); assert!(text.contains(&format!(" {sign} 1 reference to \"other\"\n"))); assert!(!text.contains("xy=")); + assert!(!text.contains("1/0")); + assert!(!text.contains("2/0")); + assert!(!text.contains("Omitted")); let limited = diff.text(2); assert!(limited.contains(&format!(" {sign} 2 references to \"leaf\"\n"))); - assert!(limited.contains("Showing 2 of 4 object changes (2 over the limit)")); + assert!(limited.contains("Omitted · 2 object changes over the limit")); + assert_eq!(text_diff.outcome(), diff.outcome()); + for mermaid in [false, true] { + assert_eq!(text_diff.hierarchy(mermaid, 8), diff.hierarchy(mermaid, 8)); + } + for show in [ + DisplayFilter::default(), + DisplayFilter { + cell: Some("other".into()), + layer: None, + }, + DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ] { + for max in [0, 2, 100] { + assert_eq!( + text_diff.render_text(max, &show, true), + diff.text_filtered(max, &show) + ); + } + } } - let text = Comparison::between(&existing_chip, &with_chip).text(100); - assert!(text.contains(" + reference \"leaf\" xy=(0, 0)\n")); + for (old, new, sign) in [ + (&existing_chip, &with_chip, "+"), + (&with_chip, &existing_chip, "-"), + ] { + let text = Comparison::between(old, new).text(100); + assert!(text.contains(&format!( + " {sign} 2 references, 1 array reference to \"leaf\" (4 placements)\n" + ))); + assert!(!text.contains("\"leaf\" xy=")); + assert!(text.contains(&format!(" {sign} reference \"other\" xy=(10, 0)\n"))); + } let dump = with_chip.canonical_dump(); assert!(dump.contains(" reference \"leaf\" xy=(5, 0)\n")); assert!(dump.contains(" array reference \"leaf\" (2 × 1) xy=[(2, 0), (102, 0), (2, 0)]\n")); @@ -1265,7 +1340,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!( String::from_utf8(changed.stdout) .unwrap() - .contains("Different · 1 changed cell · 1 removed, 1 added") + .contains("Different · 1 cell modified\n") ); fs::write(&path, b"invalid").unwrap(); let invalid = Command::new("git") @@ -1278,23 +1353,34 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!(!invalid.status.success()); assert!(String::from_utf8_lossy(&invalid.stderr).contains("pheon:")); fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + let mode_change = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["git-diff", "--hierarchy", "ascii", "--", "chip layout.gds"]) + .arg(&path) + .args(["old", "100644"]) + .arg(&path) + .args(["new", "100755"]) + .output() + .unwrap(); + assert!(mode_change.status.success()); + assert_eq!( + mode_change.stdout, + b"layout \"chip layout.gds\"\nmode \"100644\" -> \"100755\"\nEqual\n" + ); let added = directory.join("added layout.gds"); fs::write(&added, shared(square())).unwrap(); git(&["add", "added layout.gds"]); let result = git(&["diff", "--cached", "--", "added layout.gds"]); - assert!( - String::from_utf8(result.stdout) - .unwrap() - .contains("layout added") - ); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.starts_with("layout \"added layout.gds\" added\nDifferent · 4 cells added\n")); + assert!(!text.contains("\nmode ")); + assert!(!text.contains("\nlibrary ")); fs::remove_file(&path).unwrap(); git(&["add", "-u"]); let result = git(&["diff", "--cached", "--", "chip layout.gds"]); - assert!( - String::from_utf8(result.stdout) - .unwrap() - .contains("layout deleted") - ); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.starts_with("layout \"chip layout.gds\" deleted\nDifferent · 1 cell removed\n")); + assert!(!text.contains("\nmode ")); + assert!(!text.contains("\nlibrary ")); // Textconv is an alternative to the external driver, with a separate cache. git(&["commit", "-qm", "replace fixture"]); git(&["mv", "added layout.gds", "renamed layout.gds"]); @@ -1304,10 +1390,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let result = git(&["diff", "--cached", "--find-renames"]); let text = String::from_utf8(result.stdout).unwrap(); assert!(text.contains("renamed layout.gds"), "{text}"); - assert!( - text.contains("Different · 1 changed cell · 1 removed, 1 added"), - "{text}" - ); + assert!(text.contains("Different · 1 cell modified\n"), "{text}"); git(&["commit", "-qm", "rename fixture"]); fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); fs::write(&renamed, shared(square())).unwrap(); @@ -1348,7 +1431,10 @@ fn replacement_explanations_are_unique_and_keep_full_evidence() { assert!(!text.contains(" + ")); assert!(!text.contains("bbox=")); assert!(!diff.text(0).contains(" ~ ")); - assert!(diff.text(1).contains("Showing 1 of 2 object changes")); + assert_eq!( + diff.text(1), + "Different · 1 cell modified\nOmitted · 2 object changes over the limit\n" + ); } let a = layout(&[("CHIP", vec![square(), square()])]); let b = layout(&[("CHIP", vec![triangle(), triangle()])]); @@ -1440,7 +1526,7 @@ fn polygon_diff_groups_shapes_by_layer_across_dimensions() { assert_eq!( diff.text(100), concat!( - "Different · 1 changed cell · 4 removed, 4 added\n", + "Different · 1 cell modified\n", "cell \"CHIP\"\n", " 1/0: 2 rectangles, 1 polygon changed\n", " 2/0: 1 polygon changed\n", @@ -1463,20 +1549,9 @@ fn polygon_diff_distinguishes_boxes_from_rectangular_boundaries() { #[test] fn path_width_explanation_preserves_absent_and_zero() { - let path = |width: Option| { - let mut bytes = Vec::new(); - record(&mut bytes, RTAG_PATH, &[]); - record(&mut bytes, RTAG_LAYER, &[0, 1]); - record(&mut bytes, RTAG_DATATYPE, &[0, 0]); - if let Some(n) = width { - record(&mut bytes, RTAG_WIDTH, &n.to_be_bytes()); - } - xy(&mut bytes, &[[0, 0], [10, 10]]); - record(&mut bytes, RTAG_ENDEL, &[]); - layout(&[("CHIP", vec![bytes])]) - }; - let a = path(None); - let b = path(Some(0)); + let points = [[0, 0], [10, 10]]; + let a = layout(&[("CHIP", vec![path(None, &points)])]); + let b = layout(&[("CHIP", vec![path(Some(0), &points)])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); assert!( diff.text(100) @@ -1484,6 +1559,43 @@ fn path_width_explanation_preserves_absent_and_zero() { ); } +#[test] +fn unmatched_paths_summarize_coordinates_that_would_wrap() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let empty = layout(&[("CHIP", vec![])]); + let paths = layout(&[( + "CHIP", + vec![ + path(None, &points), + path(Some(-8), &points), + path(Some(0), &points[..4]), + ], + )]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let text = Comparison::between(old, new).text(100); + for width in ["absent", "-8 dbu"] { + assert!(text.contains(&format!( + " {sign} 1/0: path · width {width} · 5 vertices · bounds (0, -5) to (13, 9) dbu\n" + ))); + } + assert_eq!(text.lines().count(), 5, "{text}"); + assert!(text.contains("xy=[(0, 0), (4, 0), (4, -5), (13, -5)]")); + } + assert!(paths.canonical_dump().contains("\n (13, 9)]")); + let report = + Comparison::between(&empty, &paths).json_report(&Default::default(), &Default::default()); + assert!( + report["cell_changes"][0]["added"] + .as_array() + .unwrap() + .iter() + .flat_map(|object| object["fields"].as_array().unwrap()) + .any(|field| field["tag"] == RTAG_XY && field["value"] == serde_json::json!(points)) + ); +} + #[test] fn label_replacements_show_only_changed_fields() { let a = layout(&[("CHIP", vec![label((2, 0), "old")])]); @@ -1491,7 +1603,7 @@ fn label_replacements_show_only_changed_fields() { let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); assert_eq!( diff.text(100), - "Different · 1 changed cell · 1 removed, 1 added\ncell \"CHIP\"\n ~ 2/0: label text \"old\" -> \"new\"\n" + "Different · 1 cell modified\ncell \"CHIP\"\n ~ 2/0: label text \"old\" -> \"new\"\n" ); } @@ -1536,6 +1648,10 @@ fn library_presence_changes_distinguish_empty_and_absent_revisions() { assert_eq!(added.outcome().exit_code(), 1); assert!(added.text(0).contains("library added\n")); assert!(added.text(0).starts_with("Different · 1 library change\n")); + assert_eq!( + added.render_text(0, &Default::default(), false), + "Different\n" + ); let removed = Comparison::between(&doc, &absent); assert_eq!(removed.outcome().exit_code(), 1); assert!(removed.text(0).contains("library removed\n")); diff --git a/tests/review/reports.rs b/tests/review/reports.rs index 26e796a..71f06cd 100644 --- a/tests/review/reports.rs +++ b/tests/review/reports.rs @@ -1,6 +1,6 @@ use super::*; use pheon::review::{ - Diagnostic, Outcome, Side, + Diagnostic, Outcome, RevisionPair, Side, report::{ReportInputs, ReportOptions, failure_report}, selection::{DisplayFilter, Scope}, }; @@ -27,6 +27,78 @@ fn unknown(geometry: Vec) -> Vec { layout(&[("CHIP", vec![object])]) } +#[test] +fn adaptive_pair_rebases_diagnostics_and_keeps_metadata_and_graph_state() { + let mut stable = vec![square(); 512]; + stable.extend([ + reference("leaf", 0, 2), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "PHEON_LAYER_NAMES_V2"), + ]); + let mut unsupported = square(); + unsupported.truncate(unsupported.len() - 4); + record(&mut unsupported, 0x7f02, &[0, 42]); + record(&mut unsupported, RTAG_ENDEL, &[]); + let old = layout(&[ + ("leaf", vec![]), + ("changed", vec![square()]), + ("stable", stable.clone()), + ("unsupported", vec![unsupported.clone()]), + ]); + let mut new = layout(&[ + ("unsupported", vec![unsupported]), + ("stable", stable), + ("changed", vec![triangle()]), + ]); + let stable_offset = Document::parse(&new).unwrap().cells["stable"].offset; + new[stable_offset + 4..stable_offset + 6].copy_from_slice(&2026_u16.to_be_bytes()); + + let full = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let adaptive = RevisionPair::parse(&old, &new).unwrap().compare(); + let options = ReportOptions { + hierarchy: true, + ..ReportOptions::default() + }; + let actual = adaptive.json_report(&inputs(), &options); + assert_eq!(actual, full.json_report(&inputs(), &options)); + assert_eq!(actual["layer_legend"][0]["name"], "WG"); + assert_eq!(actual["status"], "incomplete"); + assert!( + actual["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|issue| { issue["code"] == "unresolved_reference" && issue["side"] == "new" }) + ); + let text = RevisionPair::parse(&old, &new).unwrap().compare_text(); + assert_eq!( + text.render_text(100, &DisplayFilter::default(), true), + full.text(100) + ); + assert_eq!(text.hierarchy(false, 8), full.hierarchy(false, 8)); + assert_eq!(text.hierarchy(true, 8), full.hierarchy(true, 8)); +} + +#[test] +fn adaptive_pair_prioritizes_old_structure_errors_over_new_frame_errors() { + let mut old = layout(&[("CHIP", vec![square()])]); + let name_offset = Document::parse(&old) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_LIBNAME) + .unwrap() + .offset; + old[name_offset + 2] = 0x7f; + let error = RevisionPair::parse(&old, b"bad").unwrap_err(); + assert_eq!(error.side, Some(Side::Old)); + assert!(error.message.contains("LIBNAME and UNITS")); +} + #[test] fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { let comparison = { @@ -75,7 +147,11 @@ fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { assert_eq!(limited["hierarchy"], actual["hierarchy"]); assert_eq!(limited["summary"], actual["summary"]); assert_eq!(limited["exit_code"], 1); - assert!(comparison.text(0).contains("Showing 0 of 2 object changes")); + assert!( + comparison + .text(0) + .contains("Omitted · 2 object changes over the limit") + ); } #[test] @@ -323,6 +399,33 @@ fn json_is_unbounded_by_default_and_explicit_filters_do_not_change_totals() { assert_eq!(filtered["cell_changes"][0]["added"], json!([])); } +#[test] +fn streamed_report_keeps_object_ids_and_replacements_across_layer_filter_gaps() { + let a = layout(&[("CHIP", vec![square(), label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let options = ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + max_objects: Some(2), + ..Default::default() + }; + let mut bytes = Vec::new(); + diff.write_json_report(&mut bytes, &inputs(), &options) + .unwrap(); + let actual: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(actual, diff.json_report(&inputs(), &options)); + assert_eq!(actual["report_selection"]["filtered_objects"], "2"); + let cell = &actual["cell_changes"][0]; + assert_eq!(cell["added"].as_array().unwrap().len(), 1); + assert_eq!(cell["removed"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"][0]["removed"], "c0:old:1"); + assert_eq!(cell["replacements"][0]["added"], "c0:new:1"); +} + #[test] fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { let mut a = reference("leaf", 0, 1); @@ -710,6 +813,22 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { let full = command(&["--hierarchy", "graph"]); assert_eq!(full["result"], "different"); assert!(full["hierarchy"].is_object()); + let large = |mut bytes: Vec| { + bytes.resize(3 * 1024 * 1024, 0); + bytes + }; + let mut large_old = large(shared(square())); + let mut large_new = large(shared(triangle())); + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&["--hierarchy", "graph"]), full); + *large_old.last_mut().unwrap() = 1; + *large_new.last_mut().unwrap() = 1; + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&[])["diagnostics"][0]["side"], "old"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); let limited = command(&["--max-changes", "0"]); assert_eq!(limited["summary"], full["summary"]); assert_eq!(limited["report_selection"]["emitted_objects"], "0"); From f474e9848ccc1ff884180f2078c66c5b1ec289fa Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 12:36:58 -0800 Subject: [PATCH 27/41] Require PyArrow in the Python package Keep the arrow extra available for existing requirements. Update wheel checks and install instructions, and verify that integer packing leaves PyArrow unloaded. --- .github/workflows/packages.yml | 3 +-- README.md | 2 +- docs/development.md | 4 ++-- docs/python.md | 13 ++++++------- docs/releases.md | 9 ++++----- pheon/__init__.py | 2 +- pheon/arrow.py | 3 +-- pyproject.toml | 3 ++- tools/check_installed.py | 4 ++-- 9 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 710c22f..321f3c6 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -56,7 +56,6 @@ jobs: CIBW_ARCHS: native CIBW_ENVIRONMENT_LINUX: 'CARGO_HOME=/tmp/pheon-cargo RUSTUP_HOME=/tmp/pheon-rustup PATH=/tmp/pheon-cargo/bin:$PATH' CIBW_BEFORE_ALL_LINUX: 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable' - CIBW_TEST_EXTRAS: arrow CIBW_TEST_COMMAND: 'python -I {project}/tools/check_installed.py' - name: Check wheel ABI tags shell: python @@ -91,7 +90,7 @@ jobs: path: wheelhouse - name: Install the same artifact on another Python version shell: bash - run: python -m pip install wheelhouse/*.whl pyarrow + run: python -m pip install wheelhouse/*.whl - run: python -I tools/check_installed.py cli: diff --git a/README.md b/README.md index 470c168..5d035ba 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ comparisons. Start with the [CLI guide](docs/cli.md), then set up Python 3.11 or newer is required. From a source checkout: ```sh -python -m pip install '.[arrow]' +python -m pip install . ``` Run Python outside the source checkout to use the installed package: diff --git a/docs/development.md b/docs/development.md index 493f35b..e884456 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,7 +13,7 @@ the tracked `Cargo.lock`; update dependencies explicitly with Cargo. cargo test --locked cargo test --release --locked cargo build --release --locked -python -m pip install '.[arrow]' +python -m pip install . python -I tools/check_installed.py ``` @@ -27,7 +27,7 @@ Unset this override when checking installed artifacts with `check_installed.py`. ## Check upstream layouts The [GDSII corpus](../contrib/gds-corpus.toml) pins open-source fixtures and cell -libraries. After installing `.[arrow]` and building the release CLI, run: +libraries. After installing Pheon and building the release CLI, run: ```sh python -I tools/check_gds_corpus.py --fetch diff --git a/docs/python.md b/docs/python.md index fa48cf4..bc03060 100644 --- a/docs/python.md +++ b/docs/python.md @@ -6,16 +6,15 @@ Python 3.11 or newer is required. ## Install -To install a PyPI release with Arrow support: +To install a PyPI release: ```sh -python -m pip install 'pheon[arrow]' +python -m pip install pheon ``` -For a local wheel, pass its file path followed by `[arrow]` in the same quotes. -Prebuilt wheels bundle the native library. From a source checkout, use -`python -m pip install '.[arrow]'` with Rust 1.88 or newer. The base `pheon` -package is enough for integer packing. +The package installs PyArrow for layout reads. Existing `pheon[arrow]` +requirements still work. Prebuilt wheels bundle the native library. From a +source checkout, use `python -m pip install .` with Rust 1.88 or newer. Run Python outside the source checkout to use the installed package. @@ -89,7 +88,7 @@ errors use Python's normal I/O exceptions. ## Use from Klamath or Masque Klamath's `native` extra installs Pheon's packing helpers. Masque's `arrow` extra -installs Pheon with Arrow support. Use the same Python environment as the +installs Pheon for layout reads. Use the same Python environment as the companion package. See [development](development.md) to test a locally built native library from diff --git a/docs/releases.md b/docs/releases.md index 4384205..b21c70d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -55,7 +55,6 @@ export CIBW_CONTAINER_ENGINE="podman; disable_host_mount: true; create_args: \ export CIBW_MANYLINUX_X86_64_IMAGE=localhost/pheon-manylinux:2.28-2026.10.26-1 export CIBW_ENVIRONMENT_LINUX='PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo CARGO_NET_OFFLINE=true PIP_CACHE_DIR=/pheon-cache/pip' export CIBW_BUILD='cp31{1,2,3,4}-manylinux_x86_64' -export CIBW_TEST_EXTRAS=arrow export CIBW_TEST_COMMAND='python -I {project}/tools/check_installed.py' unset PHEON_LIBRARY python -m cibuildwheel --platform linux --archs x86_64 \ @@ -93,12 +92,12 @@ Wheel filenames use `py3-none` plus a platform tag, such as `manylinux_2_28_x86_64` or `win_amd64`. Use repaired `manylinux_*` wheels for Linux releases. -To test a wheel manually, install it with its `arrow` extra in a test -environment on the target OS. Clear `PHEON_LIBRARY` so the checks load the -bundled library. Replace the example path with the wheel you built. +To test a wheel manually, install it in a test environment on the target OS. +Clear `PHEON_LIBRARY` so the checks load the bundled library. Replace the +example path with the wheel you built. ```sh -python -m pip install "path/to/pheon-wheel.whl[arrow]" +python -m pip install "path/to/pheon-wheel.whl" python -I tools/check_installed.py ``` diff --git a/pheon/__init__.py b/pheon/__init__.py index b71eb35..535f808 100644 --- a/pheon/__init__.py +++ b/pheon/__init__.py @@ -1,4 +1,4 @@ -"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers.""" +"""Native GDSII helpers. Use pheon.arrow to read layouts into PyArrow.""" from importlib.metadata import version as _version diff --git a/pheon/arrow.py b/pheon/arrow.py index 5aa6ee1..af3a39a 100644 --- a/pheon/arrow.py +++ b/pheon/arrow.py @@ -1,7 +1,6 @@ """GDSII readers returning a one-row Arrow StructArray for the library. -The returned arrays own their buffers independently of the input. This module -requires the optional ``arrow`` extra; integer packing does not import PyArrow. +The returned arrays own their buffers independently of the input. """ from __future__ import annotations diff --git a/pyproject.toml b/pyproject.toml index 835880a..aaa9512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,11 @@ dynamic = ["version"] dependencies = [ "cffi", "numpy", + "pyarrow>=14", ] [project.optional-dependencies] -arrow = ["pyarrow>=14"] +arrow = [] # Compatibility with existing requirements for pheon[arrow]. [project.urls] Homepage = "https://mpxd.net/code/jan/pheon" diff --git a/tools/check_installed.py b/tools/check_installed.py index 970dc90..437aadb 100644 --- a/tools/check_installed.py +++ b/tools/check_installed.py @@ -7,7 +7,7 @@ import unittest ROOT = Path(__file__).resolve().parents[1] -# A separate interpreter verifies that base packing does not import PyArrow. +# A separate interpreter checks that PyArrow stays unloaded during packing. subprocess.run( [sys.executable, "-I", "-c", """ import sys @@ -15,9 +15,9 @@ from importlib.metadata import version import pheon from pheon._native import library assert pheon.__version__ == version('pheon') -assert 'pyarrow' not in sys.modules assert library.parent == __import__('pathlib').Path(pheon.__file__).parent assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') +assert 'pyarrow' not in sys.modules """], check=True, ) From 9d5fe27516a59ad6f3b27d994a4fa33ad9c54491 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 12:37:10 -0800 Subject: [PATCH 28/41] Clarify diff report controls and README links Document the text limit, JSON omission counters, and hierarchy options. Use absolute links for the documentation referenced by the README. --- README.md | 22 +++++++++++++++------- docs/cli.md | 23 +++++++++++++---------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 5d035ba..10f3fc5 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ bindings and a C API. Comparisons ignore timestamps and element order while preserving duplicates and other object details. They compare object fields and coordinates exactly; see the -[comparison rules](docs/comparison.md) for supported normalization and limits. +[comparison rules][comparison] for supported normalization and limits. ## Try the command-line tools @@ -24,8 +24,8 @@ pheon hierarchy chip.gds --format mermaid ``` `diff` returns 0 for equality, 1 for differences, and 2 for invalid or incomplete -comparisons. Start with the [CLI guide](docs/cli.md), then set up -[Git integration](docs/git.md) if you review layouts in a repository. +comparisons. Start with the [CLI guide][cli], then set up +[Git integration][git] if you review layouts in a repository. ## Read layouts in Python @@ -45,13 +45,21 @@ print(layout.field("lib_name")[0].as_py()) ``` Source installation needs Rust; prebuilt Python wheels bundle the native library. -The [Python guide](docs/python.md) covers installation, selected reads, and integer -packing. The [Arrow reference](docs/arrow-schema.md) describes the returned data. +The [Python guide][python] covers installation, selected reads, and integer +packing. The [Arrow reference][arrow] describes the returned data. ## Learn more -Browse the [documentation](docs/index.md) for layer names, native integration, +Browse the [documentation][docs] for layer names, native integration, JSON reports, development, and releases. For writing, Pheon provides low-level GDSII record helpers in Rust. -Licensed under [GPL-3.0-only](LICENSE.md). +Licensed under [GPL-3.0-only][license]. + +[arrow]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/arrow-schema.md +[cli]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/cli.md +[comparison]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/comparison.md +[docs]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/index.md +[git]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/git.md +[license]: https://mpxd.net/code/jan/pheon/src/branch/master/LICENSE.md +[python]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/python.md diff --git a/docs/cli.md b/docs/cli.md index ffee2fc..437dee9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -99,18 +99,21 @@ pheon diff before.gds after.gds --hierarchy mermaid --depth 4 `--show-cell` and `--show-layer` filter displayed changes without changing the comparison, totals, or exit status. A layer display filter hides references -without a numeric layer. `--max-changes N` limits displayed object occurrences. -Text shows a replacement only when both objects fit within the limit. -A single omission line lists changes hidden by filters or the limit. JSON -includes all changed objects unless the limit is explicit. +without a numeric layer. `--max-changes N` limits displayed object occurrences +after filtering. Text defaults to 100 occurrences and shows a replacement only +when both objects fit within the limit. A single omission line lists changes +hidden by filters or the limit. JSON has no default object limit and reports +omissions from both display filters and any explicit limit in `report_selection`. +See the [JSON reference](diff-reports.md) for the selection counters. The diff hierarchy shows directly changed cells, affected ancestors, and targets of modified references. It includes references from both revisions so removed -branches remain visible. Counts represent direct placements; -arrays remain symbolic. ASCII omits the section when there are no cell changes -and marks reference changes whose placement count stays the same. -`--depth` limits text diagrams. For a complete graph in -JSON, use `--format json --hierarchy graph`; text depth and display filters do -not trim that graph. +branches remain visible. Counts represent direct placements, with symbolic +counts for arrays. ASCII omits the section when there are no cell changes and +marks reference changes whose placement count stays the same. Use `--depth` +with text output. +For the complete graph within the comparison scope, use +`--format json --hierarchy graph` without `--depth`. Display filters leave the +graph intact. Next: configure [Git integration](git.md) or add [layer names](layer-names.md). From ed1e0595bce2fee2305f01ec5fccc3eac0772b6b Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 20:12:42 -0800 Subject: [PATCH 29/41] Simplify text diff and hierarchy reports Use signed cell rows and keep whole-cell contents out of the text diff. Group reference changes, fold redundant terminal edges into the cell list, and inline one-off layer names. Count identical unmatched objects while retaining occurrence-based display limits. --- docs/cli.md | 38 ++- docs/native-api.md | 4 +- src/bin/pheon.rs | 16 +- src/review.rs | 409 +++++++++++++++++++++--------- src/review/changes.rs | 156 ++++++++++-- src/review/pair.rs | 2 +- tests/golden/review-diff.txt | 5 +- tests/golden/review-hierarchy.txt | 10 +- tests/review.rs | 361 +++++++++++++++++++++----- 9 files changed, 775 insertions(+), 226 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 437dee9..b22f4ca 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -46,13 +46,23 @@ pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 pheon diff before.gds after.gds --format json > report.json ``` -Text reports start with the result and counts of modified, added, and removed -cells. Added and removed cells show their object counts and group references by -target. Modified cells show counts of rectangles, polygons, and boxes changed, -moved, added, or removed on each layer. Repeated reference additions or removals -to the same target also appear as a count. Object details use `-` for removals, -`+` for additions, and `~` for likely replacements. Unmatched paths with five -or more vertices show their width, vertex count, and vertex bounds in one line. +Text reports start with the result. The headline includes cell counts for +reports with more than three changed cells, incomplete comparisons, or omitted +changes. The `Cells:` section lists removals with `-`, additions with `+`, and +modifications with `*`. Modified cells show counts of rectangles, polygons, and +boxes changed, moved, added, or removed on each layer. Repeated reference +additions or removals to the same target appear as a count. When both occur for +one target, they share a row. A complete set of edits to a target also shows +the total direct placements when there are unchanged placements, for example +`leaf · -3 refs, +6 refs · placements 10 -> 13`. + +An unchanged layer name appears inline when the layer occurs on one displayed +row, as in `1/0 (WG): 1 rectangle moved`. A layer on several displayed rows +gets one legend entry. Object details use `-` for removals, `+` for additions, +and `~` for likely replacements. Identical unmatched objects share a row +with a count such as `+ 12× 1/0: path …`. Unmatched paths with five or more +vertices show their width, vertex count, and vertex bounds in one line. + Use `--format json` for details of individual objects and `--hierarchy` to see which parent cells are affected. @@ -101,17 +111,21 @@ pheon diff before.gds after.gds --hierarchy mermaid --depth 4 comparison, totals, or exit status. A layer display filter hides references without a numeric layer. `--max-changes N` limits displayed object occurrences after filtering. Text defaults to 100 occurrences and shows a replacement only -when both objects fit within the limit. A single omission line lists changes -hidden by filters or the limit. JSON has no default object limit and reports +when both objects fit within the limit. Grouped rows count each object toward +the limit. A single omission line lists changes hidden by filters or the +limit. JSON has no default object limit and reports omissions from both display filters and any explicit limit in `report_selection`. See the [JSON reference](diff-reports.md) for the selection counters. The diff hierarchy shows directly changed cells, affected ancestors, and targets of modified references. It includes references from both revisions so removed branches remain visible. Counts represent direct placements, with symbolic -counts for arrays. ASCII omits the section when there are no cell changes and -marks reference changes whose placement count stays the same. Use `--depth` -with text output. +counts for arrays. ASCII uses the cell list's change markers and marks +reference changes whose placement count stays the same. When the cell list +shows all edits to an unchanged target on a single branch, ASCII can omit +that terminal edge. The hierarchy retains ancestor paths, shared branches, +and edges with edits hidden by display filters or limits. ASCII appears when +there are relationships to show. Use `--depth` with text output. For the complete graph within the comparison scope, use `--format json --hierarchy graph` without `--depth`. Display filters leave the graph intact. diff --git a/docs/native-api.md b/docs/native-api.md index f953d71..14f3c6e 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -33,8 +33,8 @@ single revision. A comparison captures reference counts and analyzes the union graph and ancestors when a hierarchy is requested. `TextComparison::between` captures the data needed for text and hierarchy -reports. Layered objects in added and removed cells contribute to the cell's -object count. +reports. Both comparison types provide `render_text_with_ascii_hierarchy` +to render a combined report with redundant terminal edges omitted. Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison that can reuse matching cells. Its `compare` and `compare_text` methods return diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 33a891e..ca92916 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -403,13 +403,19 @@ fn run(args: &[OsString]) -> Result { output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); } } - output.push_str(&diff.render_text(max_changes, &show, !old_missing && !new_missing)); - if format != "none" { + let show_library_presence = !old_missing && !new_missing; + if format == "ascii" { + output.push_str(&diff.render_text_with_ascii_hierarchy( + max_changes, + &show, + show_library_presence, + depth, + )); + } else { + output.push_str(&diff.render_text(max_changes, &show, show_library_presence)); if format == "mermaid" { output.push_str("Hierarchy\n```mermaid\n"); - } - output.push_str(&diff.hierarchy(format == "mermaid", depth)); - if format == "mermaid" { + output.push_str(&diff.hierarchy(true, depth)); output.push_str("```\n"); } } diff --git a/src/review.rs b/src/review.rs index 7c2ee16..05fadf5 100644 --- a/src/review.rs +++ b/src/review.rs @@ -886,7 +886,7 @@ impl<'a> Document<'a> { pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { let revision = Revision::from_document(self); - render_hierarchy(&revision, &revision, None, mermaid, max_depth) + render_hierarchy(&revision, &revision, None, mermaid, max_depth, None) } pub fn canonical_dump(&self) -> String { @@ -1194,7 +1194,7 @@ impl Revision { pub struct Comparison { pub old: Revision, pub new: Revision, - /// Unchanged names are displayed once when objects on that layer change. + /// Unchanged layer names available to the text report. pub layer_legend: BTreeMap, pub library_changes: Vec, pub layer_names: Vec, @@ -1202,12 +1202,10 @@ pub struct Comparison { pub diagnostics: Vec, } -/// A comparison for text and hierarchy reports. Layered objects in whole-cell -/// changes contribute to the cell's object count. +/// A comparison for text and hierarchy reports. #[derive(Debug)] pub struct TextComparison { comparison: Comparison, - whole_cell_objects: BTreeMap, } impl TextComparison { @@ -1216,11 +1214,8 @@ impl TextComparison { } fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self { - let mut whole_cell_objects = BTreeMap::new(); - let comparison = Comparison::collect(old, new, reused_cells, Some(&mut whole_cell_objects)); Self { - comparison, - whole_cell_objects, + comparison: Comparison::collect(old, new, reused_cells, false), } } @@ -1230,11 +1225,23 @@ impl TextComparison { show: &DisplayFilter, show_library_presence: bool, ) -> String { - self.comparison.render_text_with_counts( + self.comparison + .render_text(max_objects, show, show_library_presence) + } + + /// Render the cell changes and their ASCII hierarchy together. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.comparison.render_text_with_ascii_hierarchy( max_objects, show, show_library_presence, - &self.whole_cell_objects, + max_depth, ) } @@ -1249,14 +1256,14 @@ impl TextComparison { impl Comparison { pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { - Self::collect(old, new, &BTreeSet::new(), None) + Self::collect(old, new, &BTreeSet::new(), true) } fn collect( old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>, - mut whole_cell_objects: Option<&mut BTreeMap>, + include_whole_cell_objects: bool, ) -> Self { let mut result = Self { old: Revision::from_document(old), @@ -1350,27 +1357,20 @@ impl Comparison { added: Vec::new(), replacements: Vec::new(), }; - if let Some(counts) = whole_cell_objects.as_deref_mut() - && (a.is_none() || b.is_none()) - { + if !include_whole_cell_objects && (a.is_none() || b.is_none()) { let details = if a.is_some() { &mut change.removed } else { &mut change.added }; - let mut count = 0; - for object in a.or(b).into_iter().flat_map(|cell| &cell.objects) { - if object.metadata { - continue; - } - count += 1; - if object.layer.is_none() || object.target.is_some() { - details.push(object.into()); - } - } - if count != details.len() { - counts.insert(name.clone(), count); - } + // The hierarchy uses the changed reference targets. + details.extend( + a.or(b) + .into_iter() + .flat_map(|cell| &cell.objects) + .filter(|object| !object.metadata && object.target.is_some()) + .map(ObjectChange::from), + ); result.cells.push(change); continue; } @@ -1428,9 +1428,16 @@ impl Comparison { } } - /// Render the affected hierarchy. ASCII output is empty when no cells changed. + /// Render the affected hierarchy. ASCII output shows relationships between cells. pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { - render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) + render_hierarchy( + &self.old, + &self.new, + Some(&self.cells), + mermaid, + max_depth, + None, + ) } /// Limit presentation only: the complete comparison has already been made. @@ -1450,45 +1457,36 @@ impl Comparison { show: &DisplayFilter, show_library_presence: bool, ) -> String { - self.render_text_with_counts(max_objects, show, show_library_presence, &BTreeMap::new()) + self.render_text_report(max_objects, show, show_library_presence, None) } - fn render_text_with_counts( + /// Render the cell changes and omit simple hierarchy edges already shown there. + pub fn render_text_with_ascii_hierarchy( &self, max_objects: usize, show: &DisplayFilter, show_library_presence: bool, - whole_cell_objects: &BTreeMap, + max_depth: usize, ) -> String { - let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize); - for cell in &self.cells { - match (&cell.old, &cell.new) { - (None, Some(_)) => added += 1, - (Some(_), None) => removed += 1, - _ => modified += 1, - } - } - let mut out = match self.outcome() { + self.render_text_report(max_objects, show, show_library_presence, Some(max_depth)) + } + + fn render_text_report( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ascii_depth: Option, + ) -> String { + let outcome = self.outcome(); + let mut out = match outcome { Outcome::Equal => "Equal", Outcome::Different => "Different", Outcome::Incomplete { .. } => "Incomplete", Outcome::Failed => "Failed", } .to_string(); - let mut cell_counts = [ - (modified, "modified"), - (added, "added"), - (removed, "removed"), - ] - .into_iter() - .filter(|(count, _)| *count != 0); - if let Some((count, action)) = cell_counts.next() { - let noun = if count == 1 { "cell" } else { "cells" }; - write!(out, " · {count} {noun} {action}").unwrap(); - for (count, action) in cell_counts { - write!(out, ", {count} {action}").unwrap(); - } - } + let headline_end = out.len(); let library_changes = self.library_changes.iter().filter(|change| { show_library_presence || !matches!(change, LibraryChange::Presence { .. }) }); @@ -1543,51 +1541,49 @@ impl Comparison { .unwrap(); } let body_start = out.len(); - let mut shown_layers = BTreeSet::new(); + let mut removed_cells = String::new(); + let mut added_cells = String::new(); + let mut layers = changes::LayerRows::new(&self.layer_legend); + let mut annotations = Vec::new(); + let mut covered_edges = BTreeSet::new(); let mut detail_total = 0; let mut selected = 0; let mut shown: usize = 0; let mut hidden_cell_changes = 0; - let presence = |action, count| { - let objects = if count == 1 { "object" } else { "objects" }; - Cow::Owned(format!(" ({action}, {count} {objects})")) - }; + let visible = + |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); for cell in &self.cells { - let object_count = |details| { - whole_cell_objects - .get(&cell.name) - .copied() - .unwrap_or(details) - }; - let note = match (&cell.old, &cell.new) { - (None, Some(_)) => presence("added", object_count(cell.added.len())), - (Some(_), None) => presence("removed", object_count(cell.removed.len())), - _ if cell.attributes_changed() => Cow::Borrowed(" (attributes changed)"), - _ => Cow::Borrowed(""), - }; - let whole_cell = cell.old.is_none() || cell.new.is_none(); - // The heading accounts for layered objects in a whole-cell change. - detail_total += if whole_cell { - cell.removed - .iter() - .chain(&cell.added) - .filter(|object| object.layer.is_none()) - .count() - } else { - cell.removed.len() + cell.added.len() - }; + let marker = cell_marker(cell.old.is_some(), cell.new.is_some()); + let attributes_changed = marker == '*' && cell.attributes_changed(); + if marker == '*' { + detail_total += cell.removed.len() + cell.added.len(); + } if show.cell.as_ref().is_some_and(|name| name != &cell.name) { - hidden_cell_changes += usize::from(!note.is_empty()); + hidden_cell_changes += usize::from(marker != '*' || attributes_changed); continue; } + let name = cell_label(&cell.name); + match marker { + '-' => { + writeln!(removed_cells, "- {name}").unwrap(); + continue; + } + '+' => { + writeln!(added_cells, "+ {name}").unwrap(); + continue; + } + _ => (), + }; + let note = if attributes_changed { + " (attributes changed)" + } else { + "" + }; let cell_start = out.len(); let previously_shown = shown; - writeln!(out, "cell {:?}{note}", cell.name).unwrap(); - let visible = |object: &ObjectChange| { - (!whole_cell || object.layer.is_none()) - && show.layer.is_none_or(|layer| object.layer == Some(layer)) - }; + writeln!(out, "* {name}{note}").unwrap(); let mut polygons = changes::PolygonChanges::default(); + let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); let mut paired_old = BTreeSet::new(); let mut paired_new = BTreeSet::new(); for replacement in &cell.replacements { @@ -1599,18 +1595,28 @@ impl Comparison { if is_polygon(a.kind) { polygons.replacement(a, b); } else { - writeln!(out, " ~ {}", changes::replacement_summary(a, b)).unwrap(); + out.push_str(" ~ "); + let row_start = out.len(); + layers.record(a.layer, row_start); + writeln!(out, "{}", changes::replacement_summary(a, b)).unwrap(); + if let Some(target) = &a.target { + shown_refs + .entry(target) + .or_default() + .replacement(a, b, row_start); + } } - shown_layers.extend(a.layer); shown += 2; } } - for (removed, objects, paired) in [ - (true, &cell.removed, &paired_old), - (false, &cell.added, &paired_new), + let mut removed_refs = BTreeMap::new(); + let mut added_refs = BTreeMap::new(); + for (removed, objects, paired, references) in [ + (true, &cell.removed, &paired_old, &mut removed_refs), + (false, &cell.added, &paired_new, &mut added_refs), ] { let sign = if removed { "-" } else { "+" }; - let mut references = BTreeMap::new(); + let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); for (index, object) in objects.iter().enumerate() { if !visible(object) { continue; @@ -1624,33 +1630,102 @@ impl Comparison { .entry(target) .or_insert_with(|| (object, changes::ReferenceCounts::default())); counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; } else { - changes::write_unmatched(&mut out, sign, object); + unmatched.push((object, 1)); } - shown_layers.extend(object.layer); shown += 1; } } - for (target, (first, counts)) in references { - if whole_cell || counts.len() > 1 { - counts.write(&mut out, sign, target); - } else { - writeln!(out, " {sign} {first}").unwrap(); + // The comparison orders these objects by kind, fields, and properties. + for (object, count) in unmatched { + write!(out, " {sign} ").unwrap(); + if count > 1 { + write!(out, "{count}× ").unwrap(); + } + layers.record(object.layer, out.len()); + changes::write_unmatched(&mut out, object); + } + } + let no_refs = changes::ReferenceCounts::default(); + for (target, (first, counts)) in removed_refs { + let row_start = out.len(); + if let Some((_, added)) = added_refs.remove(target) { + counts.write_change(&mut out, target, &added); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &added, row_start); + } else { + counts.write(&mut out, "-", target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &no_refs, row_start); + } + } + for (target, (first, counts)) in added_refs { + let row_start = out.len(); + counts.write(&mut out, "+", target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&no_refs, &counts, row_start); + } + if !shown_refs.is_empty() { + for target in cell + .removed + .iter() + .chain(&cell.added) + .filter_map(|o| o.target.as_deref()) + { + if let Some(shown) = shown_refs.get_mut(target) { + shown.count_change(); + } + } + for (target, shown) in shown_refs { + if shown.complete() { + let edge = (cell.name.clone(), target.to_owned()); + let old = self.old.edges.get(&edge).copied().unwrap_or(0); + let new = self.new.edges.get(&edge).copied().unwrap_or(0); + shown.annotate(&out, &mut annotations, old, new); + if ascii_depth.is_some() && outcome.is_complete() { + covered_edges.insert(edge); + } } } } - polygons.write(&mut out); - if shown == previously_shown && note.is_empty() { + polygons.write(&mut out, &mut layers); + if shown == previously_shown && !attributes_changed { out.truncate(cell_start); } } - let mut legend = String::new(); - for layer in shown_layers { - if let Some(name) = self.layer_legend.get(&layer) { - writeln!(legend, "{}/{}: name {name:?}", layer.0, layer.1).unwrap(); + let mut before_cells = String::new(); + layers.finish(&out, &mut annotations, &mut before_cells); + // Resolve row offsets before inserting the section headers. + if !annotations.is_empty() { + annotations.sort_unstable_by_key(|(at, _)| *at); + let mut annotated = String::with_capacity(out.len()); + let mut start = 0; + for (at, annotation) in annotations { + annotated.push_str(&out[start..at]); + annotated.push_str(&annotation); + start = at; } + annotated.push_str(&out[start..]); + out = annotated; } - out.insert_str(body_start, &legend); + if out.len() > body_start || !removed_cells.is_empty() || !added_cells.is_empty() { + before_cells.push_str("Cells:\n"); + before_cells.push_str(&removed_cells); + before_cells.push_str(&added_cells); + } + out.insert_str(body_start, &before_cells); let mut omissions = Vec::new(); for (count, label, reason) in [ (detail_total - selected, "object change", "filtered out"), @@ -1668,6 +1743,36 @@ impl Comparison { &format!("Omitted · {}\n", omissions.join(" · ")), ); } + const COMPACT_HEADLINE_CELL_LIMIT: usize = 3; + if !self.cells.is_empty() + && (self.cells.len() > COMPACT_HEADLINE_CELL_LIMIT + || !outcome.is_complete() + || !omissions.is_empty()) + { + let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize); + for cell in &self.cells { + match cell_marker(cell.old.is_some(), cell.new.is_some()) { + '-' => removed += 1, + '+' => added += 1, + _ => modified += 1, + } + } + let mut counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + let mut summary = format!(" · {count} {noun} {action}"); + for (count, action) in counts { + write!(summary, ", {count} {action}").unwrap(); + } + out.insert_str(headline_end, &summary); + } + } // General comparison errors precede revision-specific errors in text. if !self.diagnostics.is_empty() { out.push_str("Diagnostics\n"); @@ -1680,6 +1785,16 @@ impl Comparison { writeln!(out, " {diagnostic}").unwrap(); } } + if let Some(max_depth) = ascii_depth { + out.push_str(&render_hierarchy( + &self.old, + &self.new, + Some(&self.cells), + false, + max_depth, + Some(&covered_edges), + )); + } out } } @@ -1745,6 +1860,7 @@ fn render_hierarchy( changes: Option<&[CellChange]>, mermaid: bool, max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, ) -> String { if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { return String::new(); @@ -1758,7 +1874,7 @@ fn render_hierarchy( affected, changed_refs, } = HierarchyAnalysis::between(old, new, changes); - let visible: BTreeSet<_> = if changes.is_some() { + let mut visible: BTreeSet<_> = if changes.is_some() { affected .iter() .cloned() @@ -1783,6 +1899,35 @@ fn render_hierarchy( targets.insert(target.clone()); } } + if let Some(covered) = covered_edges { + if !covered.is_empty() { + let mut incoming = BTreeMap::new(); + for target in children.values().flatten() { + *incoming.entry(target).or_insert(0_usize) += 1; + } + // Keep changed or shared descendants, branches, and target status notes. + let redundant: BTreeSet<_> = covered + .iter() + .map(|(_, target)| target) + .filter(|&target| { + incoming.get(target) == Some(&1) + && !children.contains_key(target) + && !affected.contains(target) + && old.cells.contains(target) + && new.cells.contains(target) + }) + .collect(); + children.retain(|_, children| { + children.retain(|target| !redundant.contains(target)); + !children.is_empty() + }); + targets.retain(|target| !redundant.contains(target)); + } + visible.retain(|name| children.contains_key(name) || targets.contains(name)); + } + if changes.is_some() && !mermaid && children.is_empty() { + return String::new(); + } let roots: Vec<_> = visible.difference(&targets).cloned().collect(); // Distinguish rootless cycles from descendants hidden by the depth limit. let mut reachable = BTreeSet::new(); @@ -1826,15 +1971,7 @@ fn render_hierarchy( " [unresolved]" } } else if direct.contains(name) { - if mermaid { - " [cell changed]" - } else if !old.known_cells.contains(name) { - " [added]" - } else if !new.known_cells.contains(name) { - " [removed]" - } else { - " [changed]" - } + if mermaid { " [cell changed]" } else { "" } } else if affected.contains(name) { " [affected]" } else if mermaid && changes.is_some() { @@ -1899,7 +2036,16 @@ fn render_hierarchy( new.scope.write_restrictions(&mut out); } for (name, depth, parent, seen, limited) in rows { - write!(out, "{}{name:?}", " ".repeat(depth)).unwrap(); + out.push_str(&" ".repeat(depth)); + if direct.contains(&name) { + let marker = cell_marker(old.cells.contains(&name), new.cells.contains(&name)); + write!(out, "{marker} ").unwrap(); + } + if changes.is_some() { + out.push_str(&cell_label(&name)); + } else { + write!(out, "{name:?}").unwrap(); + } if let Some(parent) = parent { write!(out, " {}", edge_label(&parent, &name)).unwrap(); } @@ -1919,6 +2065,25 @@ fn render_hierarchy( } } +fn cell_marker(old: bool, new: bool) -> char { + match (old, new) { + (true, false) => '-', + (false, true) => '+', + _ => '*', + } +} + +fn cell_label(name: &str) -> Cow<'_, str> { + if name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'?' | b'.' | b'-')) + { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("{name:?}")) + } +} + fn quoted(data: &[u8]) -> String { let mut out = "\"".to_string(); for byte in data { diff --git a/src/review/changes.rs b/src/review/changes.rs index 8131a66..0a5c8d3 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -112,7 +112,7 @@ fn object_xy(object: &ObjectChange) -> &[u8] { .map_or(&[], |field| field.data.as_slice()) } -pub(super) fn write_unmatched(out: &mut String, sign: &str, object: &ObjectChange) { +pub(super) fn write_unmatched(out: &mut String, object: &ObjectChange) { if object.kind == RTAG_PATH && let Some([xmin, ymin, xmax, ymax]) = object.bounds { @@ -124,13 +124,13 @@ pub(super) fn write_unmatched(out: &mut String, sign: &str, object: &ObjectChang .map_or_else(|| "absent".to_string(), |width| format!("{width} dbu")); writeln!( out, - " {sign} {heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" + "{heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" ) .unwrap(); return; } } - writeln!(out, " {sign} {object}").unwrap(); + writeln!(out, "{object}").unwrap(); } fn vertex_count(xy: &[u8]) -> usize { @@ -217,12 +217,14 @@ impl PolygonChanges { shapes.add(object, None); } - pub fn write(&self, out: &mut String) { + pub fn write(&self, out: &mut String, layers: &mut LayerRows<'_>) { for (layer, counts) in &self.0 { + out.push_str(" "); + layers.record(*layer, out.len()); if let Some((layer, datatype)) = layer { - write!(out, " {layer}/{datatype}: ").unwrap(); + write!(out, "{layer}/{datatype}: ").unwrap(); } else { - out.push_str(" unknown layer: "); + out.push_str("unknown layer: "); } let mut separator = ""; for (action, shapes) in [ @@ -261,41 +263,157 @@ pub(super) struct ReferenceCounts { } impl ReferenceCounts { - pub fn len(&self) -> usize { - self.singles + self.arrays - } - pub fn add(&mut self, object: &ObjectChange) { if object.kind == RTAG_AREF { self.arrays += 1; - if let Some([columns, rows]) = object.counts { - self.placements += u128::from(columns) * u128::from(rows); - } } else { self.singles += 1; - self.placements += 1; } + self.placements += reference_placements(object); } - pub fn write(&self, out: &mut String, sign: &str, target: &str) { + pub fn write(&self, out: &mut String, sign: &str, target: &str, first: &ObjectChange) { + if self.len() == 1 { + writeln!(out, " {sign} {first}").unwrap(); + return; + } write!(out, " {sign} ").unwrap(); + self.write_counts(out, ""); + write!(out, " to {}", cell_label(target)).unwrap(); + self.write_placements(out); + out.push('\n'); + } + + fn len(&self) -> usize { + self.singles + self.arrays + } + + pub fn write_change(&self, out: &mut String, target: &str, added: &Self) { + write!(out, " {} · ", cell_label(target)).unwrap(); + self.write_counts(out, "-"); + self.write_placements(out); + out.push_str(", "); + added.write_counts(out, "+"); + added.write_placements(out); + out.push('\n'); + } + + fn write_counts(&self, out: &mut String, sign: &str) { if self.singles > 0 { let suffix = if self.singles == 1 { "" } else { "s" }; - write!(out, "{} reference{suffix}", self.singles).unwrap(); + write!(out, "{sign}{} ref{suffix}", self.singles).unwrap(); } if self.arrays > 0 { if self.singles > 0 { out.push_str(", "); } let suffix = if self.arrays == 1 { "" } else { "s" }; - write!(out, "{} array reference{suffix}", self.arrays).unwrap(); + write!(out, "{sign}{} array ref{suffix}", self.arrays).unwrap(); } - write!(out, " to {target:?}").unwrap(); + } + + fn write_placements(&self, out: &mut String) { if self.arrays > 0 { let suffix = if self.placements == 1 { "" } else { "s" }; write!(out, " ({} placement{suffix})", self.placements).unwrap(); } - out.push('\n'); + } +} + +fn reference_placements(object: &ObjectChange) -> u128 { + match object.counts { + Some([columns, rows]) => u128::from(columns) * u128::from(rows), + None => 0, + } +} + +#[derive(Default)] +pub(super) struct ShownReferences { + shown: usize, + total: usize, + removed: u128, + added: u128, + row_start: usize, +} + +impl ShownReferences { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange, row_start: usize) { + self.shown += 2; + self.removed += reference_placements(old); + self.added += reference_placements(new); + self.row_start = row_start; + } + + pub fn unmatched( + &mut self, + removed: &ReferenceCounts, + added: &ReferenceCounts, + row_start: usize, + ) { + self.shown += removed.len() + added.len(); + self.removed += removed.placements; + self.added += added.placements; + self.row_start = row_start; + } + + pub fn count_change(&mut self) { + self.total += 1; + } + + pub fn complete(&self) -> bool { + self.shown == self.total + } + + pub fn annotate(&self, text: &str, annotations: &mut Vec<(usize, String)>, old: u64, new: u64) { + if (u128::from(old), u128::from(new)) == (self.removed, self.added) { + return; + } + let at = self.row_start + text[self.row_start..].find('\n').unwrap(); + let totals = if old == new { + let suffix = if new == 1 { "" } else { "s" }; + format!(" · {new} total placement{suffix}") + } else { + format!(" · placements {old} -> {new}") + }; + annotations.push((at, totals)); + } +} + +pub(super) struct LayerRows<'a> { + names: &'a BTreeMap, + rows: BTreeMap>, +} + +impl<'a> LayerRows<'a> { + pub fn new(names: &'a BTreeMap) -> Self { + Self { + names, + rows: BTreeMap::new(), + } + } + + pub fn record(&mut self, layer: Option, row_start: usize) { + if let Some(layer) = layer + && self.names.contains_key(&layer) + { + self.rows + .entry(layer) + .and_modify(|first| *first = None) + .or_insert(Some(row_start)); + } + } + + pub fn finish(self, text: &str, annotations: &mut Vec<(usize, String)>, legend: &mut String) { + for (layer, first) in self.rows { + let name = &self.names[&layer]; + if let Some(at) = first { + // Each recorded heading starts with the numeric layer/type and a colon. + let at = at + text[at..].find(':').unwrap(); + annotations.push((at, format!(" ({})", cell_label(name)))); + } else { + writeln!(legend, "{}/{}: name {name:?}", layer.0, layer.1).unwrap(); + } + } } } diff --git a/src/review/pair.rs b/src/review/pair.rs index a40128c..1b8352f 100644 --- a/src/review/pair.rs +++ b/src/review/pair.rs @@ -181,7 +181,7 @@ impl<'a> RevisionPair<'a> { } pub fn compare(self) -> Comparison { - Comparison::collect(&self.old, &self.new, &self.reused_cells, None) + Comparison::collect(&self.old, &self.new, &self.reused_cells, true) } pub fn compare_text(self) -> TextComparison { diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt index d493fa4..60cb697 100644 --- a/tests/golden/review-diff.txt +++ b/tests/golden/review-diff.txt @@ -1,3 +1,4 @@ -Different · 1 cell modified -cell "CHIP" +Different +Cells: +* CHIP 1/0: 1 polygon changed diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt index facae9e..ebaae5e 100644 --- a/tests/golden/review-hierarchy.txt +++ b/tests/golden/review-hierarchy.txt @@ -1,6 +1,6 @@ Hierarchy -"CHIP" [affected] - "left" x1 [affected] - "leaf" x2 [changed] - "right" x1 [affected] - "leaf" x3 [changed] [shown above] +CHIP [affected] + left x1 [affected] + * leaf x2 + right x1 [affected] + * leaf x3 [shown above] diff --git a/tests/review.rs b/tests/review.rs index 3c312e6..c30f5e2 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -513,11 +513,11 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { diff.hierarchy(false, 8), concat!( "Hierarchy\n", - "\"CHIP\" [affected]\n", - " \"left\" x1 [affected]\n", - " \"leaf\" x2 [changed]\n", - " \"right\" x1 [affected]\n", - " \"leaf\" x3 [changed] [shown above]\n", + "CHIP [affected]\n", + " left x1 [affected]\n", + " * leaf x2\n", + " right x1 [affected]\n", + " * leaf x3 [shown above]\n", ) ); let graph = diff.hierarchy(true, 8); @@ -525,8 +525,12 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { assert!(graph.contains("x2")); assert!(graph.contains("x3")); let limited = diff.hierarchy(false, 1); - assert!(!limited.contains("\"leaf\"")); + assert!(!limited.contains("leaf")); assert!(limited.contains("children omitted")); + assert_eq!( + diff.hierarchy(false, 0), + "Hierarchy\nCHIP [affected] [children omitted]\n" + ); } #[test] @@ -540,9 +544,9 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let diff = Comparison::between(&old, &new); let tree = diff.hierarchy(false, 8); let edge = if pair.name == "array count" { - " \"leaf\" x2 -> x3\n" + " leaf x2 -> x3\n" } else { - " \"leaf\" x1 [reference changed]\n" + " leaf x1 [reference changed]\n" }; assert!(tree.ends_with(edge), "{tree}"); } @@ -554,8 +558,8 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); for (old, new, edge) in [ - (&a, &b, " \"leaf\" x1 -> x0 [removed]\n"), - (&b, &a, " \"leaf\" x0 -> x1 [added]\n"), + (&a, &b, " - leaf x1 -> x0\n"), + (&b, &a, " + leaf x0 -> x1\n"), ] { let tree = Comparison::between(old, new).hierarchy(false, 8); assert!(tree.ends_with(edge), "{tree}"); @@ -778,6 +782,34 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { .unwrap() .starts_with("flowchart TD") ); + let with_reference = |x| layout(&[("CHIP", vec![reference("leaf", x, 1)]), ("leaf", vec![])]); + fs::write(&old, with_reference(0)).unwrap(); + fs::write(&new, with_reference(5)).unwrap(); + let moved = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); + assert_eq!(moved.status.code(), Some(1)); + assert_eq!( + moved.stdout, + b"Different\nCells:\n* CHIP\n ~ reference \"leaf\" moved by (5, 0) dbu\n" + ); + let scoped = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii", "--cell", "CHIP"]) + .output() + .unwrap(); + assert_eq!(scoped.status.code(), Some(1)); + assert!( + String::from_utf8(scoped.stdout) + .unwrap() + .ends_with("Hierarchy\n* CHIP\n leaf x1 [reference changed] [outside scope]\n") + ); fs::write(&new, b"invalid").unwrap(); assert_eq!( command() @@ -1139,7 +1171,7 @@ fn display_filters_never_change_equality_or_totals() { assert!(text.starts_with("Different · 1 cell modified\n")); assert!(text.contains("Display filter layer=42/0\n")); assert!(text.contains("Omitted · 2 object changes filtered out")); - assert!(!text.contains("cell \"CHIP\"")); + assert!(!text.contains("Cells:\n")); assert_eq!(diff.outcome().exit_code(), 1); } @@ -1152,24 +1184,31 @@ fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { let a = layout(&[ ("changed", vec![square()]), ("other", vec![square()]), - ("gone", vec![square(), triangle()]), + ( + "gone", + vec![square(), triangle(), reference("target", 0, 1)], + ), ("header", vec![]), + ("target", vec![]), ]); let b = layout(&[ ("changed", vec![triangle()]), ("other", vec![triangle()]), - ("new", vec![square()]), + ("new", vec![square(), reference("target", 0, 1)]), ("empty", vec![]), ("header", vec![attributes]), + ("target", vec![]), ]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); let text = diff.text(0); assert!(text.starts_with("Different · 3 cells modified, 2 added, 1 removed\n")); - assert!(text.contains("cell \"gone\" (removed, 2 objects)\n")); - assert!(text.contains("cell \"new\" (added, 1 object)\n")); - assert!(text.contains("cell \"empty\" (added, 0 objects)\n")); - assert!(text.contains("cell \"header\" (attributes changed)\n")); - assert!(!text.contains("cell \"changed\"")); + assert!(text.ends_with(concat!( + "Cells:\n", + "- gone\n", + "+ empty\n", + "+ new\n", + "* header (attributes changed)\n", + ))); assert!(text.contains("Omitted · 4 object changes over the limit")); let text = diff.text_filtered( @@ -1184,9 +1223,29 @@ fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { "Omitted · 2 object changes filtered out · 2 object changes over the limit", " · 4 cell-level changes filtered out\n", ))); - assert!(!text.contains("cell \"changed\"")); - assert!(!text.contains("cell \"other\"")); - assert!(!text.contains("cell \"new\"")); + assert!(!text.contains("Cells:\n")); +} + +#[test] +fn cell_list_quotes_names_with_whitespace() { + let old = layout(&[("old name", vec![]), ("same\nname", vec![square()])]); + let new = layout(&[("new_name", vec![]), ("same\nname", vec![triangle()])]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\n", + "Cells:\n", + "- \"old name\"\n", + "+ new_name\n", + "* \"same\\nname\"\n", + " 1/0: 1 polygon changed\n", + ) + ); + assert!(diff.hierarchy(false, 8).is_empty()); } #[test] @@ -1214,25 +1273,16 @@ fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps( let with_chip = Document::parse(&with_chip).unwrap(); let existing_chip = Document::parse(&existing_chip).unwrap(); - for (old, new, action, sign) in [ - (&empty, &with_chip, "added", "+"), - (&with_chip, &empty, "removed", "-"), - ] { + for (old, new, sign) in [(&empty, &with_chip, '+'), (&with_chip, &empty, '-')] { let diff = Comparison::between(old, new); let text_diff = TextComparison::between(old, new); let text = diff.text(100); - assert!(text.contains(&format!("cell \"CHIP\" ({action}, 6 objects)\n"))); - assert!(text.contains(&format!( - " {sign} 2 references, 1 array reference to \"leaf\" (4 placements)\n" - ))); - assert!(text.contains(&format!(" {sign} 1 reference to \"other\"\n"))); - assert!(!text.contains("xy=")); + assert!(text.ends_with(&format!("Cells:\n{sign} CHIP\n"))); + assert!(!text.contains("\n ")); assert!(!text.contains("1/0")); assert!(!text.contains("2/0")); assert!(!text.contains("Omitted")); - let limited = diff.text(2); - assert!(limited.contains(&format!(" {sign} 2 references to \"leaf\"\n"))); - assert!(limited.contains("Omitted · 2 object changes over the limit")); + assert_eq!(diff.text(0), text); assert_eq!(text_diff.outcome(), diff.outcome()); for mermaid in [false, true] { assert_eq!(text_diff.hierarchy(mermaid, 8), diff.hierarchy(mermaid, 8)); @@ -1263,7 +1313,7 @@ fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps( ] { let text = Comparison::between(old, new).text(100); assert!(text.contains(&format!( - " {sign} 2 references, 1 array reference to \"leaf\" (4 placements)\n" + " {sign} 2 refs, 1 array ref to leaf (4 placements)\n" ))); assert!(!text.contains("\"leaf\" xy=")); assert!(text.contains(&format!(" {sign} reference \"other\" xy=(10, 0)\n"))); @@ -1273,6 +1323,139 @@ fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps( assert!(dump.contains(" array reference \"leaf\" (2 × 1) xy=[(2, 0), (102, 0), (2, 0)]\n")); } +#[test] +fn reference_summaries_combine_visible_changes_to_a_target() { + let make_layout = |positions: std::ops::Range, moving_x| { + let mut refs: Vec<_> = positions + .chain(20..27) + .map(|x| reference("leaf", x, 1)) + .collect(); + refs.push(reference("moving", moving_x, 1)); + refs.push(reference("moving", 99, 1)); + layout(&[("CHIP", refs), ("leaf", vec![]), ("moving", vec![])]) + }; + let old = make_layout(0..3, 0); + let new = make_layout(10..16, 5); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " ~ reference \"moving\" moved by (5, 0) dbu · 2 total placements\n", + " leaf · -3 refs, +6 refs · placements 10 -> 13\n", + ) + ); + let ascii = + |max, show: &DisplayFilter| diff.render_text_with_ascii_hierarchy(max, show, true, 8); + assert_eq!(ascii(100, &Default::default()), diff.text(100)); + assert!(diff.text(5).ends_with(" - 3 refs to leaf\n")); + let limited = diff.text(6); + assert!( + limited.starts_with( + "Different · 1 cell modified\nOmitted · 5 object changes over the limit\n" + ) + ); + assert!(limited.ends_with(" leaf · -3 refs, +1 ref\n")); + let limited_ascii = ascii(6, &Default::default()); + assert!(limited_ascii.starts_with(&limited)); + assert!(limited_ascii.ends_with("Hierarchy\n* CHIP\n leaf x10 -> x13\n")); + + let filtered = ascii( + 100, + &DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ); + assert!(filtered.contains("11 object changes filtered out")); + assert!(filtered.ends_with(" leaf x10 -> x13\n moving x2 [reference changed]\n")); +} + +#[test] +fn combined_ascii_hierarchy_preserves_paths_to_changed_and_shared_cells() { + let make_layout = |x, shape| { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ( + "left", + vec![ + reference("shared", x, 1), + reference("changed", x, 1), + reference("local", x, 1), + ], + ), + ("right", vec![reference("shared", x, 1)]), + ("shared", vec![]), + ("changed", vec![shape]), + ("local", vec![]), + ]) + }; + let old = make_layout(0, square()); + let new = make_layout(5, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let text = diff.render_text_with_ascii_hierarchy(100, &Default::default(), true, 8); + let (cells, hierarchy) = text.split_once("Hierarchy\n").unwrap(); + assert!(cells.contains("reference \"local\" moved by (5, 0) dbu")); + assert_eq!( + hierarchy, + concat!( + "CHIP [affected]\n", + " * left x1\n", + " * changed x1 [reference changed]\n", + " shared x1 [reference changed]\n", + " * right x1\n", + " shared x1 [reference changed] [shown above]\n", + ) + ); +} + +#[test] +fn combined_reference_summaries_keep_array_counts_and_placements() { + let old = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 0, 1), + reference("leaf", 10, 2), + reference("leaf", 20, 3), + ], + ), + ("leaf", vec![]), + ]); + let new = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 50, 1), + reference("leaf", 60, 1), + reference("leaf", 70, 2), + reference("leaf", 80, 4), + ], + ), + ("leaf", vec![]), + ]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " leaf · -1 ref, -2 array refs (6 placements), +2 refs, +2 array refs (8 placements)\n", + ) + ); +} + #[test] fn git_external_driver_handles_changes_additions_deletions_and_errors() { let directory = std::env::temp_dir().join(format!( @@ -1340,7 +1523,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!( String::from_utf8(changed.stdout) .unwrap() - .contains("Different · 1 cell modified\n") + .contains("Different\nCells:\n* CHIP\n") ); fs::write(&path, b"invalid").unwrap(); let invalid = Command::new("git") @@ -1372,15 +1555,17 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let result = git(&["diff", "--cached", "--", "added layout.gds"]); let text = String::from_utf8(result.stdout).unwrap(); assert!(text.starts_with("layout \"added layout.gds\" added\nDifferent · 4 cells added\n")); + assert!(text.contains("Cells:\n+ CHIP\n+ leaf\n+ left\n+ right\nHierarchy\n")); assert!(!text.contains("\nmode ")); assert!(!text.contains("\nlibrary ")); fs::remove_file(&path).unwrap(); git(&["add", "-u"]); let result = git(&["diff", "--cached", "--", "chip layout.gds"]); let text = String::from_utf8(result.stdout).unwrap(); - assert!(text.starts_with("layout \"chip layout.gds\" deleted\nDifferent · 1 cell removed\n")); - assert!(!text.contains("\nmode ")); - assert!(!text.contains("\nlibrary ")); + assert_eq!( + text, + "layout \"chip layout.gds\" deleted\nDifferent\nCells:\n- CHIP\n" + ); // Textconv is an alternative to the external driver, with a separate cache. git(&["commit", "-qm", "replace fixture"]); git(&["mv", "added layout.gds", "renamed layout.gds"]); @@ -1390,7 +1575,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let result = git(&["diff", "--cached", "--find-renames"]); let text = String::from_utf8(result.stdout).unwrap(); assert!(text.contains("renamed layout.gds"), "{text}"); - assert!(text.contains("Different · 1 cell modified\n"), "{text}"); + assert!(text.contains("Different\nCells:\n* leaf\n"), "{text}"); git(&["commit", "-qm", "rename fixture"]); fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); fs::write(&renamed, shared(square())).unwrap(); @@ -1526,8 +1711,8 @@ fn polygon_diff_groups_shapes_by_layer_across_dimensions() { assert_eq!( diff.text(100), concat!( - "Different · 1 cell modified\n", - "cell \"CHIP\"\n", + "Different\n", + "Cells:\n* CHIP\n", " 1/0: 2 rectangles, 1 polygon changed\n", " 2/0: 1 polygon changed\n", ) @@ -1580,7 +1765,11 @@ fn unmatched_paths_summarize_coordinates_that_would_wrap() { " {sign} 1/0: path · width {width} · 5 vertices · bounds (0, -5) to (13, 9) dbu\n" ))); } - assert_eq!(text.lines().count(), 5, "{text}"); + assert_eq!( + text.lines().filter(|line| line.starts_with(" ")).count(), + 3, + "{text}" + ); assert!(text.contains("xy=[(0, 0), (4, 0), (4, -5), (13, -5)]")); } assert!(paths.canonical_dump().contains("\n (13, 9)]")); @@ -1596,6 +1785,41 @@ fn unmatched_paths_summarize_coordinates_that_would_wrap() { ); } +#[test] +fn repeated_unmatched_rows_group_only_identical_objects_within_the_limit() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let mut other_points = points; + other_points[1][0] = 5; + let named_paths = |mut paths: Vec>| { + paths.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", paths)]) + }; + let empty = named_paths(vec![]); + let paths = named_paths(vec![ + path(None, &points), + path(None, &other_points), + path(None, &points), + ]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let diff = Comparison::between(old, new); + let details = "path · width absent · 5 vertices · bounds (0, -5) to (13, 9) dbu"; + let full = diff.text(3); + assert!(full.contains("1/0: name \"WG\"\nCells:\n")); + assert!(full.ends_with(&format!( + " {sign} 2× 1/0: {details}\n {sign} 1/0: {details}\n" + ))); + assert!( + diff.text(2) + .ends_with(&format!(" {sign} 2× 1/0 (WG): {details}\n")) + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(limited.ends_with(&format!(" {sign} 1/0 (WG): {details}\n"))); + } +} + #[test] fn label_replacements_show_only_changed_fields() { let a = layout(&[("CHIP", vec![label((2, 0), "old")])]); @@ -1603,29 +1827,50 @@ fn label_replacements_show_only_changed_fields() { let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); assert_eq!( diff.text(100), - "Different · 1 cell modified\ncell \"CHIP\"\n ~ 2/0: label text \"old\" -> \"new\"\n" + "Different\nCells:\n* CHIP\n ~ 2/0: label text \"old\" -> \"new\"\n" ); } #[test] fn unchanged_layer_names_explain_geometry_changes_once() { - let named = |geometry| { - layout(&[( - "CHIP", - vec![ - geometry, - label((900, 7), LAYER_SENTINEL), - label((900, 7), "1:0:WG"), - ], - )]) + let named = |geometry, text| { + layout(&[ + ( + "CHIP", + vec![ + geometry, + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + ), + ("leaf", vec![label((1, 0), text)]), + ]) }; - let a = named(square()); - let b = named(triangle()); - let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); - let text = diff.text(100); - assert_eq!(text.matches("1/0: name \"WG\"").count(), 1); + let old = named(square(), "old"); + let a = Document::parse(&old).unwrap(); + let new = named(triangle(), "old"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(100), + "Different\nCells:\n* CHIP\n 1/0 (WG): 1 polygon changed\n" + ); assert!(diff.layer_names.is_empty()); + let new = named(triangle(), "new"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + let text = diff.text(100); + assert!(text.contains("1/0: name \"WG\"\nCells:\n")); + assert!(!text.contains("(WG)")); + assert!(diff.text(2).ends_with(" 1/0 (WG): 1 polygon changed\n")); + let show = DisplayFilter { + cell: Some("leaf".into()), + layer: None, + }; + assert!( + diff.text_filtered(100, &show) + .ends_with(" ~ 1/0 (WG): label text \"old\" -> \"new\"\n") + ); + let names = corpus() .into_iter() .find(|case| case.name == "layer name only") From 28b7f10f6eca4fe478789a5f6c2542b5c334c74f Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 22:58:12 -0800 Subject: [PATCH 30/41] Add terminal colors and clarify elided hierarchy branches --- Cargo.lock | 42 ++++++- Cargo.toml | 1 + docs/cli.md | 18 ++- docs/git.md | 3 + src/bin/pheon.rs | 36 ++++-- src/bin/pheon/color.rs | 195 ++++++++++++++++++++++++++++++ src/review.rs | 30 ++--- tests/golden/review-hierarchy.txt | 2 +- tests/review.rs | 133 +++++++++++++++++++- 9 files changed, 421 insertions(+), 39 deletions(-) create mode 100644 src/bin/pheon/color.rs diff --git a/Cargo.lock b/Cargo.lock index 3c11424..cfec364 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,7 +133,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-link", + "windows-link 0.1.1", ] [[package]] @@ -378,6 +378,7 @@ dependencies = [ "flate2", "serde", "serde_json", + "termcolor", ] [[package]] @@ -465,6 +466,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -550,6 +560,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-core" version = "0.61.0" @@ -558,7 +577,7 @@ checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.1", "windows-result", "windows-strings", ] @@ -591,13 +610,19 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-result" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-link", + "windows-link 0.1.1", ] [[package]] @@ -606,7 +631,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" dependencies = [ - "windows-link", + "windows-link 0.1.1", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 11a6afe..93004b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ arrow-schema = "^54" flate2 = "1" serde = "1" serde_json = "1" +termcolor = "1.4" [profile.release] diff --git a/docs/cli.md b/docs/cli.md index b22f4ca..d4f59c3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -32,11 +32,15 @@ pheon hierarchy chip.gds --format mermaid --top CHIP normalized object within a cell. Coordinates are in database units. Rectangles show two corners, and longer vertex lists wrap across lines. References show the target and any array dimensions before the coordinates. Add `--records` -to inspect the original record sequence, including timestamps. +to inspect the original record sequence, including timestamps. In color +output, the object dump highlights cell headings and layer numbers, and the +record dump highlights record names. -`hierarchy` shows direct placement counts on references. Shared cells are -marked where their contents appear earlier in the tree. Mermaid output is -diagram source for a Mermaid renderer. +`hierarchy` shows direct placement counts on references. `[subtree elided]` +marks a repeated cell whose children were skipped. Repeated leaves show +each parent's placement count without the marker. At the depth limit, +`[children omitted]` marks a cell with further descendants. Mermaid +output is diagram source for a Mermaid renderer. ## Compare revisions @@ -66,6 +70,12 @@ vertices show their width, vertex count, and vertex bounds in one line. Use `--format json` for details of individual objects and `--hierarchy` to see which parent cells are affected. +`diff` and `dump` use `--color auto` by default. Colors appear on supported +terminals and are disabled when `NO_COLOR` is set or `TERM=dumb`. Use +`--color=always` to keep colors in a pipe, for example +`pheon diff before.gds after.gds --color=always | less -R`. Use +`--color=never` for plain text. JSON reports and Mermaid diagram source stay plain. + For large unscoped diffs with substantial byte overlap, Pheon reuses parsed contents of matching cells. Cell timestamps and file offsets can differ. diff --git a/docs/git.md b/docs/git.md index d17aadb..2a571d0 100644 --- a/docs/git.md +++ b/docs/git.md @@ -28,6 +28,9 @@ indicates whether it was added or deleted. Keep the final `--` because Git appen its path and revision arguments after it. Put Pheon options such as `--top CHIP` or `--show-layer 1/0` before that marker. +The external driver also accepts `--color=always` before `--` to keep colors +when its output goes through a pager that supports ANSI colors. + For an existing `diff=layout` attribute, put the configuration under `[diff "layout"]` instead. Attribute names only select a Git driver. diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index ca92916..309fd6b 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,3 +1,6 @@ +#[path = "pheon/color.rs"] +mod color; + use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ @@ -14,10 +17,12 @@ use std::{ process::ExitCode, thread, }; +use termcolor::ColorChoice; const HELP: &str = "Usage: - pheon dump FILE [--records] + pheon dump FILE [--records] [--color auto|always|never] pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T]\n [--show-cell NAME] [--show-layer L/T] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] + [--color auto|always|never] pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... @@ -268,6 +273,8 @@ fn run(args: &[OsString]) -> Result { _ => return Err(HELP.to_string()), }; let compare = command == "diff" || command == "git-diff"; + let color_supported = compare || command == "dump"; + let mut color_choice = ColorChoice::Auto; let mut scope = Scope::default(); let mut show = DisplayFilter::default(); let mut raw = false; @@ -297,6 +304,12 @@ fn run(args: &[OsString]) -> Result { return Ok(0); } Some("--records") if command == "dump" => raw = true, + Some("--color") if color_supported => { + color_choice = color::parse_choice(&value(args, &mut i)?)?; + } + Some(s) if color_supported && s.starts_with("--color=") => { + color_choice = color::parse_choice(&s["--color=".len()..])?; + } Some("--top") => scope.top = Some(value(args, &mut i)?), Some("--cell") => scope.cell = Some(value(args, &mut i)?), Some("--layer") => scope.layer = Some(parse_layer(&value(args, &mut i)?)?), @@ -354,19 +367,22 @@ fn run(args: &[OsString]) -> Result { if scope != Scope::default() { return Err("record dumps do not accept object scope selectors".to_string()); } - stdout(&dump_records(&read(&paths[0])?)?)?; + color::print( + &dump_records(&read(&paths[0])?)?, + color::Report::Records, + color_choice, + ) + .map_err(|error| error.to_string())?; return Ok(0); } if !compare { let bytes = read(&paths[0])?; let doc = Document::parse(&bytes)?.selected(&scope, false)?; - let output = if command == "dump" { - doc.canonical_dump() + if command == "dump" { + color::print(&doc.canonical_dump(), color::Report::Dump, color_choice) + .map_err(|error| error.to_string())?; } else { - doc.hierarchy(format == "mermaid", depth) - }; - stdout(&output)?; - if command == "hierarchy" { + stdout(&doc.hierarchy(format == "mermaid", depth))?; for diagnostic in &doc.diagnostics { eprintln!("INCOMPLETE: {}", diagnostic.message); } @@ -419,7 +435,7 @@ fn run(args: &[OsString]) -> Result { output.push_str("```\n"); } } - stdout(&output)?; + color::print(&output, color::Report::Diff, color_choice).map_err(|error| error.to_string())?; // Git's external driver expects successful rendering for either equality // or differences. Actual comparison/input failures remain nonzero. Ok(if git && diff.outcome().exit_code() == 1 { @@ -444,7 +460,7 @@ fn wants_json(args: &[OsString]) -> bool { } Some( "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" - | "--max-changes" | "--depth", + | "--max-changes" | "--depth" | "--color", ) => i += 1, _ => (), } diff --git a/src/bin/pheon/color.rs b/src/bin/pheon/color.rs new file mode 100644 index 0000000..e2061d7 --- /dev/null +++ b/src/bin/pheon/color.rs @@ -0,0 +1,195 @@ +use std::{ + fmt, + io::{self, IsTerminal, Write}, +}; +use termcolor::{BufferedStandardStream, Color, ColorChoice, ColorSpec, WriteColor}; + +pub(super) enum Report { + Diff, + Dump, + Records, +} + +pub(super) fn parse_choice(value: &str) -> Result { + match value { + "auto" => Ok(ColorChoice::Auto), + "always" => Ok(ColorChoice::Always), + "never" => Ok(ColorChoice::Never), + _ => Err("color must be auto, always, or never".to_string()), + } +} + +pub(super) fn print(text: &str, report: Report, choice: ColorChoice) -> io::Result<()> { + let stdout = io::stdout(); + if choice == ColorChoice::Never || (choice == ColorChoice::Auto && !stdout.is_terminal()) { + return stdout.lock().write_all(text.as_bytes()); + } + let mut out = BufferedStandardStream::stdout(choice); + if !out.supports_color() { + out.write_all(text.as_bytes())?; + return out.flush(); + } + + let mut in_cells = false; + let mut in_mermaid = false; + for line in text.split_inclusive('\n') { + let body = line.strip_suffix('\n').unwrap_or(line); + match report { + Report::Diff => match body { + "```mermaid" => { + in_mermaid = true; + out.write_all(body.as_bytes())?; + } + "```" => { + in_mermaid = false; + out.write_all(body.as_bytes())?; + } + _ if in_mermaid => out.write_all(body.as_bytes())?, + _ => { + match body { + "Cells:" => in_cells = true, + "Hierarchy" | "Diagnostics" => in_cells = false, + _ => (), + } + diff_line(&mut out, body, in_cells)?; + } + }, + Report::Dump => dump_line(&mut out, body)?, + Report::Records => record_line(&mut out, body)?, + } + if line.ends_with('\n') { + out.write_all(b"\n")?; + } + } + out.flush() +} + +fn paint( + out: &mut impl WriteColor, + value: impl fmt::Display, + color: Color, + bold: bool, +) -> io::Result<()> { + out.set_color(ColorSpec::new().set_fg(Some(color)).set_bold(bold))?; + let result = write!(out, "{value}"); + result.and(out.reset()) +} + +fn diff_line(out: &mut impl WriteColor, line: &str, in_cells: bool) -> io::Result<()> { + let content = line.trim_start_matches(' '); + let indent = &line[..line.len() - content.len()]; + let first = content.split_once(' ').map_or(content, |(word, _)| word); + if indent.is_empty() { + let color = match first { + "Equal" => Some(Color::Green), + "Different" | "Incomplete" | "Omitted" => Some(Color::Yellow), + "Failed" => Some(Color::Red), + "Cells:" | "Hierarchy" | "Diagnostics" | "Scope" | "Display" | "layout" | "mode" => { + Some(Color::Cyan) + } + _ => None, + }; + if let Some(color) = color { + return paint(out, line, color, true); + } + } + let color = match first { + "+" => Some(Color::Green), + "-" => Some(Color::Red), + "*" | "~" => Some(Color::Yellow), + _ => None, + }; + if let Some(color) = color { + out.write_all(indent.as_bytes())?; + return paint(out, content, color, false); + } + if in_cells && indent == " " { + if (layer_token(content).is_some() || content.starts_with("unknown layer: ")) + && let Some((layer, counts)) = content.rsplit_once(": ") + { + out.write_all(indent.as_bytes())?; + paint(out, layer, Color::Cyan, false)?; + out.write_all(b": ")?; + for (index, clause) in counts.split(" · ").enumerate() { + if index > 0 { + out.write_all(" · ".as_bytes())?; + } + let color = match clause.rsplit_once(' ').map(|(_, action)| action) { + Some("added") => Color::Green, + Some("removed") => Color::Red, + _ => Color::Yellow, + }; + paint(out, clause, color, false)?; + } + return Ok(()); + } + if let Some((target, counts)) = content.rsplit_once(" · -") + && let Some((removed, added)) = counts.split_once(", +") + { + let (added, totals) = added + .split_once(" · ") + .map_or((added, None), |(added, totals)| (added, Some(totals))); + write!(out, "{indent}{target} · ")?; + paint(out, format_args!("-{removed}"), Color::Red, false)?; + out.write_all(b", ")?; + paint(out, format_args!("+{added}"), Color::Green, false)?; + if let Some(totals) = totals { + write!(out, " · {totals}")?; + } + return Ok(()); + } + } + out.write_all(line.as_bytes()) +} + +fn layer_token(line: &str) -> Option<&str> { + let token = line.split([' ', ':']).next()?; + let (layer, datatype) = token.split_once('/')?; + (layer.parse::().is_ok() && datatype.parse::().is_ok()).then_some(token) +} + +fn dump_line(out: &mut impl WriteColor, line: &str) -> io::Result<()> { + if line.starts_with("cell ") || line.starts_with("library ") || line.starts_with("Scope ") { + return paint(out, line, Color::Cyan, true); + } + if line.starts_with("INCOMPLETE: ") { + return paint(out, line, Color::Yellow, true); + } + let mut object = line.trim_start_matches(' '); + if let Some((token, rest)) = object.split_once(' ') + && token + .strip_suffix('x') + .is_some_and(|count| count.parse::().is_ok()) + { + object = rest; + } + let heading = layer_token(object).or_else(|| { + ["reference", "array reference"] + .into_iter() + .find(|heading| { + object + .strip_prefix(heading) + .is_some_and(|rest| rest.starts_with(' ')) + }) + }); + if let Some(heading) = heading { + let prefix = &line[..line.len() - object.len()]; + out.write_all(prefix.as_bytes())?; + paint(out, heading, Color::Cyan, false)?; + return out.write_all(object[heading.len()..].as_bytes()); + } + out.write_all(line.as_bytes()) +} + +fn record_line(out: &mut impl WriteColor, line: &str) -> io::Result<()> { + let mut parts = line.splitn(4, ' '); + if let (Some(offset), Some(tag), Some(name), Some(data)) = + (parts.next(), parts.next(), parts.next(), parts.next()) + { + write!(out, "{offset} {tag} ")?; + paint(out, name, Color::Cyan, false)?; + write!(out, " {data}") + } else { + out.write_all(line.as_bytes()) + } +} diff --git a/src/review.rs b/src/review.rs index 05fadf5..9e49dd4 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1947,13 +1947,16 @@ fn render_hierarchy( while let Some((name, depth, parent)) = stack.pop() { let seen = !included.insert(name.clone()); let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); - rows.push(( - name.clone(), - depth, - parent, - seen, - depth >= max_depth && !descendants.is_empty(), - )); + let note = if descendants.is_empty() { + "" + } else if depth >= max_depth { + " [children omitted]" + } else if seen { + " [subtree elided]" + } else { + "" + }; + rows.push((name.clone(), depth, parent, note)); if !seen && depth < max_depth { for child in descendants.iter().rev() { stack.push((child.clone(), depth + 1, Some(name.clone()))); @@ -1961,7 +1964,7 @@ fn render_hierarchy( } } for name in visible.difference(&reachable) { - rows.push((name.clone(), 0, None, false, true)); + rows.push((name.clone(), 0, None, " [children omitted]")); } let status = |name: &String| { if !old.cells.contains(name) && !new.cells.contains(name) { @@ -2035,7 +2038,7 @@ fn render_hierarchy( if changes.is_none() { new.scope.write_restrictions(&mut out); } - for (name, depth, parent, seen, limited) in rows { + for (name, depth, parent, note) in rows { out.push_str(&" ".repeat(depth)); if direct.contains(&name) { let marker = cell_marker(old.cells.contains(&name), new.cells.contains(&name)); @@ -2049,14 +2052,7 @@ fn render_hierarchy( if let Some(parent) = parent { write!(out, " {}", edge_label(&parent, &name)).unwrap(); } - writeln!( - out, - "{}{}{}", - status(&name), - if seen { " [shown above]" } else { "" }, - if limited { " [children omitted]" } else { "" } - ) - .unwrap(); + writeln!(out, "{}{note}", status(&name)).unwrap(); } if visible.is_empty() { out.push_str("No changed hierarchy\n"); diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt index ebaae5e..60542a2 100644 --- a/tests/golden/review-hierarchy.txt +++ b/tests/golden/review-hierarchy.txt @@ -3,4 +3,4 @@ CHIP [affected] left x1 [affected] * leaf x2 right x1 [affected] - * leaf x3 [shown above] + * leaf x3 diff --git a/tests/review.rs b/tests/review.rs index c30f5e2..2b51f45 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -503,7 +503,7 @@ fn unknown_records_never_produce_a_successful_equality_claim() { } #[test] -fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { +fn shared_hierarchy_reports_ancestors_and_each_leaf_placement_count() { let a = shared(square()); let b = shared(triangle()); let a = Document::parse(&a).unwrap(); @@ -517,7 +517,7 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { " left x1 [affected]\n", " * leaf x2\n", " right x1 [affected]\n", - " * leaf x3 [shown above]\n", + " * leaf x3\n", ) ); let graph = diff.hierarchy(true, 8); @@ -533,6 +533,44 @@ fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { ); } +#[test] +fn ascii_hierarchy_marks_repeated_subtrees_and_depth_limits() { + let bytes = layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("branch", 0, 2)]), + ("right", vec![reference("branch", 0, 3)]), + ("branch", vec![reference("leaf", 0, 1)]), + ("leaf", vec![]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2\n", + " \"leaf\" x1\n", + " \"right\" x1\n", + " \"branch\" x3 [subtree elided]\n", + ) + ); + assert_eq!( + doc.hierarchy(false, 2), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2 [children omitted]\n", + " \"right\" x1\n", + " \"branch\" x3 [children omitted]\n", + ) + ); +} + #[test] fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { for pair in corpus() @@ -824,6 +862,95 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { ); } +#[test] +fn cli_color_preserves_plain_output_and_machine_formats() { + let directory = std::env::temp_dir().join(format!("pheon color {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let with_changes = |name, shape, positions: std::ops::Range| { + let objects = positions + .map(|x| reference("leaf", x, 1)) + .chain([shape]) + .collect(); + layout(&[("CHIP", objects), ("leaf", vec![]), (name, vec![])]) + }; + fs::write(&old, with_changes("obsolete", square(), 0..2)).unwrap(); + fs::write(&new, with_changes("new", triangle(), 10..13)).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + let output = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + String::from_utf8(output.stdout).unwrap() + }; + let plain = diff(&["--hierarchy", "mermaid"]); + let colored = diff(&["--hierarchy", "mermaid", "--color=always"]); + for span in [ + "\x1b[31m- obsolete", + "\x1b[32m+ new", + "\x1b[33m* CHIP", + "\x1b[31m-2 refs", + "\x1b[32m+3 refs", + "\x1b[33m1 polygon changed", + ] { + assert!(colored.contains(span), "{span:?} in {colored:?}"); + } + let strip_ansi = |text: &str| { + let mut parts = text.split("\x1b["); + let mut plain = parts.next().unwrap().to_string(); + for part in parts { + let (code, text) = part.split_once('m').unwrap(); + assert!(code.bytes().all(|b| b.is_ascii_digit() || b == b';')); + plain.push_str(text); + } + plain + }; + assert_eq!(strip_ansi(&colored), plain); + assert_eq!(diff(&["--hierarchy", "mermaid", "--color", "never"]), plain); + assert_eq!( + colored.split_once("```mermaid\n").unwrap().1, + plain.split_once("```mermaid\n").unwrap().1 + ); + assert_eq!( + diff(&["--color", "always", "--format", "json"]), + diff(&["--format", "json"]) + ); + + for (options, token) in [ + (&[][..], "\x1b[36mcell \"CHIP\""), + (&["--records"][..], "\x1b[36mBGNSTR"), + ] { + let dump = |choice| { + let output = command() + .arg("dump") + .arg(&old) + .args(options) + .args(["--color", choice]) + .output() + .unwrap(); + assert!(output.status.success()); + String::from_utf8(output.stdout).unwrap() + }; + let colored = dump("always"); + assert!(colored.contains(token), "{colored:?}"); + assert_eq!(strip_ansi(&colored), dump("auto")); + assert_eq!(dump("auto"), dump("never")); + } +} + #[test] fn cli_decodes_gzip_by_header_and_checks_the_trailer() { let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); @@ -1413,7 +1540,7 @@ fn combined_ascii_hierarchy_preserves_paths_to_changed_and_shared_cells() { " * changed x1 [reference changed]\n", " shared x1 [reference changed]\n", " * right x1\n", - " shared x1 [reference changed] [shown above]\n", + " shared x1 [reference changed]\n", ) ); } From 4789638e224231c3fe6758619b8e5fae8554043d Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 23:01:57 -0800 Subject: [PATCH 31/41] Show the displacement of a single shape move --- docs/cli.md | 12 +++++---- src/review/changes.rs | 46 ++++++++++++++++++++++++------- tests/review.rs | 63 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 94 insertions(+), 27 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index d4f59c3..045ff91 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -54,15 +54,17 @@ Text reports start with the result. The headline includes cell counts for reports with more than three changed cells, incomplete comparisons, or omitted changes. The `Cells:` section lists removals with `-`, additions with `+`, and modifications with `*`. Modified cells show counts of rectangles, polygons, and -boxes changed, moved, added, or removed on each layer. Repeated reference -additions or removals to the same target appear as a count. When both occur for -one target, they share a row. A complete set of edits to a target also shows +boxes changed, moved, added, or removed on each layer. When the only displayed +shape change on a layer is a move, the row shows its displacement in database +units. Repeated reference additions or removals to the same target appear as a +count. When both occur for one target, they share a row. A complete set of edits +to a target also shows the total direct placements when there are unchanged placements, for example `leaf · -3 refs, +6 refs · placements 10 -> 13`. An unchanged layer name appears inline when the layer occurs on one displayed -row, as in `1/0 (WG): 1 rectangle moved`. A layer on several displayed rows -gets one legend entry. Object details use `-` for removals, `+` for additions, +row, as in `1/0 (WG): rectangle moved by (20, -5) dbu`. A layer on several +displayed rows gets one legend entry. Object details use `-` for removals, `+` for additions, and `~` for likely replacements. Identical unmatched objects share a row with a count such as `+ 12× 1/0: path …`. Unmatched paths with five or more vertices show their width, vertex count, and vertex bounds in one line. diff --git a/src/review/changes.rs b/src/review/changes.rs index 0a5c8d3..2f0a6cd 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -169,6 +169,10 @@ struct ShapeCounts { } impl ShapeCounts { + fn total(&self) -> usize { + self.rectangles + self.polygons + self.boxes + } + fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) { let is_rectangle = |object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some(); @@ -188,6 +192,7 @@ struct LayerPolygonChanges { moved: ShapeCounts, removed: ShapeCounts, added: ShapeCounts, + displacement: Option<[i64; 2]>, } #[derive(Default)] @@ -196,15 +201,18 @@ pub(super) struct PolygonChanges(BTreeMap, LayerPolygonChanges>); impl PolygonChanges { pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) { let counts = self.0.entry(old.layer).or_default(); - let moved = old.properties == new.properties - && translation(object_xy(old), object_xy(new)).is_some() - && field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY); - let shapes = if moved { - &mut counts.moved + let displacement = if old.properties == new.properties { + translation(object_xy(old), object_xy(new)) + .filter(|_| field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY)) } else { - &mut counts.changed + None }; - shapes.add(old, Some(new)); + if let Some(shift) = displacement { + counts.displacement = Some(shift); + counts.moved.add(old, Some(new)); + } else { + counts.changed.add(old, Some(new)); + } } pub fn unmatched(&mut self, object: &ObjectChange, removed: bool) { @@ -226,13 +234,31 @@ impl PolygonChanges { } else { out.push_str("unknown layer: "); } - let mut separator = ""; - for (action, shapes) in [ + let groups = [ ("changed", &counts.changed), ("moved", &counts.moved), ("removed", &counts.removed), ("added", &counts.added), - ] { + ]; + if let Some([dx, dy]) = counts.displacement + && groups + .iter() + .map(|(_, shapes)| shapes.total()) + .sum::() + == 1 + { + let shape = if counts.moved.rectangles == 1 { + "rectangle" + } else if counts.moved.boxes == 1 { + "box" + } else { + "polygon" + }; + writeln!(out, "{shape} moved by ({dx}, {dy}) dbu").unwrap(); + continue; + } + let mut separator = ""; + for (action, shapes) in groups { let terms = [ (shapes.rectangles, "rectangle", "rectangles"), (shapes.polygons, "polygon", "polygons"), diff --git a/tests/review.rs b/tests/review.rs index 2b51f45..34a27cc 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -1759,7 +1759,7 @@ fn replacement_explanations_are_unique_and_keep_full_evidence() { } #[test] -fn polygon_diff_reports_moves_and_other_changes_without_individual_geometry() { +fn polygon_diff_reports_single_displacements_and_groups_bulk_moves() { let before = layout(&[("CHIP", vec![with_property(square(), "old")])]); let before = Document::parse(&before).unwrap(); for (shape, label) in [ @@ -1787,26 +1787,65 @@ fn polygon_diff_reports_moves_and_other_changes_without_individual_geometry() { assert!(!text.contains("xy=")); } + let moved_triangle = polygon(&[[2, -1], [12, -1], [2, 9], [2, -1]]); + let moved_square = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + let named_layout = |mut objects: Vec>| { + objects.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", objects)]) + }; for (shape, shifted, name) in [ + (triangle(), moved_triangle, "polygon"), + (square(), moved_square.clone(), "rectangle"), ( - triangle(), - polygon(&[[2, -1], [12, -1], [2, 9], [2, -1]]), - "polygon", - ), - ( - square(), - polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]), - "rectangle", + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ), + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]], + ), + "box", ), ] { - let before = layout(&[("CHIP", vec![shape])]); - let after = layout(&[("CHIP", vec![shifted])]); + let before = named_layout(vec![shape]); + let after = named_layout(vec![shifted]); let diff = Comparison::between( &Document::parse(&before).unwrap(), &Document::parse(&after).unwrap(), ); - assert!(diff.text(100).contains(&format!(" 1/0: 1 {name} moved\n"))); + assert!( + diff.text(100) + .contains(&format!(" 1/0 (WG): {name} moved by (2, -1) dbu\n")) + ); } + + let before = named_layout(vec![square()]); + let after = named_layout(vec![ + moved_square, + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[20, 0], [30, 0], [30, 10], [20, 10], [20, 0]], + ), + ]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!( + diff.text(100) + .contains(" 1/0 (WG): 1 rectangle moved · 1 box added\n") + ); + assert!( + diff.text(2) + .contains(" 1/0 (WG): rectangle moved by (2, -1) dbu\n") + ); } #[test] From 4ae689cd1c4df631b28f2091fa7c287a4df8d0ed Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 23:04:24 -0800 Subject: [PATCH 32/41] Fold unbranched paths in ASCII diffs --- docs/cli.md | 5 +- src/bin/pheon/color.rs | 61 +++++++++++++++++---- src/review.rs | 118 +++++++++++++++++++++++++++++++++-------- tests/review.rs | 57 +++++++++++++++++++- 4 files changed, 208 insertions(+), 33 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 045ff91..4e1768c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -137,7 +137,10 @@ reference changes whose placement count stays the same. When the cell list shows all edits to an unchanged target on a single branch, ASCII can omit that terminal edge. The hierarchy retains ancestor paths, shared branches, and edges with edits hidden by display filters or limits. ASCII appears when -there are relationships to show. Use `--depth` with text output. +there are relationships to show. ASCII diffs fold chains of three or more cells +along unchanged single-placement edges, for example `CHIP → block → * leaf`. +Branch points and changed edges keep their own rows. `--depth` counts the +original edges. For the complete graph within the comparison scope, use `--format json --hierarchy graph` without `--depth`. Display filters leave the graph intact. diff --git a/src/bin/pheon/color.rs b/src/bin/pheon/color.rs index e2061d7..e7ba8a0 100644 --- a/src/bin/pheon/color.rs +++ b/src/bin/pheon/color.rs @@ -31,6 +31,7 @@ pub(super) fn print(text: &str, report: Report, choice: ColorChoice) -> io::Resu } let mut in_cells = false; + let mut in_hierarchy = false; let mut in_mermaid = false; for line in text.split_inclusive('\n') { let body = line.strip_suffix('\n').unwrap_or(line); @@ -48,10 +49,14 @@ pub(super) fn print(text: &str, report: Report, choice: ColorChoice) -> io::Resu _ => { match body { "Cells:" => in_cells = true, - "Hierarchy" | "Diagnostics" => in_cells = false, + "Hierarchy" => { + in_cells = false; + in_hierarchy = true; + } + "Diagnostics" => in_cells = false, _ => (), } - diff_line(&mut out, body, in_cells)?; + diff_line(&mut out, body, in_cells, in_hierarchy)?; } }, Report::Dump => dump_line(&mut out, body)?, @@ -75,7 +80,40 @@ fn paint( result.and(out.reset()) } -fn diff_line(out: &mut impl WriteColor, line: &str, in_cells: bool) -> io::Result<()> { +fn change_color(marker: &str) -> Option { + match marker { + "+" => Some(Color::Green), + "-" => Some(Color::Red), + "*" | "~" => Some(Color::Yellow), + _ => None, + } +} + +fn folded_endpoint(line: &str) -> Option<(&str, &str)> { + let mut chars = line.char_indices(); + let mut quoted = false; + let mut endpoint = None; + while let Some((index, ch)) = chars.next() { + match ch { + '"' => quoted = !quoted, + '\\' if quoted => { + chars.next(); + } + '→' if !quoted && line[index..].starts_with("→ ") => { + endpoint = Some(line.split_at(index + "→ ".len())); + } + _ => (), + } + } + endpoint +} + +fn diff_line( + out: &mut impl WriteColor, + line: &str, + in_cells: bool, + in_hierarchy: bool, +) -> io::Result<()> { let content = line.trim_start_matches(' '); let indent = &line[..line.len() - content.len()]; let first = content.split_once(' ').map_or(content, |(word, _)| word); @@ -93,16 +131,19 @@ fn diff_line(out: &mut impl WriteColor, line: &str, in_cells: bool) -> io::Resul return paint(out, line, color, true); } } - let color = match first { - "+" => Some(Color::Green), - "-" => Some(Color::Red), - "*" | "~" => Some(Color::Yellow), - _ => None, - }; - if let Some(color) = color { + if let Some(color) = change_color(first) { out.write_all(indent.as_bytes())?; return paint(out, content, color, false); } + if in_hierarchy + && line.contains('→') + && let Some((path, cell)) = folded_endpoint(line) + && let Some((marker, _)) = cell.split_once(' ') + && let Some(color) = change_color(marker) + { + out.write_all(path.as_bytes())?; + return paint(out, cell, color, false); + } if in_cells && indent == " " { if (layer_token(content).is_some() || content.starts_with("unknown layer: ")) && let Some((layer, counts)) = content.rsplit_once(": ") diff --git a/src/review.rs b/src/review.rs index 9e49dd4..f1a2820 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1890,27 +1890,28 @@ fn render_hierarchy( }; let mut children = BTreeMap::>::new(); let mut targets = BTreeSet::new(); + let fold_paths = !mermaid && changes.is_some(); + let mut shared = BTreeSet::new(); for (parent, target) in &edges { if visible.contains(parent) && visible.contains(target) { children .entry(parent.clone()) .or_default() .push(target.clone()); - targets.insert(target.clone()); + if !targets.insert(target.clone()) && fold_paths { + shared.insert(target.clone()); + } } } if let Some(covered) = covered_edges { if !covered.is_empty() { - let mut incoming = BTreeMap::new(); - for target in children.values().flatten() { - *incoming.entry(target).or_insert(0_usize) += 1; - } // Keep changed or shared descendants, branches, and target status notes. let redundant: BTreeSet<_> = covered .iter() .map(|(_, target)| target) .filter(|&target| { - incoming.get(target) == Some(&1) + targets.contains(target) + && !shared.contains(target) && !children.contains_key(target) && !affected.contains(target) && old.cells.contains(target) @@ -1938,6 +1939,12 @@ fn render_hierarchy( } } let mut included = BTreeSet::new(); + struct Row { + name: String, + depth: usize, + parent: Option, + note: &'static str, + } let mut rows = Vec::new(); let mut stack: Vec<_> = roots .iter() @@ -1956,7 +1963,12 @@ fn render_hierarchy( } else { "" }; - rows.push((name.clone(), depth, parent, note)); + rows.push(Row { + name: name.clone(), + depth, + parent, + note, + }); if !seen && depth < max_depth { for child in descendants.iter().rev() { stack.push((child.clone(), depth + 1, Some(name.clone()))); @@ -1964,7 +1976,12 @@ fn render_hierarchy( } } for name in visible.difference(&reachable) { - rows.push((name.clone(), 0, None, " [children omitted]")); + rows.push(Row { + name: name.clone(), + depth: 0, + parent: None, + note: " [children omitted]", + }); } let status = |name: &String| { if !old.cells.contains(name) && !new.cells.contains(name) { @@ -2004,7 +2021,7 @@ fn render_hierarchy( if mermaid { let ids: BTreeMap<_, _> = rows .iter() - .map(|r| &r.0) + .map(|row| &row.name) .collect::>() .into_iter() .enumerate() @@ -2034,25 +2051,84 @@ fn render_hierarchy( } out } else { + let single_placement = |parent: &String, target: &String| { + let key = (parent.clone(), target.clone()); + old_edges.get(&key) == Some(&1) + && new_edges.get(&key) == Some(&1) + && !changed_refs.contains(&key) + }; + let unbranched = |name: &String| { + !shared.contains(name) + && children + .get(name) + .is_none_or(|descendants| descendants.len() == 1) + }; + let can_join = |parent: &Row, child: &Row| { + child.parent.as_ref() == Some(&parent.name) + && parent.note.is_empty() + && !direct.contains(&parent.name) + && affected.contains(&parent.name) + && unbranched(&child.name) + && (old.cells.contains(&child.name) || new.cells.contains(&child.name)) + && single_placement(&parent.name, &child.name) + }; let mut out = "Hierarchy\n".to_string(); if changes.is_none() { new.scope.write_restrictions(&mut out); } - for (name, depth, parent, note) in rows { - out.push_str(&" ".repeat(depth)); - if direct.contains(&name) { - let marker = cell_marker(old.cells.contains(&name), new.cells.contains(&name)); - write!(out, "{marker} ").unwrap(); + let mut index = 0; + let mut ancestor_depths = Vec::new(); + while let Some(first) = rows.get(index) { + let mut end = index; + if fold_paths + && unbranched(&first.name) + && first + .parent + .as_ref() + .is_none_or(|parent| single_placement(parent, &first.name)) + { + while rows + .get(end + 1) + .is_some_and(|next| can_join(&rows[end], next)) + { + end += 1; + } + // Two rows are already easy to scan. + if end - index < 2 { + end = index; + } } - if changes.is_some() { - out.push_str(&cell_label(&name)); - } else { - write!(out, "{name:?}").unwrap(); + while ancestor_depths + .last() + .is_some_and(|&depth| depth >= first.depth) + { + ancestor_depths.pop(); } - if let Some(parent) = parent { - write!(out, " {}", edge_label(&parent, &name)).unwrap(); + out.push_str(&" ".repeat(ancestor_depths.len())); + for (i, row) in rows[index..=end].iter().enumerate() { + let name = &row.name; + if i != 0 { + out.push_str(" → "); + } + if direct.contains(name) { + let marker = cell_marker(old.cells.contains(name), new.cells.contains(name)); + write!(out, "{marker} ").unwrap(); + } + if changes.is_some() { + out.push_str(&cell_label(name)); + } else { + write!(out, "{name:?}").unwrap(); + } + if i == 0 + && let Some(parent) = &row.parent + { + write!(out, " {}", edge_label(parent, name)).unwrap(); + } } - writeln!(out, "{}{note}", status(&name)).unwrap(); + let last = &rows[end]; + writeln!(out, "{}{}", status(&last.name), last.note).unwrap(); + ancestor_depths.push(last.depth); + index = end + 1; } if visible.is_empty() { out.push_str("No changed hierarchy\n"); diff --git a/tests/review.rs b/tests/review.rs index 34a27cc..26a4bd3 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -533,6 +533,48 @@ fn shared_hierarchy_reports_ancestors_and_each_leaf_placement_count() { ); } +#[test] +fn ascii_diff_folds_single_placement_paths_up_to_branches_and_changed_edges() { + let with_branch = |count, shape| { + layout(&[ + ("CHIP", vec![reference("block", 0, 1)]), + ("block", vec![reference("routing", 0, 1)]), + ("routing", vec![reference("fanout", 0, 1)]), + ( + "fanout", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("target", 0, count)]), + ("target", vec![]), + ("right", vec![reference("unit", 0, 1)]), + ("unit", vec![reference("leaf", 0, 1)]), + ("leaf", vec![shape]), + ]) + }; + let old = with_branch(2, square()); + let new = with_branch(3, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "CHIP → block → routing [affected]\n", + " fanout x1 [affected]\n", + " * left x1\n", + " target x2 -> x3\n", + " right x1 → unit → * leaf\n", + ) + ); + assert_eq!( + diff.hierarchy(false, 2), + "Hierarchy\nCHIP → block → routing [affected] [children omitted]\n" + ); + assert!(!diff.hierarchy(true, 8).contains('→')); +} + #[test] fn ascii_hierarchy_marks_repeated_subtrees_and_depth_limits() { let bytes = layout(&[ @@ -880,7 +922,13 @@ fn cli_color_preserves_plain_output_and_machine_formats() { .map(|x| reference("leaf", x, 1)) .chain([shape]) .collect(); - layout(&[("CHIP", objects), ("leaf", vec![]), (name, vec![])]) + layout(&[ + ("top", vec![reference("route", 0, 1)]), + ("route", vec![reference("CHIP", 0, 1)]), + ("CHIP", objects), + ("leaf", vec![]), + (name, vec![]), + ]) }; fs::write(&old, with_changes("obsolete", square(), 0..2)).unwrap(); fs::write(&new, with_changes("new", triangle(), 10..13)).unwrap(); @@ -928,6 +976,13 @@ fn cli_color_preserves_plain_output_and_machine_formats() { diff(&["--color", "always", "--format", "json"]), diff(&["--format", "json"]) ); + let ascii = diff(&["--hierarchy", "ascii", "--color=always"]); + let folded = ascii + .lines() + .find(|line| line.starts_with("top → route → ")) + .unwrap(); + assert!(folded.contains("\x1b[33m* CHIP")); + assert_eq!(strip_ansi(&ascii), diff(&["--hierarchy", "ascii"])); for (options, token) in [ (&[][..], "\x1b[36mcell \"CHIP\""), From 87683db7e8bd2d0a828e2842ecbaba226ed2415e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 23:07:22 -0800 Subject: [PATCH 33/41] Accept embedded layer names in CLI selectors Resolve scope and display selectors against both revisions before projection. Keep display-only selection on the large-file comparison path and list numeric choices for ambiguous names. --- docs/cli.md | 8 ++- src/bin/pheon.rs | 128 ++++++++++++++++++++++++++++++++------------ src/review/names.rs | 24 +++++++++ src/review/pair.rs | 5 ++ tests/review.rs | 79 +++++++++++++++++++++++++++ 5 files changed, 209 insertions(+), 35 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 4e1768c..2c0d51f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -103,7 +103,7 @@ pheon diff before.gds after.gds --cell RESONATOR | --- | --- | | `--top NAME` | The named cell and its reachable descendants in each revision. | | `--cell NAME` | One cell, including its references, without its descendants. | -| `--layer L/T` | Objects on a numeric layer/type pair, plus references leading to that layer. | +| `--layer L/T\|NAME` | Objects on the selected layer/type pair, plus references leading to that layer. | Selectors combine: `--top CHIP --cell RESONATOR` selects that cell within CHIP's subtree. A name absent from both revisions is an error; presence on only one @@ -112,10 +112,14 @@ layer, but library fields and metadata-version diagnostics remain relevant. The complete input is parsed before scope selection, so malformed records outside the scope still fail the command. +Both `--layer` and `--show-layer` accept exact, case-sensitive names from the +embedded metadata in either revision. If a name refers to several numeric pairs, +Pheon lists the choices. A valid numeric pair always selects that pair directly. + ## Control report detail ```sh -pheon diff before.gds after.gds --show-cell RESONATOR --show-layer 1/0 +pheon diff before.gds after.gds --show-cell RESONATOR --show-layer WG pheon diff before.gds after.gds --hierarchy mermaid --depth 4 ``` diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 309fd6b..372a8fa 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -4,8 +4,9 @@ mod color; use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ - Comparison, Diagnostic, Document, Outcome, RevisionPair, Side, TextComparison, dump_records, - names::{parse_layer, parse_layer_name, write_layer_names}, + Comparison, Diagnostic, Document, Layer, Outcome, RevisionPair, Side, TextComparison, + dump_records, + names::{parse_layer, parse_layer_name, resolve_layer_name, write_layer_names}, selection::{DisplayFilter, Scope}, }; use std::{ @@ -21,7 +22,7 @@ use termcolor::ColorChoice; const HELP: &str = "Usage: pheon dump FILE [--records] [--color auto|always|never] - pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T]\n [--show-cell NAME] [--show-layer L/T] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T|NAME]\n [--show-cell NAME] [--show-layer L/T|NAME] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] [--color auto|always|never] pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... @@ -145,10 +146,55 @@ fn parse_revision(bytes: Option<&[u8]>, side: Side) -> Result, + show_layer_name: Option, +} + +fn layer_argument(value: String) -> (Option, Option) { + match parse_layer(&value) { + Ok(pair) => (Some(pair), None), + Err(_) => (None, Some(value)), + } +} + +impl Selectors { + fn resolve_layers( + &mut self, + resolve: impl Fn(&str) -> Result, + ) -> Result<(), Diagnostic> { + for (flag, code, name, pair) in [ + ( + "--layer", + "invalid_scope", + &self.layer_name, + &mut self.scope.layer, + ), + ( + "--show-layer", + "command_error", + &self.show_layer_name, + &mut self.filter.layer, + ), + ] { + if let Some(name) = name { + *pair = Some( + resolve(name) + .map_err(|error| Diagnostic::new(code, format!("{flag} {error}"), None))?, + ); + } + } + Ok(()) + } +} + fn prepare_comparison( old_path: Option<&OsStr>, new_path: Option<&OsStr>, - scope: &Scope, + selectors: &mut Selectors, compare: impl FnOnce(&Document<'_>, &Document<'_>) -> T, compare_pair: impl FnOnce(RevisionPair<'_>) -> T, ) -> Result { @@ -168,10 +214,13 @@ fn prepare_comparison( .as_deref() .zip(new_bytes.as_deref()) .filter(|(old, new)| old.len().min(new.len()) >= PARALLEL_PARSE_MIN_BYTES); - if scope == &Scope::default() + if selectors.scope == Scope::default() + && selectors.layer_name.is_none() && let Some((old, new)) = large_pair { - return RevisionPair::parse(old, new).map(compare_pair); + let pair = RevisionPair::parse(old, new)?; + selectors.resolve_layers(|name| pair.resolve_layer_name(name))?; + return Ok(compare_pair(pair)); } let parse_old = || parse_revision(old_bytes.as_deref(), Side::Old); let parse_new = || parse_revision(new_bytes.as_deref(), Side::New); @@ -206,6 +255,9 @@ fn prepare_comparison( )); } }; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&old.layer_names, &new.layer_names]))?; + let scope = &selectors.scope; if scope == &Scope::default() { return Ok(compare(&old, &new)); } @@ -221,24 +273,35 @@ fn prepare_comparison( Ok(compare(&old, &new)) } -fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { +fn json_diff( + paths: &[OsString], + selectors: &mut Selectors, + max_objects: Option, + hierarchy: bool, +) -> Result { let inputs = ReportInputs { old: Some(PathBuf::from(&paths[0])), new: Some(PathBuf::from(&paths[1])), }; - let (result, outcome) = match prepare_comparison( + let comparison = prepare_comparison( Some(&paths[0]), Some(&paths[1]), - scope, + selectors, Comparison::between, |pair| pair.compare(), - ) { + ); + let options = ReportOptions { + filter: selectors.filter.clone(), + max_objects, + hierarchy, + }; + let (result, outcome) = match comparison { Ok(diff) => ( - json_stdout(|output| diff.write_json_report(output, &inputs, options)), + json_stdout(|output| diff.write_json_report(output, &inputs, &options)), diff.outcome(), ), Err(error) => { - let report = failure_report(&inputs, scope, options, &error); + let report = failure_report(&inputs, &selectors.scope, &options, &error); ( json_stdout(|output| serde_json::to_writer_pretty(output, &report)), Outcome::Failed, @@ -275,8 +338,7 @@ fn run(args: &[OsString]) -> Result { let compare = command == "diff" || command == "git-diff"; let color_supported = compare || command == "dump"; let mut color_choice = ColorChoice::Auto; - let mut scope = Scope::default(); - let mut show = DisplayFilter::default(); + let mut selectors = Selectors::default(); let mut raw = false; let mut format = if compare { "none".to_string() @@ -310,12 +372,16 @@ fn run(args: &[OsString]) -> Result { Some(s) if color_supported && s.starts_with("--color=") => { color_choice = color::parse_choice(&s["--color=".len()..])?; } - Some("--top") => scope.top = Some(value(args, &mut i)?), - Some("--cell") => scope.cell = Some(value(args, &mut i)?), - Some("--layer") => scope.layer = Some(parse_layer(&value(args, &mut i)?)?), - Some("--show-cell") if compare => show.cell = Some(value(args, &mut i)?), + Some("--top") => selectors.scope.top = Some(value(args, &mut i)?), + Some("--cell") => selectors.scope.cell = Some(value(args, &mut i)?), + Some("--layer") => { + (selectors.scope.layer, selectors.layer_name) = + layer_argument(value(args, &mut i)?); + } + Some("--show-cell") if compare => selectors.filter.cell = Some(value(args, &mut i)?), Some("--show-layer") if compare => { - show.layer = Some(parse_layer(&value(args, &mut i)?)?) + (selectors.filter.layer, selectors.show_layer_name) = + layer_argument(value(args, &mut i)?); } Some("--hierarchy") if compare => format = value(args, &mut i)?, Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, @@ -347,15 +413,7 @@ fn run(args: &[OsString]) -> Result { "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), ); } - return json_diff( - &paths, - &scope, - &ReportOptions { - filter: show, - max_objects: explicit_max, - hierarchy: format == "graph", - }, - ); + return json_diff(&paths, &mut selectors, explicit_max, format == "graph"); } if !matches!(format.as_str(), "ascii" | "mermaid" | "none") || (command == "hierarchy" && format == "none") @@ -364,7 +422,7 @@ fn run(args: &[OsString]) -> Result { } let read = |path: &OsString| read_input(path).map_err(|error| error.message); if raw { - if scope != Scope::default() { + if selectors.scope != Scope::default() || selectors.layer_name.is_some() { return Err("record dumps do not accept object scope selectors".to_string()); } color::print( @@ -377,7 +435,11 @@ fn run(args: &[OsString]) -> Result { } if !compare { let bytes = read(&paths[0])?; - let doc = Document::parse(&bytes)?.selected(&scope, false)?; + let doc = Document::parse(&bytes)?; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&doc.layer_names])) + .map_err(|error| error.message)?; + let doc = doc.selected(&selectors.scope, false)?; if command == "dump" { color::print(&doc.canonical_dump(), color::Report::Dump, color_choice) .map_err(|error| error.to_string())?; @@ -395,7 +457,7 @@ fn run(args: &[OsString]) -> Result { let diff = prepare_comparison( (!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()), (!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()), - &scope, + &mut selectors, TextComparison::between, |pair| pair.compare_text(), ) @@ -423,12 +485,12 @@ fn run(args: &[OsString]) -> Result { if format == "ascii" { output.push_str(&diff.render_text_with_ascii_hierarchy( max_changes, - &show, + &selectors.filter, show_library_presence, depth, )); } else { - output.push_str(&diff.render_text(max_changes, &show, show_library_presence)); + output.push_str(&diff.render_text(max_changes, &selectors.filter, show_library_presence)); if format == "mermaid" { output.push_str("Hierarchy\n```mermaid\n"); output.push_str(&diff.hierarchy(true, depth)); diff --git a/src/review/names.rs b/src/review/names.rs index 22d576a..c551526 100644 --- a/src/review/names.rs +++ b/src/review/names.rs @@ -68,6 +68,30 @@ pub fn parse_layer(value: &str) -> Result { Ok((decimal(a)?, decimal(b)?)) } +/// Find the unique numeric pair for an exact name across the supplied revisions. +pub fn resolve_layer_name<'a>( + name: &str, + revisions: impl IntoIterator>, +) -> Result { + let matches: BTreeSet<_> = revisions + .into_iter() + .flatten() + .filter_map(|(&pair, label)| (label == name).then_some(pair)) + .collect(); + match matches.len() { + 0 => Err(format!("unknown layer name {name:?}")), + 1 => Ok(*matches.first().unwrap()), + _ => { + let choices = matches + .into_iter() + .map(|(layer, datatype)| format!("{layer}/{datatype}")) + .collect::>() + .join(", "); + Err(format!("ambiguous layer name {name:?} (choose {choices})")) + } + } +} + fn decimal(value: &str) -> Result { if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { return Err("layer numbers must be unsigned decimal integers".to_string()); diff --git a/src/review/pair.rs b/src/review/pair.rs index 1b8352f..3b247e3 100644 --- a/src/review/pair.rs +++ b/src/review/pair.rs @@ -184,6 +184,11 @@ impl<'a> RevisionPair<'a> { Comparison::collect(&self.old, &self.new, &self.reused_cells, true) } + /// Resolve a layer name from the embedded metadata in either revision. + pub fn resolve_layer_name(&self, name: &str) -> Result { + names::resolve_layer_name(name, [&self.old.layer_names, &self.new.layer_names]) + } + pub fn compare_text(self) -> TextComparison { TextComparison::collect(&self.old, &self.new, &self.reused_cells) } diff --git a/tests/review.rs b/tests/review.rs index 26a4bd3..cf31509 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -2164,6 +2164,85 @@ fn cli_scope_and_display_filters_have_distinct_exit_behavior() { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn cli_resolves_layer_names_before_applying_scope_and_display_filters() { + let directory = std::env::temp_dir().join(format!("pheon-named-layer-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let fixture = |shape, text, name| { + layout(&[ + ("CHIP", vec![shape, label((2, 0), text)]), + ( + "legend", + vec![label((900, 7), LAYER_SENTINEL), label((900, 7), name)], + ), + ]) + }; + fs::write(&old, fixture(square(), "before", "1:0:WG")).unwrap(); + fs::write(&new, fixture(triangle(), "after", "1:0:CORE")).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap() + }; + for (option, name) in [("--layer", "WG"), ("--show-layer", "CORE")] { + for format in ["text", "json"] { + let named = diff(&[option, name, "--cell", "CHIP", "--format", format]); + let numeric = diff(&[option, "1/0", "--cell", "CHIP", "--format", format]); + assert_eq!( + named.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&named.stderr) + ); + assert_eq!(named.stdout, numeric.stdout); + } + } + + let dump = |layer| { + let output = command() + .arg("dump") + .arg(&old) + .args(["--layer", layer]) + .output() + .unwrap(); + assert!(output.status.success()); + output.stdout + }; + assert_eq!(dump("WG"), dump("1/0")); + + let unknown = diff(&["--show-layer", "wg", "--format", "json"]); + assert_eq!(unknown.status.code(), Some(2)); + let report: serde_json::Value = serde_json::from_slice(&unknown.stdout).unwrap(); + assert_eq!(report["diagnostics"][0]["code"], "command_error"); + assert_eq!( + report["diagnostics"][0]["message"], + "--show-layer unknown layer name \"wg\"" + ); + + fs::write(&new, fixture(triangle(), "after", "2:0:WG")).unwrap(); + let ambiguous = diff(&["--layer", "WG"]); + assert_eq!(ambiguous.status.code(), Some(2)); + assert_eq!( + String::from_utf8(ambiguous.stderr).unwrap().trim_end(), + "pheon: --layer ambiguous layer name \"WG\" (choose 1/0, 2/0)" + ); + assert_eq!(diff(&["--layer", "1/0"]).status.code(), Some(1)); +} + #[test] fn large_polygons_and_billion_instance_arrays_stay_compact() { let mut points = vec![[0, 0], [0, 1]]; From ab308f7b7fb7ed98741a0aef3496c886cabc84e2 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 23:16:09 -0800 Subject: [PATCH 34/41] Pair unique layer-only shape changes --- docs/cli.md | 7 +- docs/diff-reports.md | 9 ++- src/review.rs | 17 +++-- src/review/changes.rs | 146 ++++++++++++++++++++++++++++++++++-------- tests/review.rs | 112 ++++++++++++++++++++++++++++++++ 5 files changed, 253 insertions(+), 38 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 2c0d51f..fbaadba 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -68,6 +68,8 @@ displayed rows gets one legend entry. Object details use `-` for removals, `+` f and `~` for likely replacements. Identical unmatched objects share a row with a count such as `+ 12× 1/0: path …`. Unmatched paths with five or more vertices show their width, vertex count, and vertex bounds in one line. +A unique polygon, box, or path replacement that differs only in layer or +datatype appears on one row, for example `~ 1/0 (WG) → 2/0 (CORE) · rectangle`. Use `--format json` for details of individual objects and `--hierarchy` to see which parent cells are affected. @@ -128,8 +130,9 @@ comparison, totals, or exit status. A layer display filter hides references without a numeric layer. `--max-changes N` limits displayed object occurrences after filtering. Text defaults to 100 occurrences and shows a replacement only when both objects fit within the limit. Grouped rows count each object toward -the limit. A single omission line lists changes hidden by filters or the -limit. JSON has no default object limit and reports +the limit. When a layer filter selects one side of a layer-only replacement, +that side appears as an addition or removal. A single omission line lists +changes hidden by filters or the limit. JSON has no default object limit and reports omissions from both display filters and any explicit limit in `report_selection`. See the [JSON reference](diff-reports.md) for the selection counters. diff --git a/docs/diff-reports.md b/docs/diff-reports.md index ef870a5..c88cdfd 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -90,9 +90,12 @@ explicit defaults remain significant. For gzip inputs, source offsets locate records in the decompressed layout. Replacements link report-local removed/added IDs with `match_rule`, -`field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, -`unique_kind_layer`, or `unique_kind_target`. IDs are not persistent identities; -ambiguous candidates stay separate. Explanation text is for display only. +`field_changes`, and `properties_changed`. The `unique_layer_change` rule +matches a unique polygon, box, or path on each side with identical retained +fields and properties outside the layer and datatype fields. The other rules +are `unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. +Ambiguous candidates stay separate. Replacement IDs refer only to objects in +the current report. Explanation text is for display only. ## Diagnostics and hierarchy diff --git a/src/review.rs b/src/review.rs index f1a2820..dcdb302 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1543,7 +1543,7 @@ impl Comparison { let body_start = out.len(); let mut removed_cells = String::new(); let mut added_cells = String::new(); - let mut layers = changes::LayerRows::new(&self.layer_legend); + let mut layers = changes::LayerRows::new(&self.layer_legend, &self.layer_names); let mut annotations = Vec::new(); let mut covered_edges = BTreeSet::new(); let mut detail_total = 0; @@ -1587,12 +1587,19 @@ impl Comparison { let mut paired_old = BTreeSet::new(); let mut paired_new = BTreeSet::new(); for replacement in &cell.replacements { - paired_old.insert(replacement.removed); - paired_new.insert(replacement.added); let a = &cell.removed[replacement.removed]; let b = &cell.added[replacement.added]; - if max_objects - shown >= 2 && visible(a) && visible(b) { - if is_polygon(a.kind) { + if !visible(a) || !visible(b) { + continue; + } + paired_old.insert(replacement.removed); + paired_new.insert(replacement.added); + if max_objects - shown >= 2 { + if a.layer != b.layer { + out.push_str(" ~ "); + changes::write_layer_transition(&mut out, a, b, Some(&mut layers)); + out.push('\n'); + } else if is_polygon(a.kind) { polygons.replacement(a, b); } else { out.push_str(" ~ "); diff --git a/src/review/changes.rs b/src/review/changes.rs index 2f0a6cd..9b4e88b 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -25,38 +25,82 @@ fn group(object: &ObjectChange, bounds: bool) -> Group { ) } +fn unique_groups<'a, K: Ord>( + objects: &'a [ObjectChange], + key: impl Fn(usize, &'a ObjectChange) -> Option, +) -> BTreeMap> { + let mut groups = BTreeMap::new(); + for (index, object) in objects.iter().enumerate() { + if let Some(key) = key(index, object) { + groups + .entry(key) + .and_modify(|value| *value = None) + .or_insert(Some(index)); + } + } + groups +} + +type ShapeKey<'a> = (u16, Vec<(u16, &'a [u8])>, &'a [(u16, Vec)]); + +fn shape_without_layer(object: &ObjectChange) -> Option> { + if object.layer.is_none() || !(is_polygon(object.kind) || object.kind == RTAG_PATH) { + return None; + } + let fields = object + .fields + .iter() + .filter(|field| !matches_layer_field(object.kind, field.tag)) + .map(|field| (field.tag, field.data.as_slice())) + .collect(); + Some((object.kind, fields, &object.properties)) +} + +fn layer_replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + // Most small edits touch a single numeric pair. + let Some(layer) = old.iter().chain(new).find_map(|object| object.layer) else { + return Vec::new(); + }; + if !old + .iter() + .chain(new) + .any(|object| object.layer.is_some_and(|pair| pair != layer)) + { + return Vec::new(); + } + let a = unique_groups(old, |_, object| shape_without_layer(object)); + let b = unique_groups(new, |_, object| shape_without_layer(object)); + a.into_iter() + .filter_map(|(key, removed)| { + let removed = removed?; + let added = b.get(&key).copied().flatten()?; + (old[removed].layer != new[added].layer).then_some(Replacement { + removed, + added, + match_rule: "unique_layer_change", + }) + }) + .collect() +} + /// Pair only unique candidates in the remaining multisets. Ambiguous groups /// remain additions/removals; no identity or geometric equivalence is inferred. pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { if old.is_empty() || new.is_empty() { return Vec::new(); } - let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new()); - let mut result = Vec::new(); + let mut result = layer_replacements(old, new); + let (mut used_old, mut used_new): (BTreeSet<_>, BTreeSet<_>) = + result.iter().map(|pair| (pair.removed, pair.added)).unzip(); for bounds in [true, false] { - let (mut a, mut b) = ( - BTreeMap::>::new(), - BTreeMap::>::new(), - ); - for (i, object) in old - .iter() - .enumerate() - .filter(|(i, _)| !used_old.contains(i)) - { - a.entry(group(object, bounds)).or_default().push(i); - } - for (i, object) in new - .iter() - .enumerate() - .filter(|(i, _)| !used_new.contains(i)) - { - b.entry(group(object, bounds)).or_default().push(i); - } - for (key, indices) in a { - if let (1, Some(others)) = (indices.len(), b.get(&key)) - && others.len() == 1 - { - let (i, j) = (indices[0], others[0]); + let a = unique_groups(old, |i, object| { + (!used_old.contains(&i)).then(|| group(object, bounds)) + }); + let b = unique_groups(new, |i, object| { + (!used_new.contains(&i)).then(|| group(object, bounds)) + }); + for (key, index) in a { + if let (Some(i), Some(&Some(j))) = (index, b.get(&key)) { result.push(Replacement { removed: i, added: j, @@ -407,17 +451,37 @@ impl ShownReferences { pub(super) struct LayerRows<'a> { names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], rows: BTreeMap>, } impl<'a> LayerRows<'a> { - pub fn new(names: &'a BTreeMap) -> Self { + pub fn new(names: &'a BTreeMap, name_changes: &'a [LayerNameChange]) -> Self { Self { names, + name_changes, rows: BTreeMap::new(), } } + fn write_transition_layer(&mut self, out: &mut String, layer: Layer, side: Side) { + self.record(Some(layer), out.len()); + write!(out, "{}/{}", layer.0, layer.1).unwrap(); + if let Ok(index) = self + .name_changes + .binary_search_by_key(&layer, |change| change.layer) + { + let change = &self.name_changes[index]; + let name = match side { + Side::Old => &change.old, + Side::New => &change.new, + }; + if let Some(name) = name { + write!(out, " ({})", cell_label(name)).unwrap(); + } + } + } + pub fn record(&mut self, layer: Option, row_start: usize) { if let Some(layer) = layer && self.names.contains_key(&layer) @@ -433,8 +497,11 @@ impl<'a> LayerRows<'a> { for (layer, first) in self.rows { let name = &self.names[&layer]; if let Some(at) = first { - // Each recorded heading starts with the numeric layer/type and a colon. - let at = at + text[at..].find(':').unwrap(); + let at = at + + text[at..] + .bytes() + .take_while(|b| b.is_ascii_digit() || *b == b'/') + .count(); annotations.push((at, format!(" ({})", cell_label(name)))); } else { writeln!(legend, "{}/{}: name {name:?}", layer.0, layer.1).unwrap(); @@ -443,6 +510,24 @@ impl<'a> LayerRows<'a> { } } +pub(super) fn write_layer_transition( + out: &mut String, + old: &ObjectChange, + new: &ObjectChange, + layers: Option<&mut LayerRows<'_>>, +) { + let (from, to) = (old.layer.unwrap(), new.layer.unwrap()); + if let Some(layers) = layers { + layers.write_transition_layer(out, from, Side::Old); + out.push_str(" → "); + layers.write_transition_layer(out, to, Side::New); + } else { + write!(out, "{}/{} → {}/{}", from.0, from.1, to.0, to.1).unwrap(); + } + let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some(); + write!(out, " · {}", object_name(old.kind, rectangular)).unwrap(); +} + fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { let before = object_xy(old); let after = object_xy(new); @@ -471,6 +556,11 @@ fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { } pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { + if old.layer != new.layer && old.layer.is_some() && new.layer.is_some() { + let mut out = String::new(); + write_layer_transition(&mut out, old, new, None); + return out; + } let polygon = is_polygon(old.kind); let mut out = if polygon { polygon_replacement(old, new) diff --git a/tests/review.rs b/tests/review.rs index cf31509..72b8a05 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -1813,6 +1813,118 @@ fn replacement_explanations_are_unique_and_keep_full_evidence() { assert!(!text.contains("xy=")); } +#[test] +fn layer_only_shape_replacements_keep_evidence_and_respect_selection() { + let rect = [[10, 0], [0, 0], [0, 10], [10, 10], [10, 0]]; + let on_core = ring(RTAG_BOUNDARY, RTAG_DATATYPE, 2, &rect); + let named = |mut objects: Vec>, names: &[&str]| { + objects.push(label((900, 7), LAYER_SENTINEL)); + objects.extend(names.iter().map(|name| label((900, 7), name))); + layout(&[("CHIP", objects)]) + }; + let old = named(vec![square()], &["1:0:WG", "2:0:CORE"]); + let new = named(vec![on_core.clone()], &["1:0:WG", "2:0:CORE"]); + let old = Document::parse(&old).unwrap(); + let diff = Comparison::between(&old, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(2), + "Different\nCells:\n* CHIP\n ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n" + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(!limited.contains("Cells:")); + for (pair, suffix) in [ + ((1, 0), " 1/0 (WG): 1 rectangle removed\n"), + ((2, 0), " 2/0 (CORE): 1 rectangle added\n"), + ] { + let filtered = diff.text_filtered( + 1, + &DisplayFilter { + cell: None, + layer: Some(pair), + }, + ); + assert!(filtered.contains("1 object change filtered out")); + assert!(filtered.ends_with(suffix), "{filtered}"); + } + let json = diff.json_report(&Default::default(), &Default::default()); + let replacement = &json["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_layer_change"); + assert_eq!(replacement["field_changes"].as_array().unwrap().len(), 1); + assert_eq!(replacement["field_changes"][0]["tag"], RTAG_LAYER); + assert_eq!(json["summary"]["removed_objects"], "1"); + assert_eq!(json["summary"]["added_objects"], "1"); + + let old = named(vec![square()], &["1:0:WG"]); + let new = named(vec![on_core], &["2:0:CORE"]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!( + diff.text(2) + .ends_with(" ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n") + ); +} + +#[test] +fn layer_only_matching_requires_unique_shapes_and_identical_payloads() { + let core_square = ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ); + for (old, new) in [ + (vec![square(), square()], vec![core_square.clone()]), + ( + vec![square()], + vec![core_square.clone(), core_square.clone()], + ), + (vec![square()], vec![with_property(core_square, "new")]), + ( + vec![square()], + vec![ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + )], + ), + ] { + let old = layout(&[("CHIP", old)]); + let new = layout(&[("CHIP", new)]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!(diff.cells[0].replacements.is_empty()); + assert!(diff.text(100).contains(" removed\n")); + assert!(diff.text(100).contains(" added\n")); + } +} + +#[test] +fn datatype_only_path_changes_pair_on_the_same_layer() { + let old = layout(&[("CHIP", vec![path(Some(8), &[[0, 0], [10, 10]])])]); + let old_doc = Document::parse(&old).unwrap(); + let at = old_doc + .records + .iter() + .find(|record| record.tag == RTAG_DATATYPE) + .unwrap() + .offset + + 4; + let mut new = old.clone(); + new[at..at + 2].copy_from_slice(&3_u16.to_be_bytes()); + let diff = Comparison::between(&old_doc, &Document::parse(&new).unwrap()); + assert!(diff.text(2).ends_with(" ~ 1/0 → 1/3 · path\n")); + assert_eq!( + diff.cells[0].replacements[0].match_rule, + "unique_layer_change" + ); +} + #[test] fn polygon_diff_reports_single_displacements_and_groups_bulk_moves() { let before = layout(&[("CHIP", vec![with_property(square(), "old")])]); From f3275fc16154160e76b02463f4f77c58edba24b0 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 18 Nov 2026 23:20:00 -0800 Subject: [PATCH 35/41] Add an opt-in detailed text diff --- docs/cli.md | 7 +++ docs/native-api.md | 4 +- src/bin/pheon.rs | 33 ++++++------ src/review.rs | 102 +++++++++++++++++++++++++++-------- src/review/changes.rs | 45 ++++++++++++++-- tests/review.rs | 120 +++++++++++++++++++++++++++++++++++++++++- 6 files changed, 266 insertions(+), 45 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index fbaadba..bee1df3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -122,6 +122,7 @@ Pheon lists the choices. A valid numeric pair always selects that pair directly. ```sh pheon diff before.gds after.gds --show-cell RESONATOR --show-layer WG +pheon diff before.gds after.gds --show-cell RESONATOR --details --max-changes 20 pheon diff before.gds after.gds --hierarchy mermaid --depth 4 ``` @@ -136,6 +137,12 @@ changes hidden by filters or the limit. JSON has no default object limit and rep omissions from both display filters and any explicit limit in `report_selection`. See the [JSON reference](diff-reports.md) for the selection counters. +`--details` shows individual object evidence inside modified cells in text +reports. Replacement rows include a nested `-` row for the old object and a +`+` row for the new object, with cell-local coordinates in database units, +retained fields, and properties. Both sides count toward `--max-changes`. +For added and removed cells, the report lists the cell name. + The diff hierarchy shows directly changed cells, affected ancestors, and targets of modified references. It includes references from both revisions so removed branches remain visible. Counts represent direct placements, with symbolic diff --git a/docs/native-api.md b/docs/native-api.md index 14f3c6e..ee55e50 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -34,7 +34,9 @@ graph and ancestors when a hierarchy is requested. `TextComparison::between` captures the data needed for text and hierarchy reports. Both comparison types provide `render_text_with_ascii_hierarchy` -to render a combined report with redundant terminal edges omitted. +to render a combined report with redundant terminal edges omitted. Pass +`TextReportOptions { details: true, ..Default::default() }` to +`render_text_with_options` to show individual object evidence. Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison that can reuse matching cells. Its `compare` and `compare_text` methods return diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 372a8fa..5b99354 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -5,7 +5,7 @@ use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ Comparison, Diagnostic, Document, Layer, Outcome, RevisionPair, Side, TextComparison, - dump_records, + TextReportOptions, dump_records, names::{parse_layer, parse_layer_name, resolve_layer_name, write_layer_names}, selection::{DisplayFilter, Scope}, }; @@ -22,7 +22,7 @@ use termcolor::ColorChoice; const HELP: &str = "Usage: pheon dump FILE [--records] [--color auto|always|never] - pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T|NAME]\n [--show-cell NAME] [--show-layer L/T|NAME] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T|NAME]\n [--show-cell NAME] [--show-layer L/T|NAME] [--hierarchy ascii|mermaid|none|graph]\n [--details] [--max-changes N] [--depth N] [--format text|json] [--color auto|always|never] pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... @@ -346,6 +346,7 @@ fn run(args: &[OsString]) -> Result { "ascii".to_string() }; let mut max_changes = 100; + let mut details = false; let mut explicit_max = None; let mut output_format = "text".to_string(); let mut depth = 8; @@ -383,6 +384,7 @@ fn run(args: &[OsString]) -> Result { (selectors.filter.layer, selectors.show_layer_name) = layer_argument(value(args, &mut i)?); } + Some("--details") if compare => details = true, Some("--hierarchy") if compare => format = value(args, &mut i)?, Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, Some("--format") if command == "diff" => output_format = value(args, &mut i)?, @@ -482,20 +484,19 @@ fn run(args: &[OsString]) -> Result { } } let show_library_presence = !old_missing && !new_missing; - if format == "ascii" { - output.push_str(&diff.render_text_with_ascii_hierarchy( - max_changes, - &selectors.filter, - show_library_presence, - depth, - )); - } else { - output.push_str(&diff.render_text(max_changes, &selectors.filter, show_library_presence)); - if format == "mermaid" { - output.push_str("Hierarchy\n```mermaid\n"); - output.push_str(&diff.hierarchy(true, depth)); - output.push_str("```\n"); - } + output.push_str(&diff.render_text_with_options( + max_changes, + &selectors.filter, + show_library_presence, + TextReportOptions { + details, + ascii_depth: (format == "ascii").then_some(depth), + }, + )); + if format == "mermaid" { + output.push_str("Hierarchy\n```mermaid\n"); + output.push_str(&diff.hierarchy(true, depth)); + output.push_str("```\n"); } color::print(&output, color::Report::Diff, color_choice).map_err(|error| error.to_string())?; // Git's external driver expects successful rendering for either equality diff --git a/src/review.rs b/src/review.rs index dcdb302..70b2b4b 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1202,6 +1202,15 @@ pub struct Comparison { pub diagnostics: Vec, } +/// Additional presentation options for a text comparison. +#[derive(Clone, Copy, Debug, Default)] +pub struct TextReportOptions { + /// Print individual object evidence, including both sides of replacements. + pub details: bool, + /// Append the ASCII hierarchy at this depth when provided. + pub ascii_depth: Option, +} + /// A comparison for text and hierarchy reports. #[derive(Debug)] pub struct TextComparison { @@ -1229,6 +1238,17 @@ impl TextComparison { .render_text(max_objects, show, show_library_presence) } + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + self.comparison + .render_text_with_options(max_objects, show, show_library_presence, options) + } + /// Render the cell changes and their ASCII hierarchy together. pub fn render_text_with_ascii_hierarchy( &self, @@ -1457,7 +1477,12 @@ impl Comparison { show: &DisplayFilter, show_library_presence: bool, ) -> String { - self.render_text_report(max_objects, show, show_library_presence, None) + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions::default(), + ) } /// Render the cell changes and omit simple hierarchy edges already shown there. @@ -1468,16 +1493,29 @@ impl Comparison { show_library_presence: bool, max_depth: usize, ) -> String { - self.render_text_report(max_objects, show, show_library_presence, Some(max_depth)) + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions { + ascii_depth: Some(max_depth), + ..TextReportOptions::default() + }, + ) } - fn render_text_report( + /// Render the selected changes with optional object details and ASCII hierarchy. + pub fn render_text_with_options( &self, max_objects: usize, show: &DisplayFilter, show_library_presence: bool, - ascii_depth: Option, + options: TextReportOptions, ) -> String { + let TextReportOptions { + details, + ascii_depth, + } = options; let outcome = self.outcome(); let mut out = match outcome { Outcome::Equal => "Equal", @@ -1594,27 +1632,36 @@ impl Comparison { } paired_old.insert(replacement.removed); paired_new.insert(replacement.added); - if max_objects - shown >= 2 { + if max_objects - shown < 2 { + continue; + } + if !details && a.layer == b.layer && is_polygon(a.kind) { + polygons.replacement(a, b); + } else { + out.push_str(" ~ "); + let row_start = out.len(); if a.layer != b.layer { - out.push_str(" ~ "); changes::write_layer_transition(&mut out, a, b, Some(&mut layers)); - out.push('\n'); - } else if is_polygon(a.kind) { - polygons.replacement(a, b); } else { - out.push_str(" ~ "); - let row_start = out.len(); layers.record(a.layer, row_start); - writeln!(out, "{}", changes::replacement_summary(a, b)).unwrap(); - if let Some(target) = &a.target { - shown_refs - .entry(target) - .or_default() - .replacement(a, b, row_start); + if details { + out.push_str(&changes::replacement_heading(a, b)); + } else { + out.push_str(&changes::replacement_summary(a, b)); } } - shown += 2; + out.push('\n'); + if details { + changes::write_replacement_objects(&mut out, a, b); + } + if let Some(target) = &a.target { + shown_refs + .entry(target) + .or_default() + .replacement(a, b, row_start); + } } + shown += 2; } let mut removed_refs = BTreeMap::new(); let mut added_refs = BTreeMap::new(); @@ -1630,9 +1677,9 @@ impl Comparison { } selected += 1; if !paired.contains(&index) && shown < max_objects { - if is_polygon(object.kind) { + if !details && is_polygon(object.kind) { polygons.unmatched(object, removed); - } else if let Some(target) = object.target.as_deref() { + } else if !details && let Some(target) = object.target.as_deref() { let (_, counts) = references .entry(target) .or_insert_with(|| (object, changes::ReferenceCounts::default())); @@ -1655,8 +1702,19 @@ impl Comparison { if count > 1 { write!(out, "{count}× ").unwrap(); } - layers.record(object.layer, out.len()); - changes::write_unmatched(&mut out, object); + let row_start = out.len(); + layers.record(object.layer, row_start); + if details { + writeln!(out, "{object}").unwrap(); + if let Some(target) = object.target.as_deref() { + shown_refs + .entry(target) + .or_default() + .individual(object, removed, count, row_start); + } + } else { + changes::write_unmatched(&mut out, object); + } } } let no_refs = changes::ReferenceCounts::default(); diff --git a/src/review/changes.rs b/src/review/changes.rs index 9b4e88b..824b00d 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -426,6 +426,23 @@ impl ShownReferences { self.row_start = row_start; } + pub fn individual( + &mut self, + object: &ObjectChange, + removed: bool, + count: usize, + row_start: usize, + ) { + let placements = if removed { + &mut self.removed + } else { + &mut self.added + }; + *placements += reference_placements(object) * count as u128; + self.shown += count; + self.row_start = row_start; + } + pub fn count_change(&mut self) { self.total += 1; } @@ -555,6 +572,28 @@ fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { out } +pub(super) fn replacement_heading(old: &ObjectChange, new: &ObjectChange) -> String { + let rectangular = is_polygon(old.kind) + && rectangle_size(object_xy(old)).is_some() + && rectangle_size(object_xy(new)).is_some(); + let mut heading = object_heading(old, object_name(old.kind, rectangular)); + if let Some(target) = &old.target { + write!(heading, " {target:?}").unwrap(); + } + heading +} + +pub(super) fn write_replacement_objects(out: &mut String, old: &ObjectChange, new: &ObjectChange) { + for (sign, object) in [('-', old), ('+', new)] { + let description = object.to_string(); + let mut lines = description.split('\n'); + writeln!(out, " {sign} {}", lines.next().unwrap()).unwrap(); + for line in lines { + writeln!(out, " {line}").unwrap(); + } + } +} + pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { if old.layer != new.layer && old.layer.is_some() && new.layer.is_some() { let mut out = String::new(); @@ -565,11 +604,7 @@ pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> Str let mut out = if polygon { polygon_replacement(old, new) } else { - let mut heading = object_heading(old, object_name(old.kind, false)); - if let Some(target) = &old.target { - write!(heading, " {target:?}").unwrap(); - } - heading + replacement_heading(old, new) }; let mut separator = if polygon { "\n " } else { " " }; diff --git a/tests/review.rs b/tests/review.rs index 72b8a05..ad625bb 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -6,7 +6,8 @@ use common::{library_header, record}; use pheon::{ records::*, review::{ - Comparison, Document, TextComparison, names::LAYER_SENTINEL, selection::DisplayFilter, + Comparison, Document, TextComparison, TextReportOptions, names::LAYER_SENTINEL, + selection::DisplayFilter, }, }; use std::{fs, path::PathBuf, process::Command}; @@ -1006,6 +1007,107 @@ fn cli_color_preserves_plain_output_and_machine_formats() { } } +#[test] +fn cli_details_show_coordinates_and_properties_within_filters_and_limits() { + let directory = std::env::temp_dir().join(format!("pheon-details-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let common_ref = reference("leaf", 0, 1); + let additional_ref = reference("leaf", 5, 1); + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let shifted = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + fs::write( + &old, + layout(&[ + ( + "CHIP", + vec![with_property(square(), "old"), common_ref.clone()], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "before other")]), + ("removed", vec![label((2, 0), "payload in removed cell")]), + ]), + ) + .unwrap(); + fs::write( + &new, + layout(&[ + ( + "CHIP", + vec![ + with_property(shifted, "new"), + with_property(path(Some(8), &points), "path property"), + common_ref, + additional_ref.clone(), + additional_ref, + ], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "after other")]), + ("added", vec![label((2, 0), "payload in added cell")]), + ]), + ) + .unwrap(); + let diff = |options: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + }; + let detailed = diff(&["--show-cell", "CHIP", "--details", "--hierarchy", "ascii"]); + assert!(detailed.contains(concat!( + " ~ 1/0: rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10) property[17]=\"old\"\n", + " + 1/0: rectangle (2, -1) to (12, 9) property[17]=\"new\"\n", + ))); + assert!(detailed.contains(concat!( + " + 1/0: path width=[8] xy=[(0, 0), (4, 0), (4, -5), (13, -5),\n", + " (13, 9)] property[17]=\"path property\"\n", + ))); + assert!(detailed.contains(" + 2× reference \"leaf\" xy=(5, 0) · placements 1 -> 3\n")); + assert!(!detailed.contains("Hierarchy\n")); + assert!(!detailed.contains("after other")); + let limited = diff(&[ + "--show-cell", + "CHIP", + "--details", + "--max-changes", + "1", + "--hierarchy", + "ascii", + ]); + assert!(limited.contains("4 object changes over the limit")); + assert!(!limited.contains("~ 1/0: rectangle")); + assert!(!limited.contains("property[17]=\"old\"")); + assert!(limited.ends_with("Hierarchy\n* CHIP\n leaf x1 -> x3\n")); + + let all = diff(&["--details"]); + assert!(all.contains("Cells:\n- removed\n+ added\n* CHIP\n")); + assert!(!all.contains("payload in")); + assert_eq!( + diff(&["--details", "--format", "json"]), + diff(&["--format", "json"]) + ); +} + #[test] fn cli_decodes_gzip_by_header_and_checks_the_trailer() { let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); @@ -1830,6 +1932,22 @@ fn layer_only_shape_replacements_keep_evidence_and_respect_selection() { diff.text(2), "Different\nCells:\n* CHIP\n ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n" ); + assert!( + diff.render_text_with_options( + 2, + &Default::default(), + true, + TextReportOptions { + details: true, + ..Default::default() + }, + ) + .ends_with(concat!( + " ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10)\n", + " + 2/0: rectangle (0, 0) to (10, 10)\n", + )) + ); let limited = diff.text(1); assert!(limited.contains("2 object changes over the limit")); assert!(!limited.contains("Cells:")); From 852660bcc608e109e56b5ed60cb2b8d01b4c22c9 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 08:35:08 -0800 Subject: [PATCH 36/41] Separate text evidence selection from report formatting --- src/review.rs | 122 ++++++++++++++-------------------------- src/review/selection.rs | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 81 deletions(-) diff --git a/src/review.rs b/src/review.rs index 70b2b4b..fc9e16e 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1579,62 +1579,22 @@ impl Comparison { .unwrap(); } let body_start = out.len(); - let mut removed_cells = String::new(); - let mut added_cells = String::new(); + let selection = selection::TextSelection::new(&self.cells, show, max_objects); let mut layers = changes::LayerRows::new(&self.layer_legend, &self.layer_names); let mut annotations = Vec::new(); let mut covered_edges = BTreeSet::new(); - let mut detail_total = 0; - let mut selected = 0; - let mut shown: usize = 0; - let mut hidden_cell_changes = 0; - let visible = - |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); - for cell in &self.cells { - let marker = cell_marker(cell.old.is_some(), cell.new.is_some()); - let attributes_changed = marker == '*' && cell.attributes_changed(); - if marker == '*' { - detail_total += cell.removed.len() + cell.added.len(); - } - if show.cell.as_ref().is_some_and(|name| name != &cell.name) { - hidden_cell_changes += usize::from(marker != '*' || attributes_changed); - continue; - } + for selected in &selection.modified_cells { + let cell = selected.cell; let name = cell_label(&cell.name); - match marker { - '-' => { - writeln!(removed_cells, "- {name}").unwrap(); - continue; - } - '+' => { - writeln!(added_cells, "+ {name}").unwrap(); - continue; - } - _ => (), - }; - let note = if attributes_changed { + let note = if selected.attributes_changed { " (attributes changed)" } else { "" }; - let cell_start = out.len(); - let previously_shown = shown; writeln!(out, "* {name}{note}").unwrap(); let mut polygons = changes::PolygonChanges::default(); let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); - let mut paired_old = BTreeSet::new(); - let mut paired_new = BTreeSet::new(); - for replacement in &cell.replacements { - let a = &cell.removed[replacement.removed]; - let b = &cell.added[replacement.added]; - if !visible(a) || !visible(b) { - continue; - } - paired_old.insert(replacement.removed); - paired_new.insert(replacement.added); - if max_objects - shown < 2 { - continue; - } + for &(a, b) in &selected.replacements { if !details && a.layer == b.layer && is_polygon(a.kind) { polygons.replacement(a, b); } else { @@ -1661,39 +1621,31 @@ impl Comparison { .replacement(a, b, row_start); } } - shown += 2; } let mut removed_refs = BTreeMap::new(); let mut added_refs = BTreeMap::new(); - for (removed, objects, paired, references) in [ - (true, &cell.removed, &paired_old, &mut removed_refs), - (false, &cell.added, &paired_new, &mut added_refs), + for (removed, objects, references) in [ + (true, &selected.removed, &mut removed_refs), + (false, &selected.added, &mut added_refs), ] { let sign = if removed { "-" } else { "+" }; let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); - for (index, object) in objects.iter().enumerate() { - if !visible(object) { - continue; - } - selected += 1; - if !paired.contains(&index) && shown < max_objects { - if !details && is_polygon(object.kind) { - polygons.unmatched(object, removed); - } else if !details && let Some(target) = object.target.as_deref() { - let (_, counts) = references - .entry(target) - .or_insert_with(|| (object, changes::ReferenceCounts::default())); - counts.add(object); - } else if let Some((last, count)) = unmatched.last_mut() - && last.kind == object.kind - && last.fields == object.fields - && last.properties == object.properties - { - *count += 1; - } else { - unmatched.push((object, 1)); - } - shown += 1; + for &object in objects { + if !details && is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if !details && let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; + } else { + unmatched.push((object, 1)); } } // The comparison orders these objects by kind, fields, and properties. @@ -1766,9 +1718,6 @@ impl Comparison { } } polygons.write(&mut out, &mut layers); - if shown == previously_shown && !attributes_changed { - out.truncate(cell_start); - } } let mut before_cells = String::new(); layers.finish(&out, &mut annotations, &mut before_cells); @@ -1785,17 +1734,28 @@ impl Comparison { annotated.push_str(&out[start..]); out = annotated; } - if out.len() > body_start || !removed_cells.is_empty() || !added_cells.is_empty() { + if !selection.modified_cells.is_empty() + || !selection.removed_cells.is_empty() + || !selection.added_cells.is_empty() + { before_cells.push_str("Cells:\n"); - before_cells.push_str(&removed_cells); - before_cells.push_str(&added_cells); + for name in selection.removed_cells { + writeln!(before_cells, "- {}", cell_label(name)).unwrap(); + } + for name in selection.added_cells { + writeln!(before_cells, "+ {}", cell_label(name)).unwrap(); + } } out.insert_str(body_start, &before_cells); let mut omissions = Vec::new(); for (count, label, reason) in [ - (detail_total - selected, "object change", "filtered out"), - (selected - shown, "object change", "over the limit"), - (hidden_cell_changes, "cell-level change", "filtered out"), + (selection.filtered_objects, "object change", "filtered out"), + (selection.limited_objects, "object change", "over the limit"), + ( + selection.filtered_cells, + "cell-level change", + "filtered out", + ), ] { if count != 0 { let plural = if count == 1 { "" } else { "s" }; diff --git a/src/review/selection.rs b/src/review/selection.rs index db52a59..71b2594 100644 --- a/src/review/selection.rs +++ b/src/review/selection.rs @@ -16,6 +16,104 @@ pub struct DisplayFilter { pub layer: Option, } +#[derive(Default)] +pub(super) struct TextSelection<'a> { + pub removed_cells: Vec<&'a str>, + pub added_cells: Vec<&'a str>, + pub modified_cells: Vec>, + pub filtered_objects: usize, + pub limited_objects: usize, + pub filtered_cells: usize, +} + +pub(super) struct TextCellSelection<'a> { + pub cell: &'a CellChange, + pub attributes_changed: bool, + pub replacements: Vec<(&'a ObjectChange, &'a ObjectChange)>, + pub removed: Vec<&'a ObjectChange>, + pub added: Vec<&'a ObjectChange>, +} + +impl<'a> TextSelection<'a> { + pub fn new(cells: &'a [CellChange], show: &DisplayFilter, mut remaining: usize) -> Self { + let mut selection = Self::default(); + let visible = + |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); + for cell in cells { + let marker = cell_marker(cell.old.is_some(), cell.new.is_some()); + let attributes_changed = marker == '*' && cell.attributes_changed(); + if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + selection.filtered_cells += usize::from(marker != '*' || attributes_changed); + if marker == '*' { + selection.filtered_objects += cell.removed.len() + cell.added.len(); + } + continue; + } + match marker { + '-' => { + selection.removed_cells.push(&cell.name); + continue; + } + '+' => { + selection.added_cells.push(&cell.name); + continue; + } + _ => (), + } + + let mut selected = TextCellSelection { + cell, + attributes_changed, + replacements: Vec::new(), + removed: Vec::new(), + added: Vec::new(), + }; + let mut paired_old = BTreeSet::new(); + let mut paired_new = BTreeSet::new(); + for pair in &cell.replacements { + let old = &cell.removed[pair.removed]; + let new = &cell.added[pair.added]; + if visible(old) && visible(new) { + paired_old.insert(pair.removed); + paired_new.insert(pair.added); + if remaining >= 2 { + selected.replacements.push((old, new)); + remaining -= 2; + } else { + selection.limited_objects += 2; + } + } + } + // A layer filter may leave one side of a pair eligible on its own. + for (objects, paired, selected) in [ + (&cell.removed, &paired_old, &mut selected.removed), + (&cell.added, &paired_new, &mut selected.added), + ] { + for (index, object) in objects.iter().enumerate() { + if !visible(object) { + selection.filtered_objects += 1; + } else if !paired.contains(&index) { + if remaining == 0 { + selection.limited_objects += 1; + } else { + selected.push(object); + remaining -= 1; + } + } + } + } + if attributes_changed + || !selected.replacements.is_empty() + || !selected.removed.is_empty() + || !selected.added.is_empty() + { + selection.modified_cells.push(selected); + } + } + selection + } +} + impl Scope { pub(super) fn write_restrictions(&self, out: &mut String) { if self == &Self::default() { From b5071444f5c9648a8bd627bb499cd2ee81f4b711 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 08:44:28 -0800 Subject: [PATCH 37/41] Render layer labels and terminal styles from report roles --- docs/native-api.md | 2 + src/bin/pheon.rs | 53 ++-- src/bin/pheon/color.rs | 251 +++------------ src/review.rs | 668 ++++++++++++++++++++++++---------------- src/review/changes.rs | 261 ++++++++-------- src/review/selection.rs | 4 +- src/review/text.rs | 252 +++++++++++++++ 7 files changed, 843 insertions(+), 648 deletions(-) create mode 100644 src/review/text.rs diff --git a/docs/native-api.md b/docs/native-api.md index ee55e50..7e0a020 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -37,6 +37,8 @@ reports. Both comparison types provide `render_text_with_ascii_hierarchy` to render a combined report with redundant terminal edges omitted. Pass `TextReportOptions { details: true, ..Default::default() }` to `render_text_with_options` to show individual object evidence. +Use `TextReport::new(true)` with `write_text_report`, `write_canonical_dump`, +or `write_record_dump` to get plain text and style ranges for presentation. Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison that can reuse matching cells. Its `compare` and `compare_text` methods return diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 5b99354..72db205 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -5,13 +5,15 @@ use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ Comparison, Diagnostic, Document, Layer, Outcome, RevisionPair, Side, TextComparison, - TextReportOptions, dump_records, + TextReportOptions, TextStyle, names::{parse_layer, parse_layer_name, resolve_layer_name, write_layer_names}, selection::{DisplayFilter, Scope}, + write_record_dump, }; use std::{ collections::BTreeMap, ffi::{OsStr, OsString}, + fmt::Write as _, fs, io::{self, Read, Write}, path::PathBuf, @@ -423,16 +425,14 @@ fn run(args: &[OsString]) -> Result { return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); } let read = |path: &OsString| read_input(path).map_err(|error| error.message); + let printer = color::Printer::new(color_choice); if raw { if selectors.scope != Scope::default() || selectors.layer_name.is_some() { return Err("record dumps do not accept object scope selectors".to_string()); } - color::print( - &dump_records(&read(&paths[0])?)?, - color::Report::Records, - color_choice, - ) - .map_err(|error| error.to_string())?; + let mut report = printer.report(); + write_record_dump(&mut report, &read(&paths[0])?)?; + printer.print(&report).map_err(|error| error.to_string())?; return Ok(0); } if !compare { @@ -443,8 +443,9 @@ fn run(args: &[OsString]) -> Result { .map_err(|error| error.message)?; let doc = doc.selected(&selectors.scope, false)?; if command == "dump" { - color::print(&doc.canonical_dump(), color::Report::Dump, color_choice) - .map_err(|error| error.to_string())?; + let mut report = printer.report(); + doc.write_canonical_dump(&mut report); + printer.print(&report).map_err(|error| error.to_string())?; } else { stdout(&doc.hierarchy(format == "mermaid", depth))?; for diagnostic in &doc.diagnostics { @@ -464,7 +465,7 @@ fn run(args: &[OsString]) -> Result { |pair| pair.compare_text(), ) .map_err(|error| error.message)?; - let mut output = String::new(); + let mut output = printer.report(); if git { let action = match (old_missing, new_missing) { (true, false) => " added", @@ -472,19 +473,28 @@ fn run(args: &[OsString]) -> Result { _ => "", }; if paths.len() == 9 { - output.push_str(&format!( - "layout {:?} -> {:?}{action}\n", - paths[0], paths[7] - )); + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?} -> {:?}{action}", paths[0], paths[7]), + ); } else { - output.push_str(&format!("layout {:?}{action}\n", paths[0])); + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?}{action}", paths[0]), + ); } + writeln!(output).unwrap(); if !old_missing && !new_missing && paths[3] != paths[6] { - output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); + output.write_styled( + TextStyle::Heading, + format_args!("mode {:?} -> {:?}", paths[3], paths[6]), + ); + writeln!(output).unwrap(); } } let show_library_presence = !old_missing && !new_missing; - output.push_str(&diff.render_text_with_options( + diff.write_text_report( + &mut output, max_changes, &selectors.filter, show_library_presence, @@ -492,13 +502,12 @@ fn run(args: &[OsString]) -> Result { details, ascii_depth: (format == "ascii").then_some(depth), }, - )); + ); if format == "mermaid" { - output.push_str("Hierarchy\n```mermaid\n"); - output.push_str(&diff.hierarchy(true, depth)); - output.push_str("```\n"); + output.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + writeln!(output, "\n```mermaid\n{}```", diff.hierarchy(true, depth)).unwrap(); } - color::print(&output, color::Report::Diff, color_choice).map_err(|error| error.to_string())?; + printer.print(&output).map_err(|error| error.to_string())?; // Git's external driver expects successful rendering for either equality // or differences. Actual comparison/input failures remain nonzero. Ok(if git && diff.outcome().exit_code() == 1 { diff --git a/src/bin/pheon/color.rs b/src/bin/pheon/color.rs index e7ba8a0..82fb933 100644 --- a/src/bin/pheon/color.rs +++ b/src/bin/pheon/color.rs @@ -1,15 +1,7 @@ -use std::{ - fmt, - io::{self, IsTerminal, Write}, -}; +use pheon::review::{TextReport, TextStyle}; +use std::io::{self, IsTerminal, Write}; use termcolor::{BufferedStandardStream, Color, ColorChoice, ColorSpec, WriteColor}; -pub(super) enum Report { - Diff, - Dump, - Records, -} - pub(super) fn parse_choice(value: &str) -> Result { match value { "auto" => Ok(ColorChoice::Auto), @@ -19,218 +11,47 @@ pub(super) fn parse_choice(value: &str) -> Result { } } -pub(super) fn print(text: &str, report: Report, choice: ColorChoice) -> io::Result<()> { - let stdout = io::stdout(); - if choice == ColorChoice::Never || (choice == ColorChoice::Auto && !stdout.is_terminal()) { - return stdout.lock().write_all(text.as_bytes()); - } - let mut out = BufferedStandardStream::stdout(choice); - if !out.supports_color() { - out.write_all(text.as_bytes())?; - return out.flush(); - } +pub(super) struct Printer(Option); - let mut in_cells = false; - let mut in_hierarchy = false; - let mut in_mermaid = false; - for line in text.split_inclusive('\n') { - let body = line.strip_suffix('\n').unwrap_or(line); - match report { - Report::Diff => match body { - "```mermaid" => { - in_mermaid = true; - out.write_all(body.as_bytes())?; - } - "```" => { - in_mermaid = false; - out.write_all(body.as_bytes())?; - } - _ if in_mermaid => out.write_all(body.as_bytes())?, - _ => { - match body { - "Cells:" => in_cells = true, - "Hierarchy" => { - in_cells = false; - in_hierarchy = true; - } - "Diagnostics" => in_cells = false, - _ => (), - } - diff_line(&mut out, body, in_cells, in_hierarchy)?; - } - }, - Report::Dump => dump_line(&mut out, body)?, - Report::Records => record_line(&mut out, body)?, - } - if line.ends_with('\n') { - out.write_all(b"\n")?; +impl Printer { + pub fn new(choice: ColorChoice) -> Self { + if choice == ColorChoice::Never + || (choice == ColorChoice::Auto && !io::stdout().is_terminal()) + { + return Self(None); } + let stream = BufferedStandardStream::stdout(choice); + Self(stream.supports_color().then_some(stream)) } - out.flush() -} -fn paint( - out: &mut impl WriteColor, - value: impl fmt::Display, - color: Color, - bold: bool, -) -> io::Result<()> { - out.set_color(ColorSpec::new().set_fg(Some(color)).set_bold(bold))?; - let result = write!(out, "{value}"); - result.and(out.reset()) -} - -fn change_color(marker: &str) -> Option { - match marker { - "+" => Some(Color::Green), - "-" => Some(Color::Red), - "*" | "~" => Some(Color::Yellow), - _ => None, + pub fn report(&self) -> TextReport { + TextReport::new(self.0.is_some()) } -} -fn folded_endpoint(line: &str) -> Option<(&str, &str)> { - let mut chars = line.char_indices(); - let mut quoted = false; - let mut endpoint = None; - while let Some((index, ch)) = chars.next() { - match ch { - '"' => quoted = !quoted, - '\\' if quoted => { - chars.next(); - } - '→' if !quoted && line[index..].starts_with("→ ") => { - endpoint = Some(line.split_at(index + "→ ".len())); - } - _ => (), - } - } - endpoint -} - -fn diff_line( - out: &mut impl WriteColor, - line: &str, - in_cells: bool, - in_hierarchy: bool, -) -> io::Result<()> { - let content = line.trim_start_matches(' '); - let indent = &line[..line.len() - content.len()]; - let first = content.split_once(' ').map_or(content, |(word, _)| word); - if indent.is_empty() { - let color = match first { - "Equal" => Some(Color::Green), - "Different" | "Incomplete" | "Omitted" => Some(Color::Yellow), - "Failed" => Some(Color::Red), - "Cells:" | "Hierarchy" | "Diagnostics" | "Scope" | "Display" | "layout" | "mode" => { - Some(Color::Cyan) - } - _ => None, + pub fn print(self, report: &TextReport) -> io::Result<()> { + let text = report.as_str().as_bytes(); + let Some(mut out) = self.0 else { + return io::stdout().lock().write_all(text); }; - if let Some(color) = color { - return paint(out, line, color, true); + let mut end = 0; + for (range, style) in report.styles() { + out.write_all(&text[end..range.start])?; + let (color, bold) = match style { + TextStyle::Heading => (Color::Cyan, true), + TextStyle::Layer => (Color::Cyan, false), + TextStyle::Added => (Color::Green, false), + TextStyle::Removed => (Color::Red, false), + TextStyle::Changed => (Color::Yellow, false), + TextStyle::Success => (Color::Green, true), + TextStyle::Warning => (Color::Yellow, true), + TextStyle::Error => (Color::Red, true), + }; + out.set_color(ColorSpec::new().set_fg(Some(color)).set_bold(bold))?; + let written = out.write_all(&text[range.clone()]); + written.and(out.reset())?; + end = range.end; } - } - if let Some(color) = change_color(first) { - out.write_all(indent.as_bytes())?; - return paint(out, content, color, false); - } - if in_hierarchy - && line.contains('→') - && let Some((path, cell)) = folded_endpoint(line) - && let Some((marker, _)) = cell.split_once(' ') - && let Some(color) = change_color(marker) - { - out.write_all(path.as_bytes())?; - return paint(out, cell, color, false); - } - if in_cells && indent == " " { - if (layer_token(content).is_some() || content.starts_with("unknown layer: ")) - && let Some((layer, counts)) = content.rsplit_once(": ") - { - out.write_all(indent.as_bytes())?; - paint(out, layer, Color::Cyan, false)?; - out.write_all(b": ")?; - for (index, clause) in counts.split(" · ").enumerate() { - if index > 0 { - out.write_all(" · ".as_bytes())?; - } - let color = match clause.rsplit_once(' ').map(|(_, action)| action) { - Some("added") => Color::Green, - Some("removed") => Color::Red, - _ => Color::Yellow, - }; - paint(out, clause, color, false)?; - } - return Ok(()); - } - if let Some((target, counts)) = content.rsplit_once(" · -") - && let Some((removed, added)) = counts.split_once(", +") - { - let (added, totals) = added - .split_once(" · ") - .map_or((added, None), |(added, totals)| (added, Some(totals))); - write!(out, "{indent}{target} · ")?; - paint(out, format_args!("-{removed}"), Color::Red, false)?; - out.write_all(b", ")?; - paint(out, format_args!("+{added}"), Color::Green, false)?; - if let Some(totals) = totals { - write!(out, " · {totals}")?; - } - return Ok(()); - } - } - out.write_all(line.as_bytes()) -} - -fn layer_token(line: &str) -> Option<&str> { - let token = line.split([' ', ':']).next()?; - let (layer, datatype) = token.split_once('/')?; - (layer.parse::().is_ok() && datatype.parse::().is_ok()).then_some(token) -} - -fn dump_line(out: &mut impl WriteColor, line: &str) -> io::Result<()> { - if line.starts_with("cell ") || line.starts_with("library ") || line.starts_with("Scope ") { - return paint(out, line, Color::Cyan, true); - } - if line.starts_with("INCOMPLETE: ") { - return paint(out, line, Color::Yellow, true); - } - let mut object = line.trim_start_matches(' '); - if let Some((token, rest)) = object.split_once(' ') - && token - .strip_suffix('x') - .is_some_and(|count| count.parse::().is_ok()) - { - object = rest; - } - let heading = layer_token(object).or_else(|| { - ["reference", "array reference"] - .into_iter() - .find(|heading| { - object - .strip_prefix(heading) - .is_some_and(|rest| rest.starts_with(' ')) - }) - }); - if let Some(heading) = heading { - let prefix = &line[..line.len() - object.len()]; - out.write_all(prefix.as_bytes())?; - paint(out, heading, Color::Cyan, false)?; - return out.write_all(object[heading.len()..].as_bytes()); - } - out.write_all(line.as_bytes()) -} - -fn record_line(out: &mut impl WriteColor, line: &str) -> io::Result<()> { - let mut parts = line.splitn(4, ' '); - if let (Some(offset), Some(tag), Some(name), Some(data)) = - (parts.next(), parts.next(), parts.next(), parts.next()) - { - write!(out, "{offset} {tag} ")?; - paint(out, name, Color::Cyan, false)?; - write!(out, " {data}") - } else { - out.write_all(line.as_bytes()) + out.write_all(&text[end..])?; + out.flush() } } diff --git a/src/review.rs b/src/review.rs index fc9e16e..a9f23ae 100644 --- a/src/review.rs +++ b/src/review.rs @@ -17,11 +17,14 @@ pub mod names; mod pair; pub mod report; pub mod selection; +mod text; use changes::Replacement; use pair::CellReuse; pub use pair::RevisionPair; use selection::{DisplayFilter, Scope}; +use text::CellText; +pub use text::{TextReport, TextStyle}; /// Input revision associated with a diagnostic. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -166,12 +169,13 @@ impl Object<'_> { } } -fn format_object<'a>( - out: &mut fmt::Formatter<'_>, +fn format_object<'a, W: fmt::Write>( + out: &mut W, kind: u16, layer: Option, fields: impl Iterator + Clone, properties: impl IntoIterator, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, ) -> fmt::Result { let field = |tag| { fields @@ -193,10 +197,7 @@ fn format_object<'a>( } else { None }; - if let Some((layer, datatype)) = layer { - write!(out, "{layer}/{datatype}: ")?; - } - out.write_str(object_name(kind, rectangle.is_some()))?; + heading(out, object_name(kind, rectangle.is_some()))?; if let Some(target) = target { write!(out, " {}", quoted(target))?; } @@ -236,7 +237,7 @@ fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> { const POINTS_PER_LINE: usize = 4; -fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result { +fn write_points(out: &mut impl fmt::Write, data: &[u8]) -> fmt::Result { let mut points = XY::points(data); if points.len() == 1 { let [x, y] = points.next().unwrap(); @@ -258,6 +259,27 @@ fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result { impl fmt::Display for Object<'_> { fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +fn write_object_heading( + out: &mut impl fmt::Write, + layer: Option, + name: &str, +) -> fmt::Result { + if let Some((layer, datatype)) = layer { + write!(out, "{layer}/{datatype}: ")?; + } + out.write_str(name) +} + +impl Object<'_> { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { format_object( out, self.kind, @@ -267,6 +289,7 @@ impl fmt::Display for Object<'_> { .iter() .map(|(tag, data)| (*tag, data.as_ref())), self.key.properties.iter().copied(), + heading, ) } } @@ -886,12 +909,21 @@ impl<'a> Document<'a> { pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { let revision = Revision::from_document(self); - render_hierarchy(&revision, &revision, None, mermaid, max_depth, None) + render_hierarchy(&revision, &revision, None, mermaid, max_depth) } pub fn canonical_dump(&self) -> String { + let mut out = TextReport::default(); + self.write_canonical_dump(&mut out); + out.into_string() + } + + /// Append the canonical object dump and its presentation roles. + pub fn write_canonical_dump(&self, out: &mut TextReport) { if !self.present { - return "library absent\n".to_string(); + out.write_styled(TextStyle::Heading, format_args!("library absent")); + out.push('\n'); + return; } let version_index = self .header @@ -901,23 +933,29 @@ impl<'a> Document<'a> { let header = |tag| self.header.iter().find(|(t, _)| *t == tag).unwrap().1; let (user, meters) = header(RTAG_UNITS).split_at(8); let version = i16::from_be_bytes(self.header[version_index].1.try_into().unwrap()); - let mut out = format!( - "library {} (GDSII {version}, {} user units/dbu, {} m/dbu)\n", - quoted(header(RTAG_LIBNAME)), - format_exact_value(RTAG_UNITS, user), - format_exact_value(RTAG_UNITS, meters) + out.write_styled( + TextStyle::Heading, + format_args!( + "library {} (GDSII {version}, {} user units/dbu, {} m/dbu)", + quoted(header(RTAG_LIBNAME)), + format_exact_value(RTAG_UNITS, user), + format_exact_value(RTAG_UNITS, meters) + ), ); - self.scope.write_restrictions(&mut out); + out.push('\n'); + self.scope.write_restrictions(out); for (index, (tag, data)) in self.header.iter().enumerate() { if index != version_index && !matches!(*tag, RTAG_LIBNAME | RTAG_UNITS) { writeln!(out, "{}={}", tag_name(*tag), format_exact_value(*tag, data)).unwrap(); } } for ((layer, dtype), name) in &self.layer_names { - writeln!(out, "{layer}/{dtype}: name {name:?}").unwrap(); + out.write_styled(TextStyle::Layer, format_args!("{layer}/{dtype}")); + writeln!(out, ": name {name:?}").unwrap(); } for (name, cell) in &self.cells { - writeln!(out, "cell {name:?}").unwrap(); + out.write_styled(TextStyle::Heading, format_args!("cell {name:?}")); + out.push('\n'); for (tag, data) in &cell.attributes { writeln!( out, @@ -937,17 +975,34 @@ impl<'a> Document<'a> { while objects.next_if(|next| next.key == object.key).is_some() { count += 1; } - if count == 1 { - writeln!(out, " {object}").unwrap(); - } else { - writeln!(out, " {count}x {object}").unwrap(); + out.push_str(" "); + if count != 1 { + write!(out, "{count}x ").unwrap(); } + object + .format_with_heading(out, |out, name| { + if let Some((layer, datatype)) = object.layer { + out.write_styled(TextStyle::Layer, format_args!("{layer}/{datatype}")); + out.push_str(": "); + } + if matches!(object.kind, RTAG_SREF | RTAG_AREF) && object.layer.is_none() { + out.write_styled(TextStyle::Layer, format_args!("{name}")); + } else { + out.push_str(name); + } + Ok(()) + }) + .unwrap(); + out.push('\n'); } } for diagnostic in &self.diagnostics { - writeln!(out, "INCOMPLETE: {}", diagnostic.message).unwrap(); + out.write_styled( + TextStyle::Warning, + format_args!("INCOMPLETE: {}", diagnostic.message), + ); + out.push('\n'); } - out } pub fn record_dump(&self) -> String { @@ -957,18 +1012,30 @@ impl<'a> Document<'a> { /// Inspect record boundaries even when object grammar or metadata is invalid. pub fn dump_records(input: &[u8]) -> Result { - Ok(format_records(&read_records(input)?)) + let mut out = TextReport::default(); + write_record_dump(&mut out, input)?; + Ok(out.into_string()) +} + +/// Append the raw record dump and its presentation roles. +pub fn write_record_dump(out: &mut TextReport, input: &[u8]) -> Result<(), String> { + write_records(out, &read_records(input)?); + Ok(()) } fn format_records(records: &[Record<'_>]) -> String { - let mut out = String::new(); + let mut out = TextReport::default(); + write_records(&mut out, records); + out.into_string() +} + +fn write_records(out: &mut TextReport, records: &[Record<'_>]) { for record in records { + write!(out, "{:010} {:04x} ", record.offset, record.tag).unwrap(); + out.write_styled(TextStyle::Layer, format_args!("{}", tag_name(record.tag))); writeln!( out, - "{:010} {:04x} {} size={} {}", - record.offset, - record.tag, - tag_name(record.tag), + " size={} {}", record.data.len() + 4, format_data( record.tag, @@ -981,7 +1048,6 @@ fn format_records(records: &[Record<'_>]) -> String { ) .unwrap(); } - out } /// A retained comparison field. Polygon coordinates and string padding follow @@ -1019,6 +1085,16 @@ pub struct ObjectChange { impl fmt::Display for ObjectChange { fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +impl ObjectChange { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { format_object( out, self.kind, @@ -1029,6 +1105,7 @@ impl fmt::Display for ObjectChange { self.properties .iter() .map(|(attribute, data)| (*attribute, data.as_slice())), + heading, ) } } @@ -1249,6 +1326,18 @@ impl TextComparison { .render_text_with_options(max_objects, show, show_library_presence, options) } + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + self.comparison + .write_text_report(out, max_objects, show, show_library_presence, options); + } + /// Render the cell changes and their ASCII hierarchy together. pub fn render_text_with_ascii_hierarchy( &self, @@ -1450,14 +1539,7 @@ impl Comparison { /// Render the affected hierarchy. ASCII output shows relationships between cells. pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { - render_hierarchy( - &self.old, - &self.new, - Some(&self.cells), - mermaid, - max_depth, - None, - ) + render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) } /// Limit presentation only: the complete comparison has already been made. @@ -1512,19 +1594,59 @@ impl Comparison { show_library_presence: bool, options: TextReportOptions, ) -> String { - let TextReportOptions { - details, - ascii_depth, - } = options; + let mut out = TextReport::default(); + self.write_text_report(&mut out, max_objects, show, show_library_presence, options); + out.into_string() + } + + /// Append the selected changes and their presentation roles to a text report. + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + let selection = selection::TextSelection::new(&self.cells, show, max_objects); let outcome = self.outcome(); - let mut out = match outcome { - Outcome::Equal => "Equal", - Outcome::Different => "Different", - Outcome::Incomplete { .. } => "Incomplete", - Outcome::Failed => "Failed", + let headline = out.len(); + let (result, style) = match outcome { + Outcome::Equal => ("Equal", TextStyle::Success), + Outcome::Different => ("Different", TextStyle::Warning), + Outcome::Incomplete { .. } => ("Incomplete", TextStyle::Warning), + Outcome::Failed => ("Failed", TextStyle::Error), + }; + out.push_str(result); + let has_omissions = selection.filtered_objects != 0 + || selection.limited_objects != 0 + || selection.filtered_cells != 0; + if !self.cells.is_empty() + && (self.cells.len() > 3 || !outcome.is_complete() || has_omissions) + { + let (mut modified, mut added, mut removed) = (0, 0, 0); + for cell in &self.cells { + match cell_marker(cell.old.is_some(), cell.new.is_some()) { + '-' => removed += 1, + '+' => added += 1, + _ => modified += 1, + } + } + let mut counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + write!(out, " · {count} {noun} {action}").unwrap(); + for (count, action) in counts { + write!(out, ", {count} {action}").unwrap(); + } + } } - .to_string(); - let headline_end = out.len(); let library_changes = self.library_changes.iter().filter(|change| { show_library_presence || !matches!(change, LibraryChange::Presence { .. }) }); @@ -1537,9 +1659,11 @@ impl Comparison { write!(out, " · {count} {label}{plural}").unwrap(); } } + out.mark(headline, style); out.push('\n'); - self.old.scope.write_restrictions(&mut out); + self.old.scope.write_restrictions(out); if show.cell.is_some() || show.layer.is_some() { + let start = out.len(); out.push_str("Display filter"); if let Some(cell) = &show.cell { write!(out, " cell={cell:?}").unwrap(); @@ -1547,9 +1671,29 @@ impl Comparison { if let Some((layer, datatype)) = show.layer { write!(out, " layer={layer}/{datatype}").unwrap(); } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + if has_omissions { + let start = out.len(); + out.push_str("Omitted"); + for (count, label, reason) in [ + (selection.filtered_objects, "object change", "filtered out"), + (selection.limited_objects, "object change", "over the limit"), + ( + selection.filtered_cells, + "cell-level change", + "filtered out", + ), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural} {reason}").unwrap(); + } + } + out.mark(start, TextStyle::Warning); out.push('\n'); } - let notice_start = out.len(); for change in library_changes { writeln!(out, "library {change}").unwrap(); } @@ -1578,229 +1722,39 @@ impl Comparison { ) .unwrap(); } - let body_start = out.len(); - let selection = selection::TextSelection::new(&self.cells, show, max_objects); - let mut layers = changes::LayerRows::new(&self.layer_legend, &self.layer_names); - let mut annotations = Vec::new(); + + let mut body = CellText::new(&self.layer_legend, &self.layer_names, out.is_styled()); let mut covered_edges = BTreeSet::new(); for selected in &selection.modified_cells { - let cell = selected.cell; - let name = cell_label(&cell.name); - let note = if selected.attributes_changed { - " (attributes changed)" - } else { - "" - }; - writeln!(out, "* {name}{note}").unwrap(); - let mut polygons = changes::PolygonChanges::default(); - let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); - for &(a, b) in &selected.replacements { - if !details && a.layer == b.layer && is_polygon(a.kind) { - polygons.replacement(a, b); - } else { - out.push_str(" ~ "); - let row_start = out.len(); - if a.layer != b.layer { - changes::write_layer_transition(&mut out, a, b, Some(&mut layers)); - } else { - layers.record(a.layer, row_start); - if details { - out.push_str(&changes::replacement_heading(a, b)); - } else { - out.push_str(&changes::replacement_summary(a, b)); - } - } - out.push('\n'); - if details { - changes::write_replacement_objects(&mut out, a, b); - } - if let Some(target) = &a.target { - shown_refs - .entry(target) - .or_default() - .replacement(a, b, row_start); - } - } - } - let mut removed_refs = BTreeMap::new(); - let mut added_refs = BTreeMap::new(); - for (removed, objects, references) in [ - (true, &selected.removed, &mut removed_refs), - (false, &selected.added, &mut added_refs), - ] { - let sign = if removed { "-" } else { "+" }; - let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); - for &object in objects { - if !details && is_polygon(object.kind) { - polygons.unmatched(object, removed); - } else if !details && let Some(target) = object.target.as_deref() { - let (_, counts) = references - .entry(target) - .or_insert_with(|| (object, changes::ReferenceCounts::default())); - counts.add(object); - } else if let Some((last, count)) = unmatched.last_mut() - && last.kind == object.kind - && last.fields == object.fields - && last.properties == object.properties - { - *count += 1; - } else { - unmatched.push((object, 1)); - } - } - // The comparison orders these objects by kind, fields, and properties. - for (object, count) in unmatched { - write!(out, " {sign} ").unwrap(); - if count > 1 { - write!(out, "{count}× ").unwrap(); - } - let row_start = out.len(); - layers.record(object.layer, row_start); - if details { - writeln!(out, "{object}").unwrap(); - if let Some(target) = object.target.as_deref() { - shown_refs - .entry(target) - .or_default() - .individual(object, removed, count, row_start); - } - } else { - changes::write_unmatched(&mut out, object); - } - } - } - let no_refs = changes::ReferenceCounts::default(); - for (target, (first, counts)) in removed_refs { - let row_start = out.len(); - if let Some((_, added)) = added_refs.remove(target) { - counts.write_change(&mut out, target, &added); - shown_refs - .entry(target) - .or_default() - .unmatched(&counts, &added, row_start); - } else { - counts.write(&mut out, "-", target, first); - shown_refs - .entry(target) - .or_default() - .unmatched(&counts, &no_refs, row_start); - } - } - for (target, (first, counts)) in added_refs { - let row_start = out.len(); - counts.write(&mut out, "+", target, first); - shown_refs - .entry(target) - .or_default() - .unmatched(&no_refs, &counts, row_start); - } - if !shown_refs.is_empty() { - for target in cell - .removed - .iter() - .chain(&cell.added) - .filter_map(|o| o.target.as_deref()) - { - if let Some(shown) = shown_refs.get_mut(target) { - shown.count_change(); - } - } - for (target, shown) in shown_refs { - if shown.complete() { - let edge = (cell.name.clone(), target.to_owned()); - let old = self.old.edges.get(&edge).copied().unwrap_or(0); - let new = self.new.edges.get(&edge).copied().unwrap_or(0); - shown.annotate(&out, &mut annotations, old, new); - if ascii_depth.is_some() && outcome.is_complete() { - covered_edges.insert(edge); - } - } - } - } - polygons.write(&mut out, &mut layers); - } - let mut before_cells = String::new(); - layers.finish(&out, &mut annotations, &mut before_cells); - // Resolve row offsets before inserting the section headers. - if !annotations.is_empty() { - annotations.sort_unstable_by_key(|(at, _)| *at); - let mut annotated = String::with_capacity(out.len()); - let mut start = 0; - for (at, annotation) in annotations { - annotated.push_str(&out[start..at]); - annotated.push_str(&annotation); - start = at; - } - annotated.push_str(&out[start..]); - out = annotated; + self.write_cell_details( + &mut body, + selected, + options.details, + (options.ascii_depth.is_some() && outcome.is_complete()) + .then_some(&mut covered_edges), + ); } + body.write_legend(out); if !selection.modified_cells.is_empty() || !selection.removed_cells.is_empty() || !selection.added_cells.is_empty() { - before_cells.push_str("Cells:\n"); + out.write_styled(TextStyle::Heading, format_args!("Cells:")); + out.push('\n'); for name in selection.removed_cells { - writeln!(before_cells, "- {}", cell_label(name)).unwrap(); + out.write_styled(TextStyle::Removed, format_args!("- {}", cell_label(name))); + out.push('\n'); } for name in selection.added_cells { - writeln!(before_cells, "+ {}", cell_label(name)).unwrap(); - } - } - out.insert_str(body_start, &before_cells); - let mut omissions = Vec::new(); - for (count, label, reason) in [ - (selection.filtered_objects, "object change", "filtered out"), - (selection.limited_objects, "object change", "over the limit"), - ( - selection.filtered_cells, - "cell-level change", - "filtered out", - ), - ] { - if count != 0 { - let plural = if count == 1 { "" } else { "s" }; - omissions.push(format!("{count} {label}{plural} {reason}")); - } - } - if !omissions.is_empty() { - out.insert_str( - notice_start, - &format!("Omitted · {}\n", omissions.join(" · ")), - ); - } - const COMPACT_HEADLINE_CELL_LIMIT: usize = 3; - if !self.cells.is_empty() - && (self.cells.len() > COMPACT_HEADLINE_CELL_LIMIT - || !outcome.is_complete() - || !omissions.is_empty()) - { - let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize); - for cell in &self.cells { - match cell_marker(cell.old.is_some(), cell.new.is_some()) { - '-' => removed += 1, - '+' => added += 1, - _ => modified += 1, - } - } - let mut counts = [ - (modified, "modified"), - (added, "added"), - (removed, "removed"), - ] - .into_iter() - .filter(|(count, _)| *count != 0); - if let Some((count, action)) = counts.next() { - let noun = if count == 1 { "cell" } else { "cells" }; - let mut summary = format!(" · {count} {noun} {action}"); - for (count, action) in counts { - write!(summary, ", {count} {action}").unwrap(); - } - out.insert_str(headline_end, &summary); + out.write_styled(TextStyle::Added, format_args!("+ {}", cell_label(name))); + out.push('\n'); } + body.write_body(out); } // General comparison errors precede revision-specific errors in text. if !self.diagnostics.is_empty() { - out.push_str("Diagnostics\n"); + out.write_styled(TextStyle::Heading, format_args!("Diagnostics")); + out.push('\n'); for diagnostic in self .diagnostics .iter() @@ -1810,17 +1764,167 @@ impl Comparison { writeln!(out, " {diagnostic}").unwrap(); } } - if let Some(max_depth) = ascii_depth { - out.push_str(&render_hierarchy( + if let Some(max_depth) = options.ascii_depth { + write_hierarchy( + out, &self.old, &self.new, Some(&self.cells), false, max_depth, Some(&covered_edges), - )); + ); } - out + } + + fn write_cell_details( + &self, + out: &mut CellText<'_>, + selected: &selection::TextCellSelection<'_>, + details: bool, + mut covered_edges: Option<&mut BTreeSet<(String, String)>>, + ) { + let cell = selected.cell; + let name = cell_label(&cell.name); + let note = if selected.attributes_changed { + " (attributes changed)" + } else { + "" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "* {name}{note}").unwrap(); + let mut polygons = changes::PolygonChanges::default(); + let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); + for &(a, b) in &selected.replacements { + if !details && a.layer == b.layer && is_polygon(a.kind) { + polygons.replacement(a, b); + continue; + } + out.push_str(" "); + out.style(Some(TextStyle::Changed)); + out.push_str("~ "); + if a.layer != b.layer { + changes::write_layer_transition(out, a, b); + out.push('\n'); + } else { + if let Some(layer) = a.layer { + out.write_layer(layer, None); + out.push_str(": "); + } + let description = changes::replacement_description(a, b, details); + let (heading, rest) = match description.split_once('\n') { + Some((heading, rest)) => (heading, Some(rest)), + None => (description.as_str(), None), + }; + out.push_str(heading); + if let Some(target) = &a.target { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .replacement(a, b, placements); + } + out.push('\n'); + if let Some(rest) = rest { + writeln!(out, "{rest}").unwrap(); + } + } + if details { + changes::write_replacement_objects(out, a, b); + } + } + let mut removed_refs = BTreeMap::new(); + let mut added_refs = BTreeMap::new(); + for (removed, objects, references) in [ + (true, &selected.removed, &mut removed_refs), + (false, &selected.added, &mut added_refs), + ] { + let sign = if removed { '-' } else { '+' }; + let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); + for &object in objects { + if !details && is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if !details && let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; + } else { + unmatched.push((object, 1)); + } + } + // The comparison orders these objects by kind, fields, and properties. + for (object, count) in unmatched { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if count > 1 { + write!(out, "{count}× ").unwrap(); + } + changes::write_unmatched(out, object, details); + if let Some(target) = object.target.as_deref() { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .individual(object, removed, count, placements); + } + out.push('\n'); + } + } + let no_refs = changes::ReferenceCounts::default(); + for (target, (first, counts)) in removed_refs { + if let Some((_, added)) = added_refs.remove(target) { + let placements = counts.write_change(out, target, &added); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &added, placements); + } else { + let placements = counts.write(out, '-', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &no_refs, placements); + } + } + for (target, (first, counts)) in added_refs { + let placements = counts.write(out, '+', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&no_refs, &counts, placements); + } + if !shown_refs.is_empty() { + for target in cell + .removed + .iter() + .chain(&cell.added) + .filter_map(|o| o.target.as_deref()) + { + if let Some(shown) = shown_refs.get_mut(target) { + shown.count_change(); + } + } + for (target, shown) in shown_refs { + if shown.complete() { + let edge = (cell.name.clone(), target.to_owned()); + let old = self.old.edges.get(&edge).copied().unwrap_or(0); + let new = self.new.edges.get(&edge).copied().unwrap_or(0); + shown.write_placements(out, old, new); + if let Some(covered) = covered_edges.as_deref_mut() { + covered.insert(edge); + } + } + } + } + polygons.write(out); } } @@ -1885,10 +1989,23 @@ fn render_hierarchy( changes: Option<&[CellChange]>, mermaid: bool, max_depth: usize, - covered_edges: Option<&BTreeSet<(String, String)>>, ) -> String { + let mut out = TextReport::default(); + write_hierarchy(&mut out, old, new, changes, mermaid, max_depth, None); + out.into_string() +} + +fn write_hierarchy( + out: &mut TextReport, + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, +) { if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { - return String::new(); + return; } let HierarchyAnalysis { @@ -1952,7 +2069,7 @@ fn render_hierarchy( visible.retain(|name| children.contains_key(name) || targets.contains(name)); } if changes.is_some() && !mermaid && children.is_empty() { - return String::new(); + return; } let roots: Vec<_> = visible.difference(&targets).cloned().collect(); // Distinguish rootless cycles from descendants hidden by the depth limit. @@ -2052,7 +2169,7 @@ fn render_hierarchy( .enumerate() .map(|(i, n)| (n, i)) .collect(); - let mut out = "flowchart TD\n".to_string(); + out.push_str("flowchart TD\n"); if changes.is_none() && new.scope != Scope::default() { writeln!(out, " %% {}", new.scope.description()).unwrap(); } @@ -2074,7 +2191,6 @@ fn render_hierarchy( if rows.is_empty() { out.push_str(" empty[\"No changed hierarchy\"]\n"); } - out } else { let single_placement = |parent: &String, target: &String| { let key = (parent.clone(), target.clone()); @@ -2097,9 +2213,10 @@ fn render_hierarchy( && (old.cells.contains(&child.name) || new.cells.contains(&child.name)) && single_placement(&parent.name, &child.name) }; - let mut out = "Hierarchy\n".to_string(); + out.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + out.push('\n'); if changes.is_none() { - new.scope.write_restrictions(&mut out); + new.scope.write_restrictions(out); } let mut index = 0; let mut ancestor_depths = Vec::new(); @@ -2130,6 +2247,7 @@ fn render_hierarchy( ancestor_depths.pop(); } out.push_str(&" ".repeat(ancestor_depths.len())); + let mut change = None; for (i, row) in rows[index..=end].iter().enumerate() { let name = &row.name; if i != 0 { @@ -2137,6 +2255,7 @@ fn render_hierarchy( } if direct.contains(name) { let marker = cell_marker(old.cells.contains(name), new.cells.contains(name)); + change = Some((out.len(), TextStyle::change(marker))); write!(out, "{marker} ").unwrap(); } if changes.is_some() { @@ -2151,14 +2270,17 @@ fn render_hierarchy( } } let last = &rows[end]; - writeln!(out, "{}{}", status(&last.name), last.note).unwrap(); + write!(out, "{}{}", status(&last.name), last.note).unwrap(); + if let Some((start, style)) = change { + out.mark(start, style); + } + out.push('\n'); ancestor_depths.push(last.depth); index = end + 1; } if visible.is_empty() { out.push_str("No changed hierarchy\n"); } - out } } diff --git a/src/review/changes.rs b/src/review/changes.rs index 824b00d..c5a69cf 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -137,12 +137,8 @@ pub(super) fn field_changes<'a>( .filter_map(|(tag, (before, after))| (before != after).then_some((tag, before, after))) } -fn polygon_heading(object: &ObjectChange, rectangular: bool) -> String { - object_heading(object, object_name(object.kind, rectangular)) -} - -fn object_heading(object: &ObjectChange, kind: &str) -> String { - match object.layer { +fn object_heading(object: &ObjectChange, kind: &str, with_layer: bool) -> String { + match object.layer.filter(|_| with_layer) { Some((layer, datatype)) => format!("{layer}/{datatype}: {kind}"), None => kind.to_string(), } @@ -156,25 +152,39 @@ fn object_xy(object: &ObjectChange) -> &[u8] { .map_or(&[], |field| field.data.as_slice()) } -pub(super) fn write_unmatched(out: &mut String, object: &ObjectChange) { - if object.kind == RTAG_PATH +fn write_cell_heading(out: &mut CellText<'_>, layer: Option, name: &str) { + if let Some(layer) = layer { + out.write_layer(layer, None); + out.push_str(": "); + } + out.push_str(name); +} + +pub(super) fn write_unmatched(out: &mut CellText<'_>, object: &ObjectChange, details: bool) { + if !details + && object.kind == RTAG_PATH && let Some([xmin, ymin, xmax, ymax]) = object.bounds { let vertices = XY::points(object_xy(object)).len(); if vertices > POINTS_PER_LINE { - let heading = object_heading(object, "path"); + write_cell_heading(out, object.layer, "path"); let width = object .width .map_or_else(|| "absent".to_string(), |width| format!("{width} dbu")); - writeln!( + write!( out, - "{heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" + " · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" ) .unwrap(); return; } } - writeln!(out, "{object}").unwrap(); + object + .format_with_heading(out, |out, name| { + write_cell_heading(out, object.layer, name); + Ok(()) + }) + .unwrap(); } fn vertex_count(xy: &[u8]) -> usize { @@ -269,25 +279,27 @@ impl PolygonChanges { shapes.add(object, None); } - pub fn write(&self, out: &mut String, layers: &mut LayerRows<'_>) { + pub fn write(&self, out: &mut CellText<'_>) { for (layer, counts) in &self.0 { out.push_str(" "); - layers.record(*layer, out.len()); - if let Some((layer, datatype)) = layer { - write!(out, "{layer}/{datatype}: ").unwrap(); + out.style(Some(TextStyle::Layer)); + if let Some(layer) = layer { + out.write_layer(*layer, None); } else { - out.push_str("unknown layer: "); + out.push_str("unknown layer"); } + out.style(None); + out.push_str(": "); let groups = [ - ("changed", &counts.changed), - ("moved", &counts.moved), - ("removed", &counts.removed), - ("added", &counts.added), + ("changed", TextStyle::Changed, &counts.changed), + ("moved", TextStyle::Changed, &counts.moved), + ("removed", TextStyle::Removed, &counts.removed), + ("added", TextStyle::Added, &counts.added), ]; if let Some([dx, dy]) = counts.displacement && groups .iter() - .map(|(_, shapes)| shapes.total()) + .map(|(_, _, shapes)| shapes.total()) .sum::() == 1 { @@ -298,11 +310,12 @@ impl PolygonChanges { } else { "polygon" }; + out.style(Some(TextStyle::Changed)); writeln!(out, "{shape} moved by ({dx}, {dy}) dbu").unwrap(); continue; } let mut separator = ""; - for (action, shapes) in groups { + for (action, style, shapes) in groups { let terms = [ (shapes.rectangles, "rectangle", "rectangles"), (shapes.polygons, "polygon", "polygons"), @@ -316,7 +329,10 @@ impl PolygonChanges { }) .collect::>(); if !terms.is_empty() { - write!(out, "{separator}{} {action}", terms.join(", ")).unwrap(); + out.push_str(separator); + out.style(Some(style)); + write!(out, "{} {action}", terms.join(", ")).unwrap(); + out.style(None); separator = " · "; } } @@ -342,33 +358,49 @@ impl ReferenceCounts { self.placements += reference_placements(object); } - pub fn write(&self, out: &mut String, sign: &str, target: &str, first: &ObjectChange) { + pub fn write( + &self, + out: &mut CellText<'_>, + sign: char, + target: &str, + first: &ObjectChange, + ) -> usize { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); if self.len() == 1 { - writeln!(out, " {sign} {first}").unwrap(); - return; + write!(out, "{first}").unwrap(); + } else { + self.write_counts(out, ""); + write!(out, " to {}", cell_label(target)).unwrap(); + self.write_placements(out); } - write!(out, " {sign} ").unwrap(); - self.write_counts(out, ""); - write!(out, " to {}", cell_label(target)).unwrap(); - self.write_placements(out); + let placements = out.reserve_placements(); out.push('\n'); + placements } fn len(&self) -> usize { self.singles + self.arrays } - pub fn write_change(&self, out: &mut String, target: &str, added: &Self) { + pub fn write_change(&self, out: &mut CellText<'_>, target: &str, added: &Self) -> usize { write!(out, " {} · ", cell_label(target)).unwrap(); + out.style(Some(TextStyle::Removed)); self.write_counts(out, "-"); self.write_placements(out); + out.style(None); out.push_str(", "); + out.style(Some(TextStyle::Added)); added.write_counts(out, "+"); added.write_placements(out); + out.style(None); + let placements = out.reserve_placements(); out.push('\n'); + placements } - fn write_counts(&self, out: &mut String, sign: &str) { + fn write_counts(&self, out: &mut CellText<'_>, sign: &str) { if self.singles > 0 { let suffix = if self.singles == 1 { "" } else { "s" }; write!(out, "{sign}{} ref{suffix}", self.singles).unwrap(); @@ -382,7 +414,7 @@ impl ReferenceCounts { } } - fn write_placements(&self, out: &mut String) { + fn write_placements(&self, out: &mut CellText<'_>) { if self.arrays > 0 { let suffix = if self.placements == 1 { "" } else { "s" }; write!(out, " ({} placement{suffix})", self.placements).unwrap(); @@ -403,27 +435,27 @@ pub(super) struct ShownReferences { total: usize, removed: u128, added: u128, - row_start: usize, + placement_slot: usize, } impl ShownReferences { - pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange, row_start: usize) { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange, placement_slot: usize) { self.shown += 2; self.removed += reference_placements(old); self.added += reference_placements(new); - self.row_start = row_start; + self.placement_slot = placement_slot; } pub fn unmatched( &mut self, removed: &ReferenceCounts, added: &ReferenceCounts, - row_start: usize, + placement_slot: usize, ) { self.shown += removed.len() + added.len(); self.removed += removed.placements; self.added += added.placements; - self.row_start = row_start; + self.placement_slot = placement_slot; } pub fn individual( @@ -431,7 +463,7 @@ impl ShownReferences { object: &ObjectChange, removed: bool, count: usize, - row_start: usize, + placement_slot: usize, ) { let placements = if removed { &mut self.removed @@ -440,7 +472,7 @@ impl ShownReferences { }; *placements += reference_placements(object) * count as u128; self.shown += count; - self.row_start = row_start; + self.placement_slot = placement_slot; } pub fn count_change(&mut self) { @@ -451,106 +483,36 @@ impl ShownReferences { self.shown == self.total } - pub fn annotate(&self, text: &str, annotations: &mut Vec<(usize, String)>, old: u64, new: u64) { - if (u128::from(old), u128::from(new)) == (self.removed, self.added) { - return; - } - let at = self.row_start + text[self.row_start..].find('\n').unwrap(); - let totals = if old == new { - let suffix = if new == 1 { "" } else { "s" }; - format!(" · {new} total placement{suffix}") - } else { - format!(" · placements {old} -> {new}") - }; - annotations.push((at, totals)); - } -} - -pub(super) struct LayerRows<'a> { - names: &'a BTreeMap, - name_changes: &'a [LayerNameChange], - rows: BTreeMap>, -} - -impl<'a> LayerRows<'a> { - pub fn new(names: &'a BTreeMap, name_changes: &'a [LayerNameChange]) -> Self { - Self { - names, - name_changes, - rows: BTreeMap::new(), - } - } - - fn write_transition_layer(&mut self, out: &mut String, layer: Layer, side: Side) { - self.record(Some(layer), out.len()); - write!(out, "{}/{}", layer.0, layer.1).unwrap(); - if let Ok(index) = self - .name_changes - .binary_search_by_key(&layer, |change| change.layer) - { - let change = &self.name_changes[index]; - let name = match side { - Side::Old => &change.old, - Side::New => &change.new, - }; - if let Some(name) = name { - write!(out, " ({})", cell_label(name)).unwrap(); - } - } - } - - pub fn record(&mut self, layer: Option, row_start: usize) { - if let Some(layer) = layer - && self.names.contains_key(&layer) - { - self.rows - .entry(layer) - .and_modify(|first| *first = None) - .or_insert(Some(row_start)); - } - } - - pub fn finish(self, text: &str, annotations: &mut Vec<(usize, String)>, legend: &mut String) { - for (layer, first) in self.rows { - let name = &self.names[&layer]; - if let Some(at) = first { - let at = at - + text[at..] - .bytes() - .take_while(|b| b.is_ascii_digit() || *b == b'/') - .count(); - annotations.push((at, format!(" ({})", cell_label(name)))); - } else { - writeln!(legend, "{}/{}: name {name:?}", layer.0, layer.1).unwrap(); - } + pub fn write_placements(&self, out: &mut CellText<'_>, old: u64, new: u64) { + if (u128::from(old), u128::from(new)) != (self.removed, self.added) { + out.set_placements(self.placement_slot, old, new); } } } pub(super) fn write_layer_transition( - out: &mut String, + out: &mut CellText<'_>, old: &ObjectChange, new: &ObjectChange, - layers: Option<&mut LayerRows<'_>>, ) { - let (from, to) = (old.layer.unwrap(), new.layer.unwrap()); - if let Some(layers) = layers { - layers.write_transition_layer(out, from, Side::Old); - out.push_str(" → "); - layers.write_transition_layer(out, to, Side::New); - } else { - write!(out, "{}/{} → {}/{}", from.0, from.1, to.0, to.1).unwrap(); - } - let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some(); - write!(out, " · {}", object_name(old.kind, rectangular)).unwrap(); + out.write_layer(old.layer.unwrap(), Some(Side::Old)); + out.push_str(" → "); + out.write_layer(new.layer.unwrap(), Some(Side::New)); + write!(out, " · {}", transition_shape(old)).unwrap(); } -fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { +fn transition_shape(old: &ObjectChange) -> &'static str { + let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some(); + object_name(old.kind, rectangular) +} + +fn polygon_replacement(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { let before = object_xy(old); let after = object_xy(new); let old_size = rectangle_size(before); let new_size = rectangle_size(after); - let mut out = polygon_heading(old, old_size.is_some() && new_size.is_some()); + let name = object_name(old.kind, old_size.is_some() && new_size.is_some()); + let mut out = object_heading(old, name, with_layer); if before == after { out.push_str(" modified"); } else { @@ -572,22 +534,28 @@ fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String { out } -pub(super) fn replacement_heading(old: &ObjectChange, new: &ObjectChange) -> String { +fn replacement_heading(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some() && rectangle_size(object_xy(new)).is_some(); - let mut heading = object_heading(old, object_name(old.kind, rectangular)); + let mut heading = object_heading(old, object_name(old.kind, rectangular), with_layer); if let Some(target) = &old.target { write!(heading, " {target:?}").unwrap(); } heading } -pub(super) fn write_replacement_objects(out: &mut String, old: &ObjectChange, new: &ObjectChange) { +pub(super) fn write_replacement_objects( + out: &mut CellText<'_>, + old: &ObjectChange, + new: &ObjectChange, +) { for (sign, object) in [('-', old), ('+', new)] { let description = object.to_string(); let mut lines = description.split('\n'); - writeln!(out, " {sign} {}", lines.next().unwrap()).unwrap(); + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + writeln!(out, "{sign} {}", lines.next().unwrap()).unwrap(); for line in lines { writeln!(out, " {line}").unwrap(); } @@ -595,16 +563,35 @@ pub(super) fn write_replacement_objects(out: &mut String, old: &ObjectChange, ne } pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { - if old.layer != new.layer && old.layer.is_some() && new.layer.is_some() { - let mut out = String::new(); - write_layer_transition(&mut out, old, new, None); - return out; + if let (Some((from, from_type)), Some((to, to_type))) = (old.layer, new.layer) + && old.layer != new.layer + { + return format!( + "{from}/{from_type} → {to}/{to_type} · {}", + transition_shape(old) + ); } + replacement_fields(old, new, true) +} + +pub(super) fn replacement_description( + old: &ObjectChange, + new: &ObjectChange, + details: bool, +) -> String { + if details { + replacement_heading(old, new, false) + } else { + replacement_fields(old, new, false) + } +} + +fn replacement_fields(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { let polygon = is_polygon(old.kind); let mut out = if polygon { - polygon_replacement(old, new) + polygon_replacement(old, new, with_layer) } else { - replacement_heading(old, new) + replacement_heading(old, new, with_layer) }; let mut separator = if polygon { "\n " } else { " " }; diff --git a/src/review/selection.rs b/src/review/selection.rs index 71b2594..4e3491f 100644 --- a/src/review/selection.rs +++ b/src/review/selection.rs @@ -115,10 +115,11 @@ impl<'a> TextSelection<'a> { } impl Scope { - pub(super) fn write_restrictions(&self, out: &mut String) { + pub(super) fn write_restrictions(&self, out: &mut TextReport) { if self == &Self::default() { return; } + let start = out.len(); out.push_str("Scope"); if let Some(top) = &self.top { write!(out, " top={top:?}").unwrap(); @@ -129,6 +130,7 @@ impl Scope { if let Some((layer, datatype)) = self.layer { write!(out, " layer={layer}/{datatype}").unwrap(); } + out.mark(start, TextStyle::Heading); out.push('\n'); } diff --git a/src/review/text.rs b/src/review/text.rs new file mode 100644 index 0000000..f2692da --- /dev/null +++ b/src/review/text.rs @@ -0,0 +1,252 @@ +//! Text output and presentation roles shared with the terminal. + +use super::*; +use std::ops::Range; + +/// A presentation role for a span of report text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextStyle { + Heading, + Layer, + Added, + Removed, + Changed, + Success, + Warning, + Error, +} + +impl TextStyle { + pub(super) fn change(marker: char) -> Self { + match marker { + '+' => Self::Added, + '-' => Self::Removed, + _ => Self::Changed, + } + } +} + +/// A plain text report with optional style spans. +#[derive(Default)] +pub struct TextReport { + text: String, + spans: Option, TextStyle)>>, +} + +impl TextReport { + pub fn new(styled: bool) -> Self { + Self { + text: String::new(), + spans: styled.then(Vec::new), + } + } + + pub fn as_str(&self) -> &str { + &self.text + } + + pub fn into_string(self) -> String { + self.text + } + + /// Byte ranges are in output order and never overlap. + pub fn styles(&self) -> &[(Range, TextStyle)] { + self.spans.as_deref().unwrap_or(&[]) + } + + pub(super) fn is_styled(&self) -> bool { + self.spans.is_some() + } + + pub fn write_styled(&mut self, style: TextStyle, text: fmt::Arguments<'_>) { + let start = self.len(); + self.write_fmt(text).unwrap(); + self.mark(start, style); + } + + pub(super) fn push_str(&mut self, text: &str) { + self.text.push_str(text); + } + + pub(super) fn push(&mut self, ch: char) { + self.text.push(ch); + } + + pub(super) fn len(&self) -> usize { + self.text.len() + } + + pub(super) fn mark(&mut self, start: usize, style: TextStyle) { + let end = self.text.len(); + if start < end + && let Some(spans) = &mut self.spans + { + if let Some((range, previous)) = spans.last_mut() + && range.end == start + && *previous == style + { + range.end = end; + } else { + spans.push((start..end, style)); + } + } + } +} + +impl fmt::Write for TextReport { + fn write_str(&mut self, text: &str) -> fmt::Result { + self.push_str(text); + Ok(()) + } +} + +enum Part { + Text(String), + Layer(Layer), + Placements(Option<(u64, u64)>), +} + +/// Cell details awaiting inline layer labels and reference placement totals. +pub(super) struct CellText<'a> { + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + layers: BTreeMap, + parts: Vec<(Option, Part)>, + styled: bool, + style: Option, +} + +impl<'a> CellText<'a> { + pub fn new( + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + styled: bool, + ) -> Self { + Self { + names, + name_changes, + layers: BTreeMap::new(), + parts: Vec::new(), + styled, + style: None, + } + } + + pub fn style(&mut self, style: Option) { + self.style = style.filter(|_| self.styled); + } + + pub fn write_layer(&mut self, layer: Layer, side: Option) { + if self.names.contains_key(&layer) { + self.layers + .entry(layer) + .and_modify(|repeated| *repeated = true) + .or_insert(false); + self.parts.push((self.style, Part::Layer(layer))); + } else { + write!(self, "{}/{}", layer.0, layer.1).unwrap(); + if let Some(name) = self.transition_name(layer, side) { + write!(self, " ({})", cell_label(name)).unwrap(); + } + } + } + + pub fn reserve_placements(&mut self) -> usize { + let index = self.parts.len(); + self.parts.push((self.style, Part::Placements(None))); + index + } + + pub fn set_placements(&mut self, index: usize, old: u64, new: u64) { + self.parts[index].1 = Part::Placements(Some((old, new))); + } + + pub fn push_str(&mut self, text: &str) { + self.write_str(text).unwrap(); + } + + pub fn push(&mut self, ch: char) { + self.write_char(ch).unwrap(); + } + + fn append(&mut self, text: &str) { + if text.is_empty() { + return; + } + if let Some((style, Part::Text(current))) = self.parts.last_mut() + && *style == self.style + { + current.push_str(text); + } else { + self.parts.push((self.style, Part::Text(text.to_string()))); + } + } + + fn transition_name(&self, layer: Layer, side: Option) -> Option<&'a str> { + let side = side?; + let at = self + .name_changes + .binary_search_by_key(&layer, |change| change.layer) + .ok()?; + let change = &self.name_changes[at]; + match side { + Side::Old => change.old.as_deref(), + Side::New => change.new.as_deref(), + } + } + + pub fn write_legend(&self, out: &mut TextReport) { + for (&(layer, datatype), &repeated) in &self.layers { + if repeated { + writeln!( + out, + "{layer}/{datatype}: name {:?}", + self.names[&(layer, datatype)] + ) + .unwrap(); + } + } + } + + pub fn write_body(self, out: &mut TextReport) { + for (style, part) in self.parts { + let start = out.len(); + match part { + Part::Text(text) => out.push_str(&text), + Part::Layer((layer, datatype)) => { + write!(out, "{layer}/{datatype}").unwrap(); + if !self.layers[&(layer, datatype)] { + write!(out, " ({})", cell_label(&self.names[&(layer, datatype)])).unwrap(); + } + } + Part::Placements(Some((old, new))) if old == new => { + let suffix = if new == 1 { "" } else { "s" }; + write!(out, " · {new} total placement{suffix}").unwrap(); + } + Part::Placements(Some((old, new))) => { + write!(out, " · placements {old} -> {new}").unwrap(); + } + Part::Placements(None) => (), + } + if let Some(style) = style { + out.mark(start, style); + } + } + } +} + +impl fmt::Write for CellText<'_> { + fn write_str(&mut self, text: &str) -> fmt::Result { + if self.style.is_some() + && let Some((first, rest)) = text.split_once('\n') + { + self.append(first); + self.style = None; + self.append("\n"); + self.append(rest); + } else { + self.append(text); + } + Ok(()) + } +} From 4aa36a6d652df258fc9ff315005944b7d41e0060 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 09:29:21 -0800 Subject: [PATCH 38/41] Separate hierarchy traversal and text report modules Build structured hierarchy rows and folded paths before rendering ASCII or Mermaid. Move compact and detailed diff writers alongside their text presentation helpers. --- src/review.rs | 807 +--------------------------------------- src/review/changes.rs | 2 +- src/review/hierarchy.rs | 495 ++++++++++++++++++++++++ src/review/report.rs | 3 +- src/review/text.rs | 451 +++++++++++++++++++++- 5 files changed, 951 insertions(+), 807 deletions(-) create mode 100644 src/review/hierarchy.rs diff --git a/src/review.rs b/src/review.rs index a9f23ae..baf99c4 100644 --- a/src/review.rs +++ b/src/review.rs @@ -13,6 +13,7 @@ use std::{ }; pub mod changes; +mod hierarchy; pub mod names; mod pair; pub mod report; @@ -23,8 +24,7 @@ use changes::Replacement; use pair::CellReuse; pub use pair::RevisionPair; use selection::{DisplayFilter, Scope}; -use text::CellText; -pub use text::{TextReport, TextStyle}; +pub use text::{TextReport, TextReportOptions, TextStyle}; /// Input revision associated with a diagnostic. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -909,7 +909,7 @@ impl<'a> Document<'a> { pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { let revision = Revision::from_document(self); - render_hierarchy(&revision, &revision, None, mermaid, max_depth) + hierarchy::render_hierarchy(&revision, &revision, None, mermaid, max_depth) } pub fn canonical_dump(&self) -> String { @@ -1279,15 +1279,6 @@ pub struct Comparison { pub diagnostics: Vec, } -/// Additional presentation options for a text comparison. -#[derive(Clone, Copy, Debug, Default)] -pub struct TextReportOptions { - /// Print individual object evidence, including both sides of replacements. - pub details: bool, - /// Append the ASCII hierarchy at this depth when provided. - pub ascii_depth: Option, -} - /// A comparison for text and hierarchy reports. #[derive(Debug)] pub struct TextComparison { @@ -1305,55 +1296,6 @@ impl TextComparison { } } - pub fn render_text( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - ) -> String { - self.comparison - .render_text(max_objects, show, show_library_presence) - } - - pub fn render_text_with_options( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - options: TextReportOptions, - ) -> String { - self.comparison - .render_text_with_options(max_objects, show, show_library_presence, options) - } - - pub fn write_text_report( - &self, - out: &mut TextReport, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - options: TextReportOptions, - ) { - self.comparison - .write_text_report(out, max_objects, show, show_library_presence, options); - } - - /// Render the cell changes and their ASCII hierarchy together. - pub fn render_text_with_ascii_hierarchy( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - max_depth: usize, - ) -> String { - self.comparison.render_text_with_ascii_hierarchy( - max_objects, - show, - show_library_presence, - max_depth, - ) - } - pub fn outcome(&self) -> Outcome { self.comparison.outcome() } @@ -1539,748 +1481,7 @@ impl Comparison { /// Render the affected hierarchy. ASCII output shows relationships between cells. pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { - render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) - } - - /// Limit presentation only: the complete comparison has already been made. - pub fn text(&self, max_objects: usize) -> String { - self.text_filtered(max_objects, &DisplayFilter::default()) - } - - pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { - self.render_text(max_objects, show, true) - } - - /// Render a text report. Set `show_library_presence` to false when a file - /// header already announces the library addition or deletion. - pub fn render_text( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - ) -> String { - self.render_text_with_options( - max_objects, - show, - show_library_presence, - TextReportOptions::default(), - ) - } - - /// Render the cell changes and omit simple hierarchy edges already shown there. - pub fn render_text_with_ascii_hierarchy( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - max_depth: usize, - ) -> String { - self.render_text_with_options( - max_objects, - show, - show_library_presence, - TextReportOptions { - ascii_depth: Some(max_depth), - ..TextReportOptions::default() - }, - ) - } - - /// Render the selected changes with optional object details and ASCII hierarchy. - pub fn render_text_with_options( - &self, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - options: TextReportOptions, - ) -> String { - let mut out = TextReport::default(); - self.write_text_report(&mut out, max_objects, show, show_library_presence, options); - out.into_string() - } - - /// Append the selected changes and their presentation roles to a text report. - pub fn write_text_report( - &self, - out: &mut TextReport, - max_objects: usize, - show: &DisplayFilter, - show_library_presence: bool, - options: TextReportOptions, - ) { - let selection = selection::TextSelection::new(&self.cells, show, max_objects); - let outcome = self.outcome(); - let headline = out.len(); - let (result, style) = match outcome { - Outcome::Equal => ("Equal", TextStyle::Success), - Outcome::Different => ("Different", TextStyle::Warning), - Outcome::Incomplete { .. } => ("Incomplete", TextStyle::Warning), - Outcome::Failed => ("Failed", TextStyle::Error), - }; - out.push_str(result); - let has_omissions = selection.filtered_objects != 0 - || selection.limited_objects != 0 - || selection.filtered_cells != 0; - if !self.cells.is_empty() - && (self.cells.len() > 3 || !outcome.is_complete() || has_omissions) - { - let (mut modified, mut added, mut removed) = (0, 0, 0); - for cell in &self.cells { - match cell_marker(cell.old.is_some(), cell.new.is_some()) { - '-' => removed += 1, - '+' => added += 1, - _ => modified += 1, - } - } - let mut counts = [ - (modified, "modified"), - (added, "added"), - (removed, "removed"), - ] - .into_iter() - .filter(|(count, _)| *count != 0); - if let Some((count, action)) = counts.next() { - let noun = if count == 1 { "cell" } else { "cells" }; - write!(out, " · {count} {noun} {action}").unwrap(); - for (count, action) in counts { - write!(out, ", {count} {action}").unwrap(); - } - } - } - let library_changes = self.library_changes.iter().filter(|change| { - show_library_presence || !matches!(change, LibraryChange::Presence { .. }) - }); - for (count, label) in [ - (library_changes.clone().count(), "library change"), - (self.layer_names.len(), "layer name change"), - ] { - if count != 0 { - let plural = if count == 1 { "" } else { "s" }; - write!(out, " · {count} {label}{plural}").unwrap(); - } - } - out.mark(headline, style); - out.push('\n'); - self.old.scope.write_restrictions(out); - if show.cell.is_some() || show.layer.is_some() { - let start = out.len(); - out.push_str("Display filter"); - if let Some(cell) = &show.cell { - write!(out, " cell={cell:?}").unwrap(); - } - if let Some((layer, datatype)) = show.layer { - write!(out, " layer={layer}/{datatype}").unwrap(); - } - out.mark(start, TextStyle::Heading); - out.push('\n'); - } - if has_omissions { - let start = out.len(); - out.push_str("Omitted"); - for (count, label, reason) in [ - (selection.filtered_objects, "object change", "filtered out"), - (selection.limited_objects, "object change", "over the limit"), - ( - selection.filtered_cells, - "cell-level change", - "filtered out", - ), - ] { - if count != 0 { - let plural = if count == 1 { "" } else { "s" }; - write!(out, " · {count} {label}{plural} {reason}").unwrap(); - } - } - out.mark(start, TextStyle::Warning); - out.push('\n'); - } - for change in library_changes { - writeln!(out, "library {change}").unwrap(); - } - if self.library_changes.iter().any(|change| { - matches!( - change, - LibraryChange::Record { - tag: RTAG_UNITS, - .. - } - ) - }) { - out.push_str("Coordinates use each file's database units\n"); - } - for change in &self.layer_names { - let name = |value: Option<&str>| { - value.map_or_else(|| "absent".to_string(), |value| format!("{value:?}")) - }; - writeln!( - out, - "{}/{}: name {} -> {}", - change.layer.0, - change.layer.1, - name(change.old.as_deref()), - name(change.new.as_deref()) - ) - .unwrap(); - } - - let mut body = CellText::new(&self.layer_legend, &self.layer_names, out.is_styled()); - let mut covered_edges = BTreeSet::new(); - for selected in &selection.modified_cells { - self.write_cell_details( - &mut body, - selected, - options.details, - (options.ascii_depth.is_some() && outcome.is_complete()) - .then_some(&mut covered_edges), - ); - } - body.write_legend(out); - if !selection.modified_cells.is_empty() - || !selection.removed_cells.is_empty() - || !selection.added_cells.is_empty() - { - out.write_styled(TextStyle::Heading, format_args!("Cells:")); - out.push('\n'); - for name in selection.removed_cells { - out.write_styled(TextStyle::Removed, format_args!("- {}", cell_label(name))); - out.push('\n'); - } - for name in selection.added_cells { - out.write_styled(TextStyle::Added, format_args!("+ {}", cell_label(name))); - out.push('\n'); - } - body.write_body(out); - } - // General comparison errors precede revision-specific errors in text. - if !self.diagnostics.is_empty() { - out.write_styled(TextStyle::Heading, format_args!("Diagnostics")); - out.push('\n'); - for diagnostic in self - .diagnostics - .iter() - .filter(|d| d.side.is_none()) - .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) - { - writeln!(out, " {diagnostic}").unwrap(); - } - } - if let Some(max_depth) = options.ascii_depth { - write_hierarchy( - out, - &self.old, - &self.new, - Some(&self.cells), - false, - max_depth, - Some(&covered_edges), - ); - } - } - - fn write_cell_details( - &self, - out: &mut CellText<'_>, - selected: &selection::TextCellSelection<'_>, - details: bool, - mut covered_edges: Option<&mut BTreeSet<(String, String)>>, - ) { - let cell = selected.cell; - let name = cell_label(&cell.name); - let note = if selected.attributes_changed { - " (attributes changed)" - } else { - "" - }; - out.style(Some(TextStyle::Changed)); - writeln!(out, "* {name}{note}").unwrap(); - let mut polygons = changes::PolygonChanges::default(); - let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); - for &(a, b) in &selected.replacements { - if !details && a.layer == b.layer && is_polygon(a.kind) { - polygons.replacement(a, b); - continue; - } - out.push_str(" "); - out.style(Some(TextStyle::Changed)); - out.push_str("~ "); - if a.layer != b.layer { - changes::write_layer_transition(out, a, b); - out.push('\n'); - } else { - if let Some(layer) = a.layer { - out.write_layer(layer, None); - out.push_str(": "); - } - let description = changes::replacement_description(a, b, details); - let (heading, rest) = match description.split_once('\n') { - Some((heading, rest)) => (heading, Some(rest)), - None => (description.as_str(), None), - }; - out.push_str(heading); - if let Some(target) = &a.target { - let placements = out.reserve_placements(); - shown_refs - .entry(target) - .or_default() - .replacement(a, b, placements); - } - out.push('\n'); - if let Some(rest) = rest { - writeln!(out, "{rest}").unwrap(); - } - } - if details { - changes::write_replacement_objects(out, a, b); - } - } - let mut removed_refs = BTreeMap::new(); - let mut added_refs = BTreeMap::new(); - for (removed, objects, references) in [ - (true, &selected.removed, &mut removed_refs), - (false, &selected.added, &mut added_refs), - ] { - let sign = if removed { '-' } else { '+' }; - let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); - for &object in objects { - if !details && is_polygon(object.kind) { - polygons.unmatched(object, removed); - } else if !details && let Some(target) = object.target.as_deref() { - let (_, counts) = references - .entry(target) - .or_insert_with(|| (object, changes::ReferenceCounts::default())); - counts.add(object); - } else if let Some((last, count)) = unmatched.last_mut() - && last.kind == object.kind - && last.fields == object.fields - && last.properties == object.properties - { - *count += 1; - } else { - unmatched.push((object, 1)); - } - } - // The comparison orders these objects by kind, fields, and properties. - for (object, count) in unmatched { - out.push_str(" "); - out.style(Some(TextStyle::change(sign))); - write!(out, "{sign} ").unwrap(); - if count > 1 { - write!(out, "{count}× ").unwrap(); - } - changes::write_unmatched(out, object, details); - if let Some(target) = object.target.as_deref() { - let placements = out.reserve_placements(); - shown_refs - .entry(target) - .or_default() - .individual(object, removed, count, placements); - } - out.push('\n'); - } - } - let no_refs = changes::ReferenceCounts::default(); - for (target, (first, counts)) in removed_refs { - if let Some((_, added)) = added_refs.remove(target) { - let placements = counts.write_change(out, target, &added); - shown_refs - .entry(target) - .or_default() - .unmatched(&counts, &added, placements); - } else { - let placements = counts.write(out, '-', target, first); - shown_refs - .entry(target) - .or_default() - .unmatched(&counts, &no_refs, placements); - } - } - for (target, (first, counts)) in added_refs { - let placements = counts.write(out, '+', target, first); - shown_refs - .entry(target) - .or_default() - .unmatched(&no_refs, &counts, placements); - } - if !shown_refs.is_empty() { - for target in cell - .removed - .iter() - .chain(&cell.added) - .filter_map(|o| o.target.as_deref()) - { - if let Some(shown) = shown_refs.get_mut(target) { - shown.count_change(); - } - } - for (target, shown) in shown_refs { - if shown.complete() { - let edge = (cell.name.clone(), target.to_owned()); - let old = self.old.edges.get(&edge).copied().unwrap_or(0); - let new = self.new.edges.get(&edge).copied().unwrap_or(0); - shown.write_placements(out, old, new); - if let Some(covered) = covered_edges.as_deref_mut() { - covered.insert(edge); - } - } - } - } - polygons.write(out); - } -} - -/// Graph facts shared by text, Mermaid, and JSON renderers. -struct HierarchyAnalysis<'a> { - old_edges: &'a BTreeMap<(String, String), u64>, - new_edges: &'a BTreeMap<(String, String), u64>, - edges: BTreeSet<(String, String)>, - direct: BTreeSet, - affected: BTreeSet, - changed_refs: BTreeSet<(String, String)>, -} - -impl<'a> HierarchyAnalysis<'a> { - fn between(old: &'a Revision, new: &'a Revision, changes: Option<&[CellChange]>) -> Self { - let old_edges = &old.edges; - let new_edges = &new.edges; - let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); - let direct: BTreeSet<_> = changes - .into_iter() - .flatten() - .map(|cell| cell.name.clone()) - .collect(); - let changed_refs: BTreeSet<_> = changes - .into_iter() - .flatten() - .flat_map(|c| { - c.removed - .iter() - .chain(&c.added) - .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) - }) - .collect(); - let mut parents = BTreeMap::<&str, Vec<&str>>::new(); - for (parent, child) in &edges { - parents.entry(child).or_default().push(parent); - } - let mut affected = direct.clone(); - let mut pending: Vec<_> = direct.iter().cloned().collect(); - while let Some(child) = pending.pop() { - for parent in parents.get(child.as_str()).into_iter().flatten() { - if affected.insert((*parent).to_string()) { - pending.push((*parent).to_string()); - } - } - } - Self { - old_edges, - new_edges, - edges, - direct, - affected, - changed_refs, - } - } -} - -/// The graph joins both revisions, retaining removed branches and old callers. -fn render_hierarchy( - old: &Revision, - new: &Revision, - changes: Option<&[CellChange]>, - mermaid: bool, - max_depth: usize, -) -> String { - let mut out = TextReport::default(); - write_hierarchy(&mut out, old, new, changes, mermaid, max_depth, None); - out.into_string() -} - -fn write_hierarchy( - out: &mut TextReport, - old: &Revision, - new: &Revision, - changes: Option<&[CellChange]>, - mermaid: bool, - max_depth: usize, - covered_edges: Option<&BTreeSet<(String, String)>>, -) { - if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { - return; - } - - let HierarchyAnalysis { - old_edges, - new_edges, - edges, - direct, - affected, - changed_refs, - } = HierarchyAnalysis::between(old, new, changes); - let mut visible: BTreeSet<_> = if changes.is_some() { - affected - .iter() - .cloned() - .chain(changed_refs.iter().map(|(_, t)| t.clone())) - .collect() - } else { - old.cells - .iter() - .chain(new.cells.iter()) - .cloned() - .chain(edges.iter().map(|(_, t)| t.clone())) - .collect() - }; - let mut children = BTreeMap::>::new(); - let mut targets = BTreeSet::new(); - let fold_paths = !mermaid && changes.is_some(); - let mut shared = BTreeSet::new(); - for (parent, target) in &edges { - if visible.contains(parent) && visible.contains(target) { - children - .entry(parent.clone()) - .or_default() - .push(target.clone()); - if !targets.insert(target.clone()) && fold_paths { - shared.insert(target.clone()); - } - } - } - if let Some(covered) = covered_edges { - if !covered.is_empty() { - // Keep changed or shared descendants, branches, and target status notes. - let redundant: BTreeSet<_> = covered - .iter() - .map(|(_, target)| target) - .filter(|&target| { - targets.contains(target) - && !shared.contains(target) - && !children.contains_key(target) - && !affected.contains(target) - && old.cells.contains(target) - && new.cells.contains(target) - }) - .collect(); - children.retain(|_, children| { - children.retain(|target| !redundant.contains(target)); - !children.is_empty() - }); - targets.retain(|target| !redundant.contains(target)); - } - visible.retain(|name| children.contains_key(name) || targets.contains(name)); - } - if changes.is_some() && !mermaid && children.is_empty() { - return; - } - let roots: Vec<_> = visible.difference(&targets).cloned().collect(); - // Distinguish rootless cycles from descendants hidden by the depth limit. - let mut reachable = BTreeSet::new(); - let mut pending = roots.clone(); - while let Some(name) = pending.pop() { - if reachable.insert(name.clone()) { - pending.extend(children.get(&name).into_iter().flatten().cloned()); - } - } - let mut included = BTreeSet::new(); - struct Row { - name: String, - depth: usize, - parent: Option, - note: &'static str, - } - let mut rows = Vec::new(); - let mut stack: Vec<_> = roots - .iter() - .rev() - .map(|n| (n.clone(), 0, None::)) - .collect(); - while let Some((name, depth, parent)) = stack.pop() { - let seen = !included.insert(name.clone()); - let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); - let note = if descendants.is_empty() { - "" - } else if depth >= max_depth { - " [children omitted]" - } else if seen { - " [subtree elided]" - } else { - "" - }; - rows.push(Row { - name: name.clone(), - depth, - parent, - note, - }); - if !seen && depth < max_depth { - for child in descendants.iter().rev() { - stack.push((child.clone(), depth + 1, Some(name.clone()))); - } - } - } - for name in visible.difference(&reachable) { - rows.push(Row { - name: name.clone(), - depth: 0, - parent: None, - note: " [children omitted]", - }); - } - let status = |name: &String| { - if !old.cells.contains(name) && !new.cells.contains(name) { - if old.known_cells.contains(name) || new.known_cells.contains(name) { - " [outside scope]" - } else { - " [unresolved]" - } - } else if direct.contains(name) { - if mermaid { " [cell changed]" } else { "" } - } else if affected.contains(name) { - " [affected]" - } else if mermaid && changes.is_some() { - " [cell unchanged]" - } else { - "" - } - }; - let edge_label = |parent: &String, target: &String| { - let key = (parent.clone(), target.clone()); - let a = old_edges.get(&key).copied().unwrap_or(0); - let b = new_edges.get(&key).copied().unwrap_or(0); - let count = if a == b { - format!("x{b}") - } else { - format!("x{a} -> x{b}") - }; - format!( - "{count}{}", - if (mermaid || a == b) && changed_refs.contains(&key) { - " [reference changed]" - } else { - "" - } - ) - }; - if mermaid { - let ids: BTreeMap<_, _> = rows - .iter() - .map(|row| &row.name) - .collect::>() - .into_iter() - .enumerate() - .map(|(i, n)| (n, i)) - .collect(); - out.push_str("flowchart TD\n"); - if changes.is_none() && new.scope != Scope::default() { - writeln!(out, " %% {}", new.scope.description()).unwrap(); - } - for (name, id) in &ids { - let label = format!("{name}{}", status(name)); - let label = label - .replace('&', "&") - .replace('"', """) - .replace('<', "<") - .replace('>', ">") - .replace(['\r', '\n'], " "); - writeln!(out, " n{id}[\"{label}\"]").unwrap(); - } - for (parent, target) in &edges { - if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { - writeln!(out, " n{a} -->|\"{}\"| n{b}", edge_label(parent, target)).unwrap(); - } - } - if rows.is_empty() { - out.push_str(" empty[\"No changed hierarchy\"]\n"); - } - } else { - let single_placement = |parent: &String, target: &String| { - let key = (parent.clone(), target.clone()); - old_edges.get(&key) == Some(&1) - && new_edges.get(&key) == Some(&1) - && !changed_refs.contains(&key) - }; - let unbranched = |name: &String| { - !shared.contains(name) - && children - .get(name) - .is_none_or(|descendants| descendants.len() == 1) - }; - let can_join = |parent: &Row, child: &Row| { - child.parent.as_ref() == Some(&parent.name) - && parent.note.is_empty() - && !direct.contains(&parent.name) - && affected.contains(&parent.name) - && unbranched(&child.name) - && (old.cells.contains(&child.name) || new.cells.contains(&child.name)) - && single_placement(&parent.name, &child.name) - }; - out.write_styled(TextStyle::Heading, format_args!("Hierarchy")); - out.push('\n'); - if changes.is_none() { - new.scope.write_restrictions(out); - } - let mut index = 0; - let mut ancestor_depths = Vec::new(); - while let Some(first) = rows.get(index) { - let mut end = index; - if fold_paths - && unbranched(&first.name) - && first - .parent - .as_ref() - .is_none_or(|parent| single_placement(parent, &first.name)) - { - while rows - .get(end + 1) - .is_some_and(|next| can_join(&rows[end], next)) - { - end += 1; - } - // Two rows are already easy to scan. - if end - index < 2 { - end = index; - } - } - while ancestor_depths - .last() - .is_some_and(|&depth| depth >= first.depth) - { - ancestor_depths.pop(); - } - out.push_str(&" ".repeat(ancestor_depths.len())); - let mut change = None; - for (i, row) in rows[index..=end].iter().enumerate() { - let name = &row.name; - if i != 0 { - out.push_str(" → "); - } - if direct.contains(name) { - let marker = cell_marker(old.cells.contains(name), new.cells.contains(name)); - change = Some((out.len(), TextStyle::change(marker))); - write!(out, "{marker} ").unwrap(); - } - if changes.is_some() { - out.push_str(&cell_label(name)); - } else { - write!(out, "{name:?}").unwrap(); - } - if i == 0 - && let Some(parent) = &row.parent - { - write!(out, " {}", edge_label(parent, name)).unwrap(); - } - } - let last = &rows[end]; - write!(out, "{}{}", status(&last.name), last.note).unwrap(); - if let Some((start, style)) = change { - out.mark(start, style); - } - out.push('\n'); - ancestor_depths.push(last.depth); - index = end + 1; - } - if visible.is_empty() { - out.push_str("No changed hierarchy\n"); - } + hierarchy::render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) } } diff --git a/src/review/changes.rs b/src/review/changes.rs index c5a69cf..157a349 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -1,6 +1,6 @@ //! Unique replacement matches that explain removed and added objects. -use super::*; +use super::{text::CellText, *}; #[derive(Debug)] pub struct Replacement { diff --git a/src/review/hierarchy.rs b/src/review/hierarchy.rs new file mode 100644 index 0000000..4d5342e --- /dev/null +++ b/src/review/hierarchy.rs @@ -0,0 +1,495 @@ +//! Hierarchy analysis, traversal, and presentation. + +use super::*; + +/// Graph facts shared by text, Mermaid, and JSON renderers. +pub(super) struct HierarchyAnalysis<'a> { + pub(super) old_edges: &'a BTreeMap<(String, String), u64>, + pub(super) new_edges: &'a BTreeMap<(String, String), u64>, + pub(super) edges: BTreeSet<(String, String)>, + pub(super) direct: BTreeSet, + pub(super) affected: BTreeSet, + pub(super) changed_refs: BTreeSet<(String, String)>, + old: &'a Revision, + new: &'a Revision, + comparison: bool, +} + +impl<'a> HierarchyAnalysis<'a> { + pub(super) fn between( + old: &'a Revision, + new: &'a Revision, + changes: Option<&[CellChange]>, + ) -> Self { + let old_edges = &old.edges; + let new_edges = &new.edges; + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = changes + .into_iter() + .flatten() + .map(|cell| cell.name.clone()) + .collect(); + let changed_refs: BTreeSet<_> = changes + .into_iter() + .flatten() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + Self { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + old, + new, + comparison: changes.is_some(), + } + } + + fn visible_rows( + &self, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, + ) -> Option { + let mut visible: BTreeSet<_> = if self.comparison { + self.affected + .iter() + .cloned() + .chain(self.changed_refs.iter().map(|(_, t)| t.clone())) + .collect() + } else { + self.old + .cells + .iter() + .chain(self.new.cells.iter()) + .cloned() + .chain(self.edges.iter().map(|(_, t)| t.clone())) + .collect() + }; + let mut children = BTreeMap::>::new(); + let mut targets = BTreeSet::new(); + let mut shared = BTreeSet::new(); + let fold_paths = self.comparison && !mermaid; + for (parent, target) in &self.edges { + if visible.contains(parent) && visible.contains(target) { + children + .entry(parent.clone()) + .or_default() + .push(target.clone()); + if !targets.insert(target.clone()) && fold_paths { + shared.insert(target.clone()); + } + } + } + if let Some(covered) = covered_edges { + if !covered.is_empty() { + // Keep changed or shared descendants, branches, and target status notes. + let redundant: BTreeSet<_> = covered + .iter() + .map(|(_, target)| target) + .filter(|&target| { + targets.contains(target) + && !shared.contains(target) + && !children.contains_key(target) + && !self.affected.contains(target) + && self.old.cells.contains(target) + && self.new.cells.contains(target) + }) + .collect(); + children.retain(|_, children| { + children.retain(|target| !redundant.contains(target)); + !children.is_empty() + }); + targets.retain(|target| !redundant.contains(target)); + } + visible.retain(|name| children.contains_key(name) || targets.contains(name)); + } + if fold_paths && children.is_empty() { + return None; + } + let rows = traverse(&visible, &children, &targets, max_depth); + Some(VisibleHierarchy { + rows, + children, + shared, + }) + } + + fn single_placement(&self, parent: &str, target: &str) -> bool { + let key = (parent.to_string(), target.to_string()); + self.old_edges.get(&key) == Some(&1) + && self.new_edges.get(&key) == Some(&1) + && !self.changed_refs.contains(&key) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RowNote { + ChildrenOmitted, + SubtreeElided, +} + +struct HierarchyRow { + name: String, + depth: usize, + parent: Option, + note: Option, +} + +fn traverse( + visible: &BTreeSet, + children: &BTreeMap>, + targets: &BTreeSet, + max_depth: usize, +) -> Vec { + let roots: Vec<_> = visible.difference(targets).cloned().collect(); + // Distinguish rootless cycles from descendants hidden by the depth limit. + let mut reachable = BTreeSet::new(); + let mut pending = roots.clone(); + while let Some(name) = pending.pop() { + if reachable.insert(name.clone()) { + pending.extend(children.get(&name).into_iter().flatten().cloned()); + } + } + let mut included = BTreeSet::new(); + let mut rows = Vec::new(); + let mut stack: Vec<_> = roots + .into_iter() + .rev() + .map(|name| (name, 0, None::)) + .collect(); + while let Some((name, depth, parent)) = stack.pop() { + let seen = !included.insert(name.clone()); + let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); + let note = if descendants.is_empty() { + None + } else if depth >= max_depth { + Some(RowNote::ChildrenOmitted) + } else if seen { + Some(RowNote::SubtreeElided) + } else { + None + }; + if !seen && depth < max_depth { + for child in descendants.iter().rev() { + stack.push((child.clone(), depth + 1, Some(name.clone()))); + } + } + rows.push(HierarchyRow { + name, + depth, + parent, + note, + }); + } + for name in visible.difference(&reachable) { + rows.push(HierarchyRow { + name: name.clone(), + depth: 0, + parent: None, + note: Some(RowNote::ChildrenOmitted), + }); + } + rows +} + +struct VisibleHierarchy { + rows: Vec, + children: BTreeMap>, + shared: BTreeSet, +} + +struct AsciiPath<'a> { + indent: usize, + rows: &'a [HierarchyRow], +} + +impl VisibleHierarchy { + fn unbranched(&self, name: &str) -> bool { + !self.shared.contains(name) + && self + .children + .get(name) + .is_none_or(|descendants| descendants.len() == 1) + } + + fn can_join( + &self, + graph: &HierarchyAnalysis<'_>, + parent: &HierarchyRow, + child: &HierarchyRow, + ) -> bool { + child.parent.as_deref() == Some(&parent.name) + && parent.note.is_none() + && !graph.direct.contains(&parent.name) + && graph.affected.contains(&parent.name) + && self.unbranched(&child.name) + && (graph.old.cells.contains(&child.name) || graph.new.cells.contains(&child.name)) + && graph.single_placement(&parent.name, &child.name) + } + + fn ascii_paths<'a>( + &'a self, + graph: &'a HierarchyAnalysis<'_>, + ) -> impl Iterator> { + let mut index = 0; + let mut ancestor_depths = Vec::new(); + std::iter::from_fn(move || { + let first = self.rows.get(index)?; + let mut end = index; + if graph.comparison + && self.unbranched(&first.name) + && first + .parent + .as_ref() + .is_none_or(|parent| graph.single_placement(parent, &first.name)) + { + while self + .rows + .get(end + 1) + .is_some_and(|next| self.can_join(graph, &self.rows[end], next)) + { + end += 1; + } + // Fold paths with at least two edges. + if end - index < 2 { + end = index; + } + } + while ancestor_depths + .last() + .is_some_and(|&depth| depth >= first.depth) + { + ancestor_depths.pop(); + } + let rows = &self.rows[index..=end]; + let indent = ancestor_depths.len(); + ancestor_depths.push(self.rows[end].depth); + index = end + 1; + Some(AsciiPath { indent, rows }) + }) + } +} + +/// The graph joins both revisions, retaining removed branches and old callers. +pub(super) fn render_hierarchy( + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, +) -> String { + let mut out = TextReport::default(); + write_hierarchy(&mut out, old, new, changes, mermaid, max_depth, None); + out.into_string() +} + +pub(super) fn write_hierarchy( + out: &mut TextReport, + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, +) { + if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { + return; + } + let graph = HierarchyAnalysis::between(old, new, changes); + let Some(view) = graph.visible_rows(mermaid, max_depth, covered_edges) else { + return; + }; + if mermaid { + graph.write_mermaid(out, &view.rows); + } else { + graph.write_ascii(out, view.ascii_paths(&graph)); + } +} + +impl HierarchyAnalysis<'_> { + fn status(&self, name: &str, mermaid: bool) -> &'static str { + if !self.old.cells.contains(name) && !self.new.cells.contains(name) { + if self.old.known_cells.contains(name) || self.new.known_cells.contains(name) { + " [outside scope]" + } else { + " [unresolved]" + } + } else if self.direct.contains(name) { + if mermaid { " [cell changed]" } else { "" } + } else if self.affected.contains(name) { + " [affected]" + } else if mermaid && self.comparison { + " [cell unchanged]" + } else { + "" + } + } + + fn edge_label(&self, parent: &str, target: &str, mermaid: bool) -> String { + let key = (parent.to_string(), target.to_string()); + let a = self.old_edges.get(&key).copied().unwrap_or(0); + let b = self.new_edges.get(&key).copied().unwrap_or(0); + let count = if a == b { + format!("x{b}") + } else { + format!("x{a} -> x{b}") + }; + format!( + "{count}{}", + if (mermaid || a == b) && self.changed_refs.contains(&key) { + " [reference changed]" + } else { + "" + } + ) + } + + fn write_mermaid(&self, out: &mut TextReport, rows: &[HierarchyRow]) { + let ids: BTreeMap<_, _> = rows + .iter() + .map(|row| &row.name) + .collect::>() + .into_iter() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + out.push_str("flowchart TD\n"); + if !self.comparison && self.new.scope != Scope::default() { + writeln!(out, " %% {}", self.new.scope.description()).unwrap(); + } + for (name, id) in &ids { + let label = format!("{name}{}", self.status(name, true)); + let label = label + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") + .replace(['\r', '\n'], " "); + writeln!(out, " n{id}[\"{label}\"]").unwrap(); + } + for (parent, target) in &self.edges { + if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { + writeln!( + out, + " n{a} -->|\"{}\"| n{b}", + self.edge_label(parent, target, true) + ) + .unwrap(); + } + } + if rows.is_empty() { + out.push_str(" empty[\"No changed hierarchy\"]\n"); + } + } + + fn write_ascii<'a>(&self, out: &mut TextReport, paths: impl Iterator>) { + out.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + out.push('\n'); + if !self.comparison { + self.new.scope.write_restrictions(out); + } + let mut paths = paths.peekable(); + if paths.peek().is_none() { + out.push_str("No changed hierarchy\n"); + } + for path in paths { + out.push_str(&" ".repeat(path.indent)); + let mut change = None; + for (i, row) in path.rows.iter().enumerate() { + let name = &row.name; + if i != 0 { + out.push_str(" → "); + } + if self.direct.contains(name) { + let marker = + cell_marker(self.old.cells.contains(name), self.new.cells.contains(name)); + change = Some((out.len(), TextStyle::change(marker))); + write!(out, "{marker} ").unwrap(); + } + if self.comparison { + out.push_str(&cell_label(name)); + } else { + write!(out, "{name:?}").unwrap(); + } + if i == 0 + && let Some(parent) = &row.parent + { + write!(out, " {}", self.edge_label(parent, name, false)).unwrap(); + } + } + let last = path.rows.last().unwrap(); + let note = match last.note { + Some(RowNote::ChildrenOmitted) => " [children omitted]", + Some(RowNote::SubtreeElided) => " [subtree elided]", + None => "", + }; + write!(out, "{}{note}", self.status(&last.name, false)).unwrap(); + if let Some((start, style)) = change { + out.mark(start, style); + } + out.push('\n'); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn traversal_distinguishes_depth_limits_from_rootless_components() { + let children = BTreeMap::from([ + ("root".into(), vec!["branch".into()]), + ("branch".into(), vec!["leaf".into()]), + ("cycle_a".into(), vec!["cycle_b".into()]), + ("cycle_b".into(), vec!["cycle_a".into()]), + ]); + let visible = ["root", "branch", "leaf", "cycle_a", "cycle_b"] + .map(String::from) + .into_iter() + .collect(); + let targets = children.values().flatten().cloned().collect(); + let rows = traverse(&visible, &children, &targets, 1); + let rows: Vec<_> = rows + .iter() + .map(|row| { + ( + row.name.as_str(), + row.depth, + row.parent.as_deref(), + row.note, + ) + }) + .collect(); + assert_eq!( + rows, + [ + ("root", 0, None, None), + ("branch", 1, Some("root"), Some(RowNote::ChildrenOmitted)), + ("cycle_a", 0, None, Some(RowNote::ChildrenOmitted)), + ("cycle_b", 0, None, Some(RowNote::ChildrenOmitted)), + ] + ); + } +} diff --git a/src/review/report.rs b/src/review/report.rs index 6450755..8004f48 100644 --- a/src/review/report.rs +++ b/src/review/report.rs @@ -1,5 +1,5 @@ //! Versioned machine-readable diff reports. No consumer needs to parse prose. -use super::*; +use super::{hierarchy::HierarchyAnalysis, *}; use serde::{ Serialize, Serializer, ser::{SerializeMap, SerializeSeq}, @@ -445,6 +445,7 @@ fn graph(diff: &Comparison) -> Value { direct, affected, changed_refs, + .. } = HierarchyAnalysis::between(old, new, Some(&diff.cells)); let names: BTreeSet<_> = old .cells diff --git a/src/review/text.rs b/src/review/text.rs index f2692da..d47ed16 100644 --- a/src/review/text.rs +++ b/src/review/text.rs @@ -1,8 +1,17 @@ -//! Text output and presentation roles shared with the terminal. +//! Text comparison reports and presentation roles shared with the terminal. -use super::*; +use super::{hierarchy::write_hierarchy, *}; use std::ops::Range; +/// Additional presentation options for a text comparison. +#[derive(Clone, Copy, Debug, Default)] +pub struct TextReportOptions { + /// Print individual object evidence, including both sides of replacements. + pub details: bool, + /// Append the ASCII hierarchy at this depth when provided. + pub ascii_depth: Option, +} + /// A presentation role for a span of report text. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TextStyle { @@ -250,3 +259,441 @@ impl fmt::Write for CellText<'_> { Ok(()) } } + +impl TextComparison { + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.comparison + .render_text(max_objects, show, show_library_presence) + } + + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + self.comparison + .render_text_with_options(max_objects, show, show_library_presence, options) + } + + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + self.comparison + .write_text_report(out, max_objects, show, show_library_presence, options); + } + + /// Render the cell changes and their ASCII hierarchy together. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.comparison.render_text_with_ascii_hierarchy( + max_objects, + show, + show_library_presence, + max_depth, + ) + } +} + +impl Comparison { + /// Limit presentation only: the complete comparison has already been made. + pub fn text(&self, max_objects: usize) -> String { + self.text_filtered(max_objects, &DisplayFilter::default()) + } + + pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { + self.render_text(max_objects, show, true) + } + + /// Render a text report. Set `show_library_presence` to false when a file + /// header already announces the library addition or deletion. + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions::default(), + ) + } + + /// Render the cell changes and omit simple hierarchy edges already shown there. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions { + ascii_depth: Some(max_depth), + ..TextReportOptions::default() + }, + ) + } + + /// Render the selected changes with optional object details and ASCII hierarchy. + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + let mut out = TextReport::default(); + self.write_text_report(&mut out, max_objects, show, show_library_presence, options); + out.into_string() + } + + /// Append the selected changes and their presentation roles to a text report. + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + let selection = selection::TextSelection::new(&self.cells, show, max_objects); + let outcome = self.outcome(); + let headline = out.len(); + let (result, style) = match outcome { + Outcome::Equal => ("Equal", TextStyle::Success), + Outcome::Different => ("Different", TextStyle::Warning), + Outcome::Incomplete { .. } => ("Incomplete", TextStyle::Warning), + Outcome::Failed => ("Failed", TextStyle::Error), + }; + out.push_str(result); + let has_omissions = selection.filtered_objects != 0 + || selection.limited_objects != 0 + || selection.filtered_cells != 0; + if !self.cells.is_empty() + && (self.cells.len() > 3 || !outcome.is_complete() || has_omissions) + { + let (mut modified, mut added, mut removed) = (0, 0, 0); + for cell in &self.cells { + match cell_marker(cell.old.is_some(), cell.new.is_some()) { + '-' => removed += 1, + '+' => added += 1, + _ => modified += 1, + } + } + let mut counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + write!(out, " · {count} {noun} {action}").unwrap(); + for (count, action) in counts { + write!(out, ", {count} {action}").unwrap(); + } + } + } + let library_changes = self.library_changes.iter().filter(|change| { + show_library_presence || !matches!(change, LibraryChange::Presence { .. }) + }); + for (count, label) in [ + (library_changes.clone().count(), "library change"), + (self.layer_names.len(), "layer name change"), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural}").unwrap(); + } + } + out.mark(headline, style); + out.push('\n'); + self.old.scope.write_restrictions(out); + if show.cell.is_some() || show.layer.is_some() { + let start = out.len(); + out.push_str("Display filter"); + if let Some(cell) = &show.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = show.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + if has_omissions { + let start = out.len(); + out.push_str("Omitted"); + for (count, label, reason) in [ + (selection.filtered_objects, "object change", "filtered out"), + (selection.limited_objects, "object change", "over the limit"), + ( + selection.filtered_cells, + "cell-level change", + "filtered out", + ), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural} {reason}").unwrap(); + } + } + out.mark(start, TextStyle::Warning); + out.push('\n'); + } + for change in library_changes { + writeln!(out, "library {change}").unwrap(); + } + if self.library_changes.iter().any(|change| { + matches!( + change, + LibraryChange::Record { + tag: RTAG_UNITS, + .. + } + ) + }) { + out.push_str("Coordinates use each file's database units\n"); + } + for change in &self.layer_names { + let name = |value: Option<&str>| { + value.map_or_else(|| "absent".to_string(), |value| format!("{value:?}")) + }; + writeln!( + out, + "{}/{}: name {} -> {}", + change.layer.0, + change.layer.1, + name(change.old.as_deref()), + name(change.new.as_deref()) + ) + .unwrap(); + } + + let mut body = CellText::new(&self.layer_legend, &self.layer_names, out.is_styled()); + let mut covered_edges = BTreeSet::new(); + for selected in &selection.modified_cells { + self.write_cell_details( + &mut body, + selected, + options.details, + (options.ascii_depth.is_some() && outcome.is_complete()) + .then_some(&mut covered_edges), + ); + } + body.write_legend(out); + if !selection.modified_cells.is_empty() + || !selection.removed_cells.is_empty() + || !selection.added_cells.is_empty() + { + out.write_styled(TextStyle::Heading, format_args!("Cells:")); + out.push('\n'); + for name in selection.removed_cells { + out.write_styled(TextStyle::Removed, format_args!("- {}", cell_label(name))); + out.push('\n'); + } + for name in selection.added_cells { + out.write_styled(TextStyle::Added, format_args!("+ {}", cell_label(name))); + out.push('\n'); + } + body.write_body(out); + } + // General comparison errors precede revision-specific errors in text. + if !self.diagnostics.is_empty() { + out.write_styled(TextStyle::Heading, format_args!("Diagnostics")); + out.push('\n'); + for diagnostic in self + .diagnostics + .iter() + .filter(|d| d.side.is_none()) + .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) + { + writeln!(out, " {diagnostic}").unwrap(); + } + } + if let Some(max_depth) = options.ascii_depth { + write_hierarchy( + out, + &self.old, + &self.new, + Some(&self.cells), + false, + max_depth, + Some(&covered_edges), + ); + } + } + + fn write_cell_details( + &self, + out: &mut CellText<'_>, + selected: &selection::TextCellSelection<'_>, + details: bool, + mut covered_edges: Option<&mut BTreeSet<(String, String)>>, + ) { + let cell = selected.cell; + let name = cell_label(&cell.name); + let note = if selected.attributes_changed { + " (attributes changed)" + } else { + "" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "* {name}{note}").unwrap(); + let mut polygons = changes::PolygonChanges::default(); + let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); + for &(a, b) in &selected.replacements { + if !details && a.layer == b.layer && is_polygon(a.kind) { + polygons.replacement(a, b); + continue; + } + out.push_str(" "); + out.style(Some(TextStyle::Changed)); + out.push_str("~ "); + if a.layer != b.layer { + changes::write_layer_transition(out, a, b); + out.push('\n'); + } else { + if let Some(layer) = a.layer { + out.write_layer(layer, None); + out.push_str(": "); + } + let description = changes::replacement_description(a, b, details); + let (heading, rest) = match description.split_once('\n') { + Some((heading, rest)) => (heading, Some(rest)), + None => (description.as_str(), None), + }; + out.push_str(heading); + if let Some(target) = &a.target { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .replacement(a, b, placements); + } + out.push('\n'); + if let Some(rest) = rest { + writeln!(out, "{rest}").unwrap(); + } + } + if details { + changes::write_replacement_objects(out, a, b); + } + } + let mut removed_refs = BTreeMap::new(); + let mut added_refs = BTreeMap::new(); + for (removed, objects, references) in [ + (true, &selected.removed, &mut removed_refs), + (false, &selected.added, &mut added_refs), + ] { + let sign = if removed { '-' } else { '+' }; + let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); + for &object in objects { + if !details && is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if !details && let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; + } else { + unmatched.push((object, 1)); + } + } + // The comparison orders these objects by kind, fields, and properties. + for (object, count) in unmatched { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if count > 1 { + write!(out, "{count}× ").unwrap(); + } + changes::write_unmatched(out, object, details); + if let Some(target) = object.target.as_deref() { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .individual(object, removed, count, placements); + } + out.push('\n'); + } + } + let no_refs = changes::ReferenceCounts::default(); + for (target, (first, counts)) in removed_refs { + if let Some((_, added)) = added_refs.remove(target) { + let placements = counts.write_change(out, target, &added); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &added, placements); + } else { + let placements = counts.write(out, '-', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &no_refs, placements); + } + } + for (target, (first, counts)) in added_refs { + let placements = counts.write(out, '+', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&no_refs, &counts, placements); + } + if !shown_refs.is_empty() { + for target in cell + .removed + .iter() + .chain(&cell.added) + .filter_map(|o| o.target.as_deref()) + { + if let Some(shown) = shown_refs.get_mut(target) { + shown.count_change(); + } + } + for (target, shown) in shown_refs { + if shown.complete() { + let edge = (cell.name.clone(), target.to_owned()); + let old = self.old.edges.get(&edge).copied().unwrap_or(0); + let new = self.new.edges.get(&edge).copied().unwrap_or(0); + shown.write_placements(out, old, new); + if let Some(covered) = covered_edges.as_deref_mut() { + covered.insert(edge); + } + } + } + } + polygons.write(out); + } +} From 623de2e585faccbeebcfdfe5c0b0a71c01907c9b Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 10:21:56 -0800 Subject: [PATCH 39/41] Clarify comparison and report controls --- docs/cli.md | 25 ++++++++++++++----------- docs/comparison.md | 11 ++++++----- docs/diff-reports.md | 11 ++++++----- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index bee1df3..fb56827 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -127,21 +127,24 @@ pheon diff before.gds after.gds --hierarchy mermaid --depth 4 ``` `--show-cell` and `--show-layer` filter displayed changes without changing the -comparison, totals, or exit status. A layer display filter hides references -without a numeric layer. `--max-changes N` limits displayed object occurrences -after filtering. Text defaults to 100 occurrences and shows a replacement only -when both objects fit within the limit. Grouped rows count each object toward -the limit. When a layer filter selects one side of a layer-only replacement, -that side appears as an addition or removal. A single omission line lists -changes hidden by filters or the limit. JSON has no default object limit and reports -omissions from both display filters and any explicit limit in `report_selection`. -See the [JSON reference](diff-reports.md) for the selection counters. +comparison, totals, or exit status. In text reports, `--show-layer` filters +object changes within modified cells and hides references without a numeric +layer. Added and removed cells appear as name-only entries outside the object +budget. Use `--show-cell` to filter those entries. + +`--max-changes N` limits displayed object occurrences after filtering. Text +defaults to 100 occurrences and shows a replacement only when both objects fit +within the limit. Grouped rows count each object toward the limit. When a +layer filter selects one side of a layer-only replacement, that side appears as +an addition or removal. The omission line lists changes hidden by filters or the +limit. JSON has no default object limit and records omissions in +`report_selection`. See the [JSON reference](diff-reports.md) for the selection +counters. `--details` shows individual object evidence inside modified cells in text reports. Replacement rows include a nested `-` row for the old object and a `+` row for the new object, with cell-local coordinates in database units, -retained fields, and properties. Both sides count toward `--max-changes`. -For added and removed cells, the report lists the cell name. +retained fields, and properties. The diff hierarchy shows directly changed cells, affected ancestors, and targets of modified references. It includes references from both revisions so removed diff --git a/docs/comparison.md b/docs/comparison.md index a596f54..739fab3 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -27,11 +27,12 @@ operations, XOR, or tolerance-based matching. ## Read change explanations -Unmatched objects can be paired to explain a replacement when the pairing is -unique by kind, layer, and bounds, then by kind and layer. References also use -their target cell. Ambiguous candidates remain independent removals and -additions. Pairing helps explain the report; it does not establish persistent -object identity or change the underlying counts. +Pheon first pairs unmatched polygons, boxes, and paths that differ only in +layer or type, with exactly one candidate in each revision. It then checks the +remaining objects for unique matches by kind, layer, and bounds, followed by +kind and layer. Reference matching uses the target cell. Ambiguous candidates +are reported as separate removals and additions. These explanations apply to +the current report and preserve the underlying addition and removal counts. Reported bounds are envelopes of stored coordinates in cell-local database units. They exclude path width, rendered text size, and recursively transformed diff --git a/docs/diff-reports.md b/docs/diff-reports.md index c88cdfd..b71b0ab 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -108,13 +108,14 @@ Limitation codes are `unsupported_record`, `unsupported_metadata_version`, `unresolved_reference`, `hierarchy_cycle`, and `scope_mismatch`. Failure codes are `invalid_input`, `io_error`, `invalid_scope`, and `command_error`. -`--hierarchy graph` includes the full scoped union graph; otherwise `hierarchy` -is null. Nodes are keyed by name, with per-side states `included`, `outside_scope`, -`unresolved`, or `absent`, and change states `direct`, `affected`, or `unchanged`. +`--hierarchy graph` includes the full scoped union graph. By default, `hierarchy` +is null. JSON output rejects `--depth` with exit code 2. Nodes are keyed by name, +with per-side states `included`, `outside_scope`, `unresolved`, or `absent`, and +change states `direct`, `affected`, or `unchanged`. Edges record parent, target, before/after direct placement counts, and changed reference records. Array multiplicities are symbolic, not expanded or recursively -accumulated. Shared cells occur once and removed edges remain. Display filters, -object limits, and text depth do not trim the graph. +accumulated. Shared cells occur once and removed edges remain. The graph covers +the full comparison scope regardless of display filters and object limits. ## Versions From 2b5b184dc6516d41ad900418b3cb5593203ce274 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 11:04:20 -0800 Subject: [PATCH 40/41] Refresh installation and walkthrough documentation --- README.md | 18 +++++---- docs/cli.md | 98 +++++++++++++++++++-------------------------- docs/development.md | 47 +++++++++++++--------- docs/native-api.md | 8 +++- docs/python.md | 18 ++++----- 5 files changed, 93 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 10f3fc5..eaf6d3c 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ other object details. They compare object fields and coordinates exactly; see th ## Try the command-line tools -From a source checkout, install the CLI with Rust 1.88 or newer: +Install the CLI [from crates.io][crate] with Rust 1.88 or newer. +Use your own GDSII paths in the examples. ```sh -cargo install --path . --locked --bin pheon +cargo install pheon --locked pheon dump chip.gds pheon diff before.gds after.gds pheon hierarchy chip.gds --format mermaid @@ -29,13 +30,13 @@ comparisons. Start with the [CLI guide][cli], then set up ## Read layouts in Python -Python 3.11 or newer is required. From a source checkout: +Install the Python package [from PyPI][pypi] with Python 3.11 or newer. ```sh -python -m pip install . +python -m pip install pheon ``` -Run Python outside the source checkout to use the installed package: +Run this example from a directory containing `chip.gds`. ```python from pheon import arrow @@ -44,9 +45,8 @@ layout = arrow.read_path("chip.gds") print(layout.field("lib_name")[0].as_py()) ``` -Source installation needs Rust; prebuilt Python wheels bundle the native library. -The [Python guide][python] covers installation, selected reads, and integer -packing. The [Arrow reference][arrow] describes the returned data. +The Python package installs PyArrow. See the [Python guide][python] for source +installs and selected reads, and the [Arrow reference][arrow] for the returned data. ## Learn more @@ -59,7 +59,9 @@ Licensed under [GPL-3.0-only][license]. [arrow]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/arrow-schema.md [cli]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/cli.md [comparison]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/comparison.md +[crate]: https://crates.io/crates/pheon [docs]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/index.md [git]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/git.md [license]: https://mpxd.net/code/jan/pheon/src/branch/master/LICENSE.md +[pypi]: https://pypi.org/project/pheon/ [python]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/python.md diff --git a/docs/cli.md b/docs/cli.md index fb56827..2d66ea6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -6,18 +6,17 @@ refer to the decompressed layout. The CLI runs independently of Python. ## Install -If you downloaded a CLI archive, extract it and add the directory containing -`pheon` (`pheon.exe` on Windows) to your `PATH`. Run `pheon --help` to check it. - -To install from a source checkout, use Rust 1.88 or newer: +Install the CLI [from crates.io](https://crates.io/crates/pheon) with Rust 1.88 +or newer. ```sh -cargo install --path . --locked --bin pheon +cargo install pheon --locked pheon --help ``` -You can also build locally with `cargo build --release --locked --bin pheon` -and run `target/release/pheon` (`pheon.exe` on Windows). +From a source checkout, install with `cargo install --path . --locked --bin pheon` +or build with `cargo build --release --locked --bin pheon` and run +`target/release/pheon` (`pheon.exe` on Windows). ## Inspect a layout @@ -44,44 +43,22 @@ output is diagram source for a Mermaid renderer. ## Compare revisions +Run `pheon diff before.gds after.gds` with your own files, or download these +small layouts from the pinned test corpus to try a comparison. +[small_rect.gds][small] · [big_rect.gds][big] + ```sh -pheon diff before.gds after.gds -pheon diff before.gds after.gds --hierarchy ascii --max-changes 20 -pheon diff before.gds after.gds --format json > report.json +pheon diff small_rect.gds big_rect.gds ``` -Text reports start with the result. The headline includes cell counts for -reports with more than three changed cells, incomplete comparisons, or omitted -changes. The `Cells:` section lists removals with `-`, additions with `+`, and -modifications with `*`. Modified cells show counts of rectangles, polygons, and -boxes changed, moved, added, or removed on each layer. When the only displayed -shape change on a layer is a move, the row shows its displacement in database -units. Repeated reference additions or removals to the same target appear as a -count. When both occur for one target, they share a row. A complete set of edits -to a target also shows -the total direct placements when there are unchanged placements, for example -`leaf · -3 refs, +6 refs · placements 10 -> 13`. +The example exits with code 1 and shows one changed rectangle in `TOP`. -An unchanged layer name appears inline when the layer occurs on one displayed -row, as in `1/0 (WG): rectangle moved by (20, -5) dbu`. A layer on several -displayed rows gets one legend entry. Object details use `-` for removals, `+` for additions, -and `~` for likely replacements. Identical unmatched objects share a row -with a count such as `+ 12× 1/0: path …`. Unmatched paths with five or more -vertices show their width, vertex count, and vertex bounds in one line. -A unique polygon, box, or path replacement that differs only in layer or -datatype appears on one row, for example `~ 1/0 (WG) → 2/0 (CORE) · rectangle`. - -Use `--format json` for details of individual objects and `--hierarchy` to see -which parent cells are affected. - -`diff` and `dump` use `--color auto` by default. Colors appear on supported -terminals and are disabled when `NO_COLOR` is set or `TERM=dumb`. Use -`--color=always` to keep colors in a pipe, for example -`pheon diff before.gds after.gds --color=always | less -R`. Use -`--color=never` for plain text. JSON reports and Mermaid diagram source stay plain. - -For large unscoped diffs with substantial byte overlap, Pheon reuses parsed -contents of matching cells. Cell timestamps and file offsets can differ. +```text +Different +Cells: +* TOP + 2/0: 1 rectangle changed +``` | Exit code | Meaning | | --- | --- | @@ -94,6 +71,21 @@ differences, but equality cannot be established. Read the [comparison rules](comparison.md) before relying on results in automation, and the [JSON reference](diff-reports.md) to consume reports programmatically. +### Read the text report + +The cell list marks additions with `+`, removals with `-`, and modifications +with `*`. Modified cells summarize shape changes by layer and repeated +references by target. A single move can show its displacement in database units. +Layer names appear inline or in a legend. Object rows use `~` for likely +replacements and `+` or `-` for unmatched additions and removals. + +Use `--details` for individual objects, `--hierarchy ascii` for affected +parents, or `--format json` for machine-readable evidence. + +`diff` and `dump` color supported terminals by default. Set `NO_COLOR`, use +`TERM=dumb`, or pass `--color=never` for plain text. Pass `--color=always` when +piping into `less -R`. JSON and Mermaid diagram source stay plain. + ## Choose the comparison scope ```sh @@ -146,20 +138,14 @@ reports. Replacement rows include a nested `-` row for the old object and a `+` row for the new object, with cell-local coordinates in database units, retained fields, and properties. -The diff hierarchy shows directly changed cells, affected ancestors, and targets -of modified references. It includes references from both revisions so removed -branches remain visible. Counts represent direct placements, with symbolic -counts for arrays. ASCII uses the cell list's change markers and marks -reference changes whose placement count stays the same. When the cell list -shows all edits to an unchanged target on a single branch, ASCII can omit -that terminal edge. The hierarchy retains ancestor paths, shared branches, -and edges with edits hidden by display filters or limits. ASCII appears when -there are relationships to show. ASCII diffs fold chains of three or more cells -along unchanged single-placement edges, for example `CHIP → block → * leaf`. -Branch points and changed edges keep their own rows. `--depth` counts the -original edges. -For the complete graph within the comparison scope, use -`--format json --hierarchy graph` without `--depth`. Display filters leave the -graph intact. +The diff hierarchy shows changed cells, affected ancestors, and modified +references from both revisions. Counts are direct placements, including +symbolic array counts. ASCII folds straight chains of unchanged edges and can +omit terminal edges already explained by the cell list. `--depth` counts the +original edges. For the full graph within the comparison scope, use +`--format json --hierarchy graph` without `--depth`. Next: configure [Git integration](git.md) or add [layer names](layer-names.md). + +[small]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/small_rect.gds +[big]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/big_rect.gds diff --git a/docs/development.md b/docs/development.md index e884456..1d960d7 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,35 +1,42 @@ # Development -This guide is for contributors building and checking Pheon. For usage, start -with the [CLI](cli.md) or [Python](python.md) guide. +The checkout selects stable Rust. The minimum supported compiler is 1.88. +Use the checked-in `Cargo.lock` and update dependencies explicitly with Cargo. -## Build and test +## Check a change -The checkout selects the stable Rust toolchain; the minimum supported compiler -is 1.88. Python bindings require Python 3.11 or newer. Source builds require -the tracked `Cargo.lock`; update dependencies explicitly with Cargo. +| Area | Start here | First check | +| --- | --- | --- | +| GDSII parsing and Arrow reads | [`src/record.rs`](../src/record.rs), [`src/library.rs`](../src/library.rs) | `cargo test --locked --test parser_errors --test selective_reads` | +| Comparisons and CLI | [`src/review.rs`](../src/review.rs), [`src/review/`](../src/review/), [`src/bin/pheon.rs`](../src/bin/pheon.rs) | `cargo test --locked --test review` | +| Python and CFFI | [`pheon/`](../pheon/), [`src/lib.rs`](../src/lib.rs) | `python -I tools/check_installed.py` after installation | + +Run `cargo test --locked` for the full Rust suite or `cargo test --release --locked` +for release builds. Set `RUSTUP_TOOLCHAIN=1.88.0` to check the minimum compiler. + +## Test the Python package + +Use Python 3.11 or newer and install into a virtual environment. ```sh -cargo test --locked -cargo test --release --locked -cargo build --release --locked +python3 -m venv .venv +. .venv/bin/activate python -m pip install . python -I tools/check_installed.py ``` -Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler. -The Python package bundles its native library during installation and reads its -version from installed package metadata. After installation, to test a newly -built library without reinstalling, set `PHEON_LIBRARY` to its absolute -path and run `python -m unittest discover -s tests/python` from the checkout. -Unset this override when checking installed artifacts with `check_installed.py`. +To test a newly built shared library, set `PHEON_LIBRARY` to its absolute path +and run `python -m unittest discover -s tests/python` from the checkout. Clear +the override when checking the bundled library with `check_installed.py`. ## Check upstream layouts The [GDSII corpus](../contrib/gds-corpus.toml) pins open-source fixtures and cell -libraries. After installing Pheon and building the release CLI, run: +libraries. Install Pheon in the virtual environment before running the corpus +check. ```sh +cargo build --release --locked --bin pheon python -I tools/check_gds_corpus.py --fetch ``` @@ -40,7 +47,7 @@ to `.cache/gds-corpus/report.json` and exits with status 1 if a check fails. ## Interfaces and reports -The CFFI declarations live in [pheon/_ffi.h](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h). Keep them aligned +The CFFI declarations live in [`pheon/_ffi.h`](../pheon/_ffi.h). Keep them aligned with the exported native functions. The C API version and [Arrow layout version](arrow-schema.md) are independent of package releases and the [JSON report schema](diff-reports.md). @@ -53,9 +60,9 @@ changing their contract: PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens ``` -Inspect the resulting diff. Goldens normalize the producer version to `test`; -a package version change alone should not alter them. JSON Schema checks verify -structure, while semantic tests verify links, totals, and result/exit agreement. +Inspect the resulting diff. Goldens normalize the producer version to `test`. +Use the [JSON Schema](diff-report-v1.schema.json) to check structure. Rust tests +verify links, totals, and agreement between the result and exit status. See [building releases](releases.md) for source archives, wheels, and standalone CLI artifacts. diff --git a/docs/native-api.md b/docs/native-api.md index 7e0a020..01ddcf4 100644 --- a/docs/native-api.md +++ b/docs/native-api.md @@ -5,7 +5,13 @@ returned data layout, see the [Arrow reference](arrow-schema.md). ## Rust -Add `pheon` as a dependency. The library readers accept GDSII byte slices: +Add [pheon](https://crates.io/crates/pheon) from your Rust project. + +```sh +cargo add pheon +``` + +The library readers accept GDSII byte slices. ```rust use pheon::library::read_library; diff --git a/docs/python.md b/docs/python.md index bc03060..f02fffb 100644 --- a/docs/python.md +++ b/docs/python.md @@ -1,22 +1,24 @@ # Python Use `pheon.arrow` to load GDSII geometry into PyArrow, scan a hierarchy, or read -only selected cells. The base `pheon` package also provides integer packing. +only selected cells. The package also provides integer packing. Python 3.11 or newer is required. ## Install -To install a PyPI release: +Install Pheon from [PyPI](https://pypi.org/project/pheon/). ```sh python -m pip install pheon ``` The package installs PyArrow for layout reads. Existing `pheon[arrow]` -requirements still work. Prebuilt wheels bundle the native library. From a -source checkout, use `python -m pip install .` with Rust 1.88 or newer. +requirements still work. [Prebuilt wheels](https://pypi.org/project/pheon/#files) +bundle the native library. On platforms without a compatible wheel, pip builds +the library from source and needs Rust 1.88 or newer and a native linker. -Run Python outside the source checkout to use the installed package. +From a source checkout, install with `python -m pip install .`, then run Python +outside the checkout to import the installed package. ## Read a layout @@ -85,11 +87,5 @@ Native parse failures raise `pheon.PheonError`, a subclass of `ValueError`. Invalid buffer or range arguments raise `ValueError` or `TypeError`; file access errors use Python's normal I/O exceptions. -## Use from Klamath or Masque - -Klamath's `native` extra installs Pheon's packing helpers. Masque's `arrow` extra -installs Pheon for layout reads. Use the same Python environment as the -companion package. - See [development](development.md) to test a locally built native library from Python, or the [native API guide](native-api.md) to call the library directly. From 6dd54987def9ab4ae0cb731af6c4463cbe2a0a83 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 11:06:13 -0800 Subject: [PATCH 41/41] Build release wheels with a checkout-local Podman store --- docs/releases.md | 72 +++++++++++---------------- tools/build_manylinux.py | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 42 deletions(-) create mode 100644 tools/build_manylinux.py diff --git a/docs/releases.md b/docs/releases.md index b21c70d..c36c7ba 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,16 +1,18 @@ # Building releases -Build packages locally for distribution. For installing and using Pheon, start -with the [CLI](cli.md) or [Python](python.md) guide. +Build packages locally for distribution. The checkout holds Python build tools +in `.venv/`, release artifacts in `dist/release/`, and Linux wheel caches in +`.cache/wheels/`. Use your installed Rust toolchain and Cargo registry cache. ## Start a release -Use Python 3.11+ and Rust 1.88+. From the repository root, create a build -environment and put the source archive in an empty `dist/release/` directory. +Use Python 3.11+ and Rust 1.88+. Run the commands from the repository root and +start with an empty `dist/release/` directory. ```sh python3 -m venv .venv . .venv/bin/activate +export PIP_CACHE_DIR="$PWD/.cache/wheels/pip" python -m pip install build twine python -m build --sdist --outdir dist/release ``` @@ -19,50 +21,28 @@ Use `python -m build --wheel` to build a wheel for your own machine. For a portable Linux wheel, use the container below. See [development](development.md) for compiler checks. -Python wheels bundle the native library. Install the CLI separately with Cargo. - ## Build Linux x86-64 wheels -On x86-64 Linux, use Podman and a Rust toolchain installed through `rustup`. -Download the [pinned manylinux image](../contrib/wheels/manylinux-image.txt) -once and save it for future releases. - -```sh -pheon_wheel_cache="$PWD/.cache/wheels" -pheon_image=$(cat contrib/wheels/manylinux-image.txt) -pheon_cached_image=localhost/pheon-manylinux:2.28-2026.10.26-1 -mkdir -p "$pheon_wheel_cache" -podman pull --platform linux/amd64 "$pheon_image" -podman tag "$pheon_image" "$pheon_cached_image" -podman save --format oci-archive \ - --output "$pheon_wheel_cache/manylinux_2_28_x86_64.oci.tar" \ - "$pheon_cached_image" -``` - -If the image is missing from Podman, restore it with -`podman load --input .cache/wheels/manylinux_2_28_x86_64.oci.tar`. -Build from the source archive using the local Rust toolchain and Cargo cache. +On x86-64 Linux, use rootless Podman and a Rust toolchain installed through +`rustup`. Build the portable wheel from the source archive. ```sh python -m pip install 'cibuildwheel==4.3.5' -cargo fetch --locked --target x86_64-unknown-linux-gnu -rust_toolchain=$(rustc --print sysroot) -cargo_registry="${CARGO_HOME:-$HOME/.cargo}/registry" -export CIBW_CONTAINER_ENGINE="podman; disable_host_mount: true; create_args: \ - --volume '$PWD/.cache/wheels:/pheon-cache' \ - --volume '$rust_toolchain:/rust-toolchain:ro' \ - --volume '$cargo_registry:/pheon-cache/cargo/registry:ro'" -export CIBW_MANYLINUX_X86_64_IMAGE=localhost/pheon-manylinux:2.28-2026.10.26-1 -export CIBW_ENVIRONMENT_LINUX='PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo CARGO_NET_OFFLINE=true PIP_CACHE_DIR=/pheon-cache/pip' -export CIBW_BUILD='cp31{1,2,3,4}-manylinux_x86_64' -export CIBW_TEST_COMMAND='python -I {project}/tools/check_installed.py' -unset PHEON_LIBRARY -python -m cibuildwheel --platform linux --archs x86_64 \ - --output-dir dist/release dist/release/pheon-*.tar.gz +python tools/build_manylinux.py dist/release/pheon-*.tar.gz ``` -cibuildwheel repairs the wheel and tests the same file on Python 3.11–3.14. -The container downloads any Python dependencies that it needs. +The helper pulls the [pinned image](../contrib/wheels/manylinux-image.txt) when +needed and reuses it from `.cache/wheels/podman/storage/`. It keeps Podman’s +storage runtime directory, download scratch space, and wheel caches under +`.cache/wheels/`. Podman still uses the normal system user runtime for host +services. To inspect this checkout’s images, use the same helper. + +```sh +python tools/build_manylinux.py podman image ls +``` + +cibuildwheel repairs the wheel and tests it on Python 3.11–3.14. The container +uses the host Rust toolchain and Cargo registry through read-only mounts. ## Build Windows x86-64 packages from Linux @@ -104,4 +84,12 @@ python -I tools/check_installed.py Build standalone CLI artifacts for each OS and architecture. Include the executable, license, README, documentation, and Git templates. -Publish the validated Python packages to PyPI and the Rust crate to crates.io. +## Publish + +Upload the validated Python files you selected, then publish the crate from a +clean release checkout. + +```sh +python -m twine upload --repository pypi FILE ... +cargo publish --locked +``` diff --git a/tools/build_manylinux.py b/tools/build_manylinux.py new file mode 100644 index 0000000..c90c7da --- /dev/null +++ b/tools/build_manylinux.py @@ -0,0 +1,104 @@ +"""Build Linux release wheels using Podman storage in this checkout.""" + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +CACHE = ROOT / ".cache/wheels" + + +def local_environment() -> dict[str, str]: + podman = CACHE / "podman" + scratch = CACHE / "tmp" + podman.mkdir(parents=True, exist_ok=True) + scratch.mkdir(exist_ok=True) + storage = podman / "storage.conf" + graphroot = json.dumps(str(podman / "storage"), ensure_ascii=False) + runroot = json.dumps(str(podman / "run"), ensure_ascii=False) + config = f"[storage]\ngraphroot = {graphroot}\nrunroot = {runroot}\n" + # Replace the config atomically so simultaneous Podman commands can read it. + if not storage.exists() or storage.read_text(encoding="utf-8") != config: + temporary = podman / f"storage.{os.getpid()}.conf" + temporary.write_text(config, encoding="utf-8") + temporary.replace(storage) + + env = dict(os.environ, CONTAINERS_STORAGE_CONF=str(storage), TMPDIR=str(scratch)) + for key in ("CONTAINER_HOST", "CONTAINER_CONNECTION"): + env.pop(key, None) + return env + + +def build(sdist: Path) -> None: + env = local_environment() + image = (ROOT / "contrib/wheels/manylinux-image.txt").read_text(encoding="utf-8").strip() + + # Use the same Podman invocation as cibuildwheel to check the active store. + graphroot = subprocess.check_output( + ["podman", "info", "--format", "{{.Store.GraphRoot}}"], env=env, text=True, + ).strip() + if Path(graphroot).resolve() != (CACHE / "podman/storage").resolve(): + raise SystemExit(f"Expected the checkout's Podman store, found {graphroot}") + + subprocess.run( + ["cargo", "fetch", "--locked", "--target", "x86_64-unknown-linux-gnu"], + cwd=ROOT, env=env, check=True, + ) + toolchain = subprocess.check_output( + ["rustc", "--print", "sysroot"], cwd=ROOT, env=env, text=True, + ).strip() + registry = Path(env.get("CARGO_HOME", Path.home() / ".cargo")).resolve() / "registry" + for directory in ("cargo/registry", "pip", "target", "cibuildwheel"): + (CACHE / directory).mkdir(parents=True, exist_ok=True) + mounts = [ + f"{toolchain}:/rust-toolchain:ro", + f"{CACHE / 'cargo'}:/pheon-cache/cargo", + f"{registry}:/pheon-cache/cargo/registry:ro", + f"{CACHE / 'pip'}:/pheon-cache/pip", + f"{CACHE / 'target'}:/pheon-cache/target", + ] + create_args = shlex.join([arg for mount in mounts for arg in ("--volume", mount)]) + env.update( + CIBW_CONTAINER_ENGINE=f"podman; disable_host_mount: true; create_args: {create_args}", + CIBW_MANYLINUX_X86_64_IMAGE=image, + CIBW_ENVIRONMENT_LINUX=( + "PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo " + "CARGO_TARGET_DIR=/pheon-cache/target CARGO_NET_OFFLINE=true " + "PIP_CACHE_DIR=/pheon-cache/pip" + ), + CIBW_CACHE_PATH=str(CACHE / "cibuildwheel"), + CIBW_BUILD="cp31{1,2,3,4}-manylinux_x86_64", + CIBW_TEST_COMMAND="python -I {project}/tools/check_installed.py", + ) + env.pop("PHEON_LIBRARY", None) + subprocess.run( + [sys.executable, "-m", "cibuildwheel", "--platform", "linux", "--archs", "x86_64", + "--output-dir", str(sdist.parent), str(sdist)], + cwd=ROOT, env=env, check=True, + ) + + +def main() -> int: + if sys.argv[1:2] == ["podman"]: + return subprocess.run( + ["podman", "--remote=false", *sys.argv[2:]], env=local_environment(), + ).returncode + + parser = argparse.ArgumentParser( + description=__doc__, epilog="Use 'podman ARGS' to inspect the local store.", + ) + parser.add_argument("sdist", type=Path, help="Pheon source archive") + args = parser.parse_args() + if not args.sdist.is_file(): + parser.error(f"source archive does not exist: {args.sdist}") + build(args.sdist.resolve()) + return 0 + + +if __name__ == "__main__": + sys.exit(main())