pheon/src/review.rs
Jan Petykiewicz b3346b40a4 Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy
scans and selective cell loading. Validate numeric packing and array pitches,
and preserve normalized REAL8 encodings exactly within the supported range.

Add standalone dump, structural diff, hierarchy, layer-name annotation and Git
diff commands. Provide scoped text and JSON reports, deterministic comparison,
and record-preserving metadata edits.

Package the shared library and optional Python Arrow readers as pheon using
CFFI ABI mode and Python-independent platform wheels. Include locked source
builds, native API checks, cross-platform artifact workflows, documentation
and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer.

Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14,
and 71 installed companion integration tests pass. Full, scanned and selected
Arrow layouts match the previous schemas.
2026-11-16 16:21:21 -08:00

1492 lines
50 KiB
Rust

//! Deterministic GDS object review without the normalization performed by the
//! Arrow reader. Raw payloads borrow the file; only polygon keys are copied.
use crate::{basic::decode_real8, record::RecordHeader, records::*};
use std::{
borrow::Cow,
collections::{BTreeMap, BTreeSet},
fmt::Write,
};
mod names;
pub use names::{parse_layer, parse_layer_name, write_layer_names};
mod selection;
pub use selection::{DisplayFilter, Scope};
mod changes;
pub use changes::Replacement;
pub mod report;
/// A comparison limitation with source context independent of its wording.
#[derive(Clone, Debug)]
pub struct Diagnostic {
pub code: &'static str,
pub message: String,
pub offset: Option<usize>,
pub cell: Option<String>,
pub layer: Option<Layer>,
pub target: Option<String>,
}
impl Diagnostic {
fn at(code: &'static str, offset: usize, message: String) -> Self {
Self {
code,
message: error(offset, message),
offset: Some(offset),
cell: None,
layer: None,
target: None,
}
}
}
pub const PROFILE: &str = "klamath-object-v2";
pub const LAYER_SENTINEL: &str = "KLAMATH_LAYER_NAMES_V1";
pub type Layer = (u16, u16);
pub type Bounds = [i32; 4];
#[derive(Clone, Debug)]
pub struct Record<'a> {
pub offset: usize,
pub tag: u16,
pub data: &'a [u8],
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct Key<'a> {
kind: u16,
fields: Vec<(u16, Cow<'a, [u8]>)>,
properties: Vec<(u16, &'a [u8])>,
}
#[derive(Debug)]
pub struct Object<'a> {
pub offset: usize,
pub end_offset: usize,
pub kind: u16,
pub layer: Option<Layer>,
pub bounds: Option<Bounds>,
pub target: Option<String>,
pub multiplicity: u64,
key: Key<'a>,
metadata: bool,
issues: Vec<Diagnostic>,
}
impl Object<'_> {
fn field(&self, tag: u16) -> Option<&[u8]> {
self.key
.fields
.iter()
.find(|(t, _)| *t == tag)
.map(|(_, v)| v.as_ref())
}
pub fn description(&self) -> String {
let mut out = kind_name(self.kind).to_string();
for (tag, value) in &self.key.fields {
write!(out, " {}={}", tag_name(*tag), format_data(*tag, value)).unwrap();
}
for (attribute, value) in &self.key.properties {
write!(out, " property[{attribute}]={}", quoted(value)).unwrap();
}
out
}
}
#[derive(Debug)]
pub struct Cell<'a> {
pub offset: usize,
/// Start of ENDSTR, for inserting records without rewriting the cell.
pub end_offset: usize,
pub objects: Vec<Object<'a>>,
attributes: Vec<(u16, &'a [u8])>,
issues: Vec<Diagnostic>,
}
#[derive(Debug)]
pub struct Document<'a> {
pub records: Vec<Record<'a>>,
pub cells: BTreeMap<String, Cell<'a>>,
pub layer_names: BTreeMap<Layer, String>,
/// Unsupported features or invalid reference graphs prevent an equality claim.
pub issues: Vec<String>,
pub diagnostics: Vec<Diagnostic>,
header: Vec<(u16, &'a [u8])>,
global_issues: Vec<Diagnostic>,
known_cells: BTreeSet<String>,
pub scope: Scope,
present: bool,
}
fn error(offset: usize, message: impl std::fmt::Display) -> String {
format!("at byte {offset}: {message}")
}
fn string(data: &[u8], offset: usize) -> Result<String, String> {
decoded_string(unpad(data), offset)
}
fn decoded_string(data: &[u8], offset: usize) -> Result<String, String> {
if data.contains(&0) {
return Err(error(offset, "name or metadata contains an embedded NUL"));
}
std::str::from_utf8(data)
.map(str::to_owned)
.map_err(|_| error(offset, "name or metadata is not UTF-8"))
}
fn unpad(data: &[u8]) -> &[u8] {
if data.last() == Some(&0) {
&data[..data.len() - 1]
} else {
data
}
}
fn integer(data: &[u8]) -> u16 {
u16::from_be_bytes(data.try_into().unwrap())
}
fn points(data: &[u8]) -> Vec<[i32; 2]> {
data.as_chunks::<8>()
.0
.iter()
.map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| {
[
i32::from_be_bytes([x0, x1, x2, x3]),
i32::from_be_bytes([y0, y1, y2, y3]),
]
})
.collect()
}
// Booth's least-rotation algorithm avoids quadratic work on repeated vertices.
fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> {
let n = points.len();
let (mut i, mut j, mut k) = (0, 1, 0);
while i < n && j < n && k < n {
match points[(i + k) % n].cmp(&points[(j + k) % n]) {
std::cmp::Ordering::Equal => k += 1,
std::cmp::Ordering::Greater => {
i += k + 1;
if i <= j {
i = j + 1;
}
k = 0;
}
std::cmp::Ordering::Less => {
j += k + 1;
if j <= i {
j = i + 1;
}
k = 0;
}
}
}
let start = i.min(j);
(0..n).map(|k| points[(start + k) % n]).collect()
}
fn canonical_ring(data: &[u8]) -> Vec<u8> {
let mut ring = points(data);
ring.pop(); // The caller has checked closure.
let forward = least_rotation(&ring);
ring.reverse();
let backward = least_rotation(&ring);
let mut ring = forward.min(backward);
ring.push(ring[0]);
ring.into_iter()
.flatten()
.flat_map(i32::to_be_bytes)
.collect()
}
fn validate_payload(record: &Record<'_>) -> Result<(), String> {
let size = record.data.len();
let valid = match record.tag {
RTAG_BGNLIB | RTAG_BGNSTR => size == 24,
RTAG_UNITS => size == 16,
RTAG_HEADER | RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE
| RTAG_PROPATTR | RTAG_PATHTYPE | RTAG_STRANS | RTAG_PRESENTATION | RTAG_ELFLAGS
| RTAG_STRCLASS | RTAG_GENERATIONS => size == 2,
RTAG_WIDTH | RTAG_BGNEXTN | RTAG_ENDEXTN | RTAG_PLEX | RTAG_COLROW => size == 4,
RTAG_MAG | RTAG_ANGLE => size == 8,
RTAG_XY => size.is_multiple_of(8),
RTAG_ENDLIB | RTAG_ENDSTR | RTAG_ENDEL | RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF
| RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX | RTAG_ENDMASKS => size == 0,
_ => true,
};
if !valid {
return Err(error(
record.offset,
format!("invalid payload size {size} for {}", tag_name(record.tag)),
));
}
if record.tag == RTAG_UNITS
&& record.data.as_chunks::<8>().0.iter().any(|&bytes| {
let value = decode_real8(u64::from_be_bytes(bytes));
!value.is_finite() || value <= 0.0
})
{
return Err(error(record.offset, "UNITS must be positive finite values"));
}
Ok(())
}
fn is_element(tag: u16) -> bool {
matches!(
tag,
RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX
)
}
fn allowed_field(kind: u16, tag: u16) -> bool {
if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) {
return true;
}
match kind {
RTAG_BOUNDARY => matches!(tag, RTAG_LAYER | RTAG_DATATYPE),
RTAG_BOX => matches!(tag, RTAG_LAYER | RTAG_BOXTYPE),
RTAG_NODE => matches!(tag, RTAG_LAYER | RTAG_NODETYPE),
RTAG_PATH => matches!(
tag,
RTAG_LAYER | RTAG_DATATYPE | RTAG_WIDTH | RTAG_PATHTYPE | RTAG_BGNEXTN | RTAG_ENDEXTN
),
RTAG_SREF => matches!(tag, RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE),
RTAG_AREF => matches!(
tag,
RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE | RTAG_COLROW
),
RTAG_TEXT => matches!(
tag,
RTAG_LAYER
| RTAG_TEXTTYPE
| RTAG_STRING
| RTAG_STRANS
| RTAG_MAG
| RTAG_ANGLE
| RTAG_PRESENTATION
| RTAG_PATHTYPE
| RTAG_WIDTH
),
_ => false,
}
}
fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Object<'a>, String> {
let first = &records[0];
let mut fields = Vec::new();
let mut properties = Vec::new();
let mut seen = BTreeSet::new();
let mut i = 1;
while i + 1 < records.len() {
let record = &records[i];
if record.tag == RTAG_PROPATTR {
let value = records
.get(i + 1)
.filter(|r| r.tag == RTAG_PROPVALUE)
.ok_or_else(|| error(record.offset, "PROPATTR without PROPVALUE"))?;
properties.push((integer(record.data), unpad(value.data)));
i += 2;
continue;
}
if !allowed_field(first.tag, record.tag) {
issues.push(Diagnostic::at(
"unsupported_record",
record.offset,
format!(
"unsupported record 0x{:04x} in {}",
record.tag,
kind_name(first.tag)
),
));
}
if !seen.insert(record.tag) {
return Err(error(record.offset, "duplicate element field"));
}
let mut data = Cow::Borrowed(record.data);
if record.tag & 0xff == 6 {
data = Cow::Borrowed(unpad(record.data));
}
if record.tag == RTAG_XY && matches!(first.tag, RTAG_BOUNDARY | RTAG_BOX) {
let p = points(record.data);
if p.len() < 4 || p.first() != p.last() {
return Err(error(
record.offset,
"polygon must have at least three vertices and be closed",
));
}
data = Cow::Owned(canonical_ring(record.data));
}
fields.push((record.tag, data));
i += 1;
}
fields.sort();
properties.sort();
let mut object = Object {
offset: first.offset,
end_offset: records.last().unwrap().offset + 4,
kind: first.tag,
layer: None,
bounds: None,
target: None,
multiplicity: 1,
key: Key {
kind: first.tag,
fields,
properties,
},
metadata: false,
issues: Vec::new(),
};
let xy = object
.field(RTAG_XY)
.ok_or_else(|| error(first.offset, "element has no XY record"))?;
let p = points(xy);
let valid_count = match first.tag {
RTAG_TEXT | RTAG_SREF => p.len() == 1,
RTAG_AREF => p.len() == 3,
RTAG_PATH => p.len() >= 2,
RTAG_NODE => !p.is_empty(),
RTAG_BOX => p.len() == 5,
_ => true,
};
if !valid_count {
return Err(error(first.offset, "invalid coordinate count"));
}
object.bounds = Some([
p.iter().map(|p| p[0]).min().unwrap(),
p.iter().map(|p| p[1]).min().unwrap(),
p.iter().map(|p| p[0]).max().unwrap(),
p.iter().map(|p| p[1]).max().unwrap(),
]);
if matches!(first.tag, RTAG_SREF | RTAG_AREF) {
object.target = Some(decoded_string(
object
.field(RTAG_SNAME)
.ok_or_else(|| error(first.offset, "reference has no SNAME"))?,
first.offset,
)?);
if first.tag == RTAG_AREF {
let counts = object
.field(RTAG_COLROW)
.ok_or_else(|| error(first.offset, "array has no COLROW"))?;
let a = i16::from_be_bytes(counts[..2].try_into().unwrap());
let b = i16::from_be_bytes(counts[2..].try_into().unwrap());
if a <= 0 || b <= 0 {
return Err(error(first.offset, "array counts must be positive"));
}
object.multiplicity = a as u64 * b as u64;
}
} else {
let dtype = match first.tag {
RTAG_TEXT => RTAG_TEXTTYPE,
RTAG_BOX => RTAG_BOXTYPE,
RTAG_NODE => RTAG_NODETYPE,
_ => RTAG_DATATYPE,
};
object.layer = Some((
integer(
object
.field(RTAG_LAYER)
.ok_or_else(|| error(first.offset, "element has no LAYER"))?,
),
integer(
object
.field(dtype)
.ok_or_else(|| error(first.offset, "element has no datatype/texttype"))?,
),
));
if first.tag == RTAG_TEXT && object.field(RTAG_STRING).is_none() {
return Err(error(first.offset, "text has no STRING"));
}
}
Ok(object)
}
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
let mut records = Vec::new();
let mut offset = 0;
while offset < input.len() {
let (payload, header) = RecordHeader::read(&input[offset..])
.map_err(|(_, e)| error(offset, format!("invalid record header: {e:?}")))?;
let size = header.data_size as usize;
if !(size + 4).is_multiple_of(2) {
return Err(error(offset, "odd record length"));
}
let data = payload
.get(..size)
.ok_or_else(|| error(offset, "truncated record"))?;
let record = Record {
offset,
tag: header.tag,
data,
};
offset += size + 4;
let ended = record.tag == RTAG_ENDLIB;
records.push(record);
if ended {
if input[offset..].iter().any(|b| *b != 0) {
return Err(error(offset, "nonzero data after ENDLIB"));
}
break;
}
}
Ok(records)
}
impl<'a> Document<'a> {
pub fn parse(input: &'a [u8]) -> Result<Self, String> {
let records = read_records(input)?;
for record in &records {
validate_payload(record)?;
}
if records.first().map(|r| r.tag) != Some(RTAG_HEADER)
|| records.get(1).map(|r| r.tag) != Some(RTAG_BGNLIB)
|| records.last().map(|r| r.tag) != Some(RTAG_ENDLIB)
{
return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string());
}
let mut doc = Self {
records,
cells: BTreeMap::new(),
layer_names: BTreeMap::new(),
issues: Vec::new(),
diagnostics: Vec::new(),
header: Vec::new(),
global_issues: Vec::new(),
known_cells: BTreeSet::new(),
scope: Scope::default(),
present: true,
};
let mut i = 2;
let mut global_seen = BTreeSet::new();
let mut cells_started = false;
doc.header.push((RTAG_HEADER, doc.records[0].data));
while i + 1 < doc.records.len() {
let record = &doc.records[i];
if record.tag != RTAG_BGNSTR {
if cells_started {
return Err(error(
record.offset,
"library field outside the library header",
));
}
if record.tag != RTAG_MASK && !global_seen.insert(record.tag) {
return Err(error(record.offset, "duplicate library field"));
}
if !matches!(
record.tag,
RTAG_LIBNAME
| RTAG_UNITS
| RTAG_REFLIBS
| RTAG_FONTS
| RTAG_GENERATIONS
| RTAG_ATTRTABLE
| RTAG_FORMAT
| RTAG_MASK
| RTAG_ENDMASKS
| RTAG_LIBDIRSIZE
| RTAG_SRFNAME
| RTAG_LIBSECUR
) {
doc.global_issues.push(Diagnostic::at(
"unsupported_record",
record.offset,
format!("unsupported library record 0x{:04x}", record.tag),
));
}
doc.header.push((
record.tag,
if record.tag & 0xff == 6 {
unpad(record.data)
} else {
record.data
},
));
i += 1;
continue;
}
if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) {
return Err(error(record.offset, "LIBNAME and UNITS must precede cells"));
}
cells_started = true;
let offset = record.offset;
i += 1;
let name_record = doc
.records
.get(i)
.filter(|r| r.tag == RTAG_STRNAME)
.ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?;
let name = string(name_record.data, name_record.offset)?;
if name.is_empty() {
return Err(error(name_record.offset, "empty cell name"));
}
i += 1;
let mut cell = Cell {
offset,
end_offset: 0,
objects: Vec::new(),
attributes: Vec::new(),
issues: Vec::new(),
};
loop {
let record = doc
.records
.get(i)
.ok_or_else(|| error(offset, "missing ENDSTR"))?;
if record.tag == RTAG_ENDSTR {
cell.end_offset = record.offset;
i += 1;
break;
}
if is_element(record.tag) {
let start = i;
i += 1;
while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) {
if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR)
|| is_element(doc.records[i].tag)
{
return Err(error(doc.records[i].offset, "missing ENDEL"));
}
i += 1;
}
if i == doc.records.len() {
return Err(error(offset, "missing ENDEL"));
}
let mut issues = Vec::new();
let mut parsed = object(&doc.records[start..=i], &mut issues)?;
parsed.issues = issues;
cell.objects.push(parsed);
i += 1;
} else {
if matches!(record.tag, RTAG_ENDLIB | RTAG_BGNSTR) {
return Err(error(record.offset, "missing ENDSTR"));
}
if record.tag != RTAG_STRCLASS {
cell.issues.push(Diagnostic::at(
"unsupported_record",
record.offset,
format!("unsupported cell record 0x{:04x}", record.tag),
));
} else if !cell.objects.is_empty()
|| cell.attributes.iter().any(|(tag, _)| *tag == RTAG_STRCLASS)
{
return Err(error(
record.offset,
"STRCLASS must occur once before elements",
));
}
cell.attributes.push((record.tag, record.data));
i += 1;
}
}
cell.attributes.sort();
cell.objects.sort_by(|a, b| a.key.cmp(&b.key));
if doc.cells.insert(name.clone(), cell).is_some() {
return Err(error(offset, format!("duplicate cell name {name:?}")));
}
}
if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) {
return Err("library must contain LIBNAME and UNITS".to_string());
}
doc.header.sort();
doc.read_metadata()?;
doc.known_cells = doc.cells.keys().cloned().collect();
doc.refresh_issues();
Ok(doc)
}
fn read_metadata(&mut self) -> Result<(), String> {
for (cell_name, cell) in &mut self.cells {
for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) {
let text = object.field(RTAG_STRING).unwrap();
if text.starts_with(b"KLAMATH_LAYER_NAMES_") && text != LAYER_SENTINEL.as_bytes() {
// Layer names apply across the library. An unknown version
// may describe layers used outside this carrier or cell, so
// geometry scope selection cannot establish its relevance.
let mut diagnostic = Diagnostic::at(
"unsupported_metadata_version",
object.offset,
"unsupported layer-name metadata version".to_string(),
);
diagnostic.cell = Some(cell_name.clone());
diagnostic.layer = object.layer;
diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message);
self.global_issues.push(diagnostic);
}
}
let carriers: BTreeSet<_> = cell
.objects
.iter()
.filter(|o| {
o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes())
})
.filter_map(|o| o.layer)
.collect();
for object in &mut cell.objects {
if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l))
{
continue;
}
let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?;
if text != LAYER_SENTINEL {
let ((layer, dtype), name) =
parse_layer_name(&text).map_err(|message| error(object.offset, message))?;
if self
.layer_names
.get(&(layer, dtype))
.is_some_and(|old| old != &name)
{
return Err(error(
object.offset,
format!("conflicting names for {layer}/{dtype}"),
));
}
self.layer_names.insert((layer, dtype), name.to_string());
}
object.metadata = true;
}
}
Ok(())
}
fn check_graph(&mut self) {
let mut unresolved = BTreeMap::new();
for (parent, cell) in &self.cells {
for object in &cell.objects {
if let Some(target) = &object.target
&& !self.known_cells.contains(target)
{
unresolved.entry((parent, target)).or_insert(object.offset);
}
}
}
for ((parent, target), offset) in unresolved {
self.diagnostics.push(Diagnostic {
code: "unresolved_reference",
message: format!("cell {parent:?}: unresolved reference {target:?}"),
offset: Some(offset),
cell: Some(parent.clone()),
layer: None,
target: Some(target.clone()),
});
}
let mut state = BTreeMap::<&str, u8>::new();
for root in self.cells.keys() {
let mut stack = vec![(root.as_str(), false)];
while let Some((name, leaving)) = stack.pop() {
if leaving {
state.insert(name, 2);
continue;
}
match state.get(name) {
Some(2) => continue,
Some(1) => {
self.diagnostics.push(Diagnostic {
code: "hierarchy_cycle",
message: format!("cyclic hierarchy involving {name:?}"),
offset: self.cells.get(name).map(|c| c.offset),
cell: Some(name.to_string()),
layer: None,
target: None,
});
continue;
}
_ => (),
}
state.insert(name, 1);
stack.push((name, true));
if let Some(cell) = self.cells.get(name) {
for object in &cell.objects {
if let Some(target) = &object.target {
stack.push((target, false));
}
}
}
}
}
}
pub fn edges(&self) -> BTreeMap<(String, String), u64> {
let mut edges = BTreeMap::new();
for (name, cell) in &self.cells {
for object in &cell.objects {
if let Some(target) = &object.target {
let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64);
*count = count.saturating_add(object.multiplicity);
}
}
}
edges
}
pub fn canonical_dump(&self) -> String {
let mut out = format!(
"profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n",
self.scope.description()
);
for (tag, data) in &self.header {
writeln!(out, "{}={}", tag_name(*tag), format_data(*tag, data)).unwrap();
}
for ((layer, dtype), name) in &self.layer_names {
writeln!(out, "layer {layer}/{dtype} name={name:?}").unwrap();
}
for (name, cell) in &self.cells {
writeln!(out, "cell {name:?}").unwrap();
for (tag, data) in &cell.attributes {
writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap();
}
for object in cell.objects.iter().filter(|o| !o.metadata) {
writeln!(out, " {}", object.description()).unwrap();
}
}
for issue in &self.issues {
writeln!(out, "INCOMPLETE: {issue}").unwrap();
}
out
}
pub fn record_dump(&self) -> String {
format_records(&self.records)
}
}
/// Inspect record boundaries even when object grammar or metadata is invalid.
pub fn dump_records(input: &[u8]) -> Result<String, String> {
Ok(format_records(&read_records(input)?))
}
fn format_records(records: &[Record<'_>]) -> String {
let mut out = String::new();
for record in records {
writeln!(
out,
"{:010} {:04x} {} size={} {}",
record.offset,
record.tag,
tag_name(record.tag),
record.data.len() + 4,
format_data(
record.tag,
if record.tag & 0xff == 6 {
unpad(record.data)
} else {
record.data
}
)
)
.unwrap();
}
out
}
/// A retained comparison field. Polygon coordinates and string padding follow
/// the comparison profile; REAL encodings and unsupported payloads are exact.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct FieldValue {
pub tag: u16,
pub data: Vec<u8>,
}
fn owned_fields<'a>(fields: impl IntoIterator<Item = (u16, &'a [u8])>) -> Vec<FieldValue> {
fields
.into_iter()
.map(|(tag, data)| FieldValue {
tag,
data: data.to_vec(),
})
.collect()
}
#[derive(Debug)]
pub struct ObjectChange {
pub offset: usize,
pub end_offset: usize,
pub kind: u16,
pub layer: Option<Layer>,
pub bounds: Option<Bounds>,
pub target: Option<String>,
pub origin: Option<[i32; 2]>,
pub width: Option<i32>,
pub counts: Option<[u16; 2]>,
pub description: String,
pub fields: Vec<FieldValue>,
pub properties: Vec<(u16, Vec<u8>)>,
}
impl From<&Object<'_>> for ObjectChange {
fn from(object: &Object<'_>) -> Self {
Self {
offset: object.offset,
end_offset: object.end_offset,
kind: object.kind,
layer: object.layer,
bounds: object.bounds,
target: object.target.clone(),
origin: object.field(RTAG_XY).map(|data| {
[
i32::from_be_bytes(data[..4].try_into().unwrap()),
i32::from_be_bytes(data[4..8].try_into().unwrap()),
]
}),
width: object
.field(RTAG_WIDTH)
.map(|data| i32::from_be_bytes(data.try_into().unwrap())),
counts: object.target.as_ref().map(|_| {
if object.kind == RTAG_AREF {
let data = object.field(RTAG_COLROW).unwrap();
[integer(&data[..2]), integer(&data[2..])]
} else {
[1, 1]
}
}),
description: object.description(),
fields: owned_fields(
object
.key
.fields
.iter()
.map(|(tag, data)| (*tag, data.as_ref())),
),
properties: object
.key
.properties
.iter()
.map(|(a, v)| (*a, v.to_vec()))
.collect(),
}
}
}
#[derive(Debug)]
pub struct CellChange {
pub name: String,
pub old_offset: Option<usize>,
pub new_offset: Option<usize>,
pub attributes_changed: bool,
pub old_attributes: Vec<FieldValue>,
pub new_attributes: Vec<FieldValue>,
pub removed: Vec<ObjectChange>,
pub added: Vec<ObjectChange>,
pub replacements: Vec<Replacement>,
}
#[derive(Debug)]
pub struct LayerNameChange {
pub layer: Layer,
pub old: Option<String>,
pub new: Option<String>,
}
#[derive(Debug)]
pub struct LibraryFieldChange {
pub field: String,
pub tag: Option<u16>,
pub old_fields: Vec<FieldValue>,
pub new_fields: Vec<FieldValue>,
pub old: Vec<String>,
pub new: Vec<String>,
}
#[derive(Debug)]
pub struct Comparison {
pub scope: Scope,
/// Unchanged names are displayed once when objects on that layer change.
pub layer_legend: BTreeMap<Layer, String>,
pub library_fields: Vec<LibraryFieldChange>,
pub layer_names: Vec<LayerNameChange>,
pub cells: Vec<CellChange>,
pub issues: Vec<String>,
}
impl Comparison {
pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self {
let mut result = Self {
scope: old.scope.clone(),
layer_legend: old
.layer_names
.iter()
.filter(|(pair, name)| new.layer_names.get(pair) == Some(*name))
.map(|(pair, name)| (*pair, name.clone()))
.collect(),
library_fields: Vec::new(),
layer_names: Vec::new(),
cells: Vec::new(),
issues: Vec::new(),
};
if old.scope != new.scope {
result.issues.push("comparison scopes differ".to_string());
}
if old.present != new.present {
result.library_fields.push(LibraryFieldChange {
field: "presence".to_string(),
tag: None,
old_fields: Vec::new(),
new_fields: Vec::new(),
old: vec![if old.present { "present" } else { "absent" }.to_string()],
new: vec![if new.present { "present" } else { "absent" }.to_string()],
});
}
for (side, doc) in [("old", old), ("new", new)] {
result
.issues
.extend(doc.issues.iter().map(|i| format!("{side}: {i}")));
}
let header_tags: BTreeSet<_> = old
.header
.iter()
.chain(&new.header)
.map(|(tag, _)| *tag)
.collect();
for tag in header_tags {
let a: Vec<_> = old
.header
.iter()
.filter(|(t, _)| *t == tag)
.map(|(_, value)| *value)
.collect();
let b: Vec<_> = new
.header
.iter()
.filter(|(t, _)| *t == tag)
.map(|(_, value)| *value)
.collect();
if a != b {
result.library_fields.push(LibraryFieldChange {
field: tag_name(tag),
tag: Some(tag),
old_fields: owned_fields(a.iter().map(|data| (tag, *data))),
new_fields: owned_fields(b.iter().map(|data| (tag, *data))),
old: a.iter().map(|data| format_data(tag, data)).collect(),
new: b.iter().map(|data| format_data(tag, data)).collect(),
});
}
}
let layers: BTreeSet<_> = old
.layer_names
.keys()
.chain(new.layer_names.keys())
.copied()
.collect();
for layer in layers {
if old.layer_names.get(&layer) != new.layer_names.get(&layer) {
result.layer_names.push(LayerNameChange {
layer,
old: old.layer_names.get(&layer).cloned(),
new: new.layer_names.get(&layer).cloned(),
});
}
}
let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect();
for name in names {
let a = old.cells.get(name);
let b = new.cells.get(name);
let mut change = CellChange {
name: name.clone(),
old_offset: a.map(|c| c.offset),
new_offset: b.map(|c| c.offset),
attributes_changed: a.map(|c| &c.attributes) != b.map(|c| &c.attributes),
old_attributes: owned_fields(
a.into_iter().flat_map(|c| c.attributes.iter().copied()),
),
new_attributes: owned_fields(
b.into_iter().flat_map(|c| c.attributes.iter().copied()),
),
removed: Vec::new(),
added: Vec::new(),
replacements: Vec::new(),
};
let a: Vec<_> = a
.into_iter()
.flat_map(|c| &c.objects)
.filter(|o| !o.metadata)
.collect();
let b: Vec<_> = b
.into_iter()
.flat_map(|c| &c.objects)
.filter(|o| !o.metadata)
.collect();
let (mut i, mut j) = (0, 0);
while i < a.len() || j < b.len() {
let order = match (a.get(i), b.get(j)) {
(Some(a), Some(b)) => a.key.cmp(&b.key),
(Some(_), None) => std::cmp::Ordering::Less,
_ => std::cmp::Ordering::Greater,
};
match order {
std::cmp::Ordering::Equal => {
i += 1;
j += 1;
}
std::cmp::Ordering::Less => {
change.removed.push(a[i].into());
i += 1;
}
std::cmp::Ordering::Greater => {
change.added.push(b[j].into());
j += 1;
}
}
}
if change.old_offset.is_none()
|| change.new_offset.is_none()
|| change.attributes_changed
|| !change.removed.is_empty()
|| !change.added.is_empty()
{
change.replacements = changes::replacements(&change.removed, &change.added);
result.cells.push(change);
}
}
result
}
pub fn is_equal(&self) -> bool {
self.issues.is_empty()
&& self.library_fields.is_empty()
&& self.layer_names.is_empty()
&& self.cells.is_empty()
}
pub fn exit_code(&self) -> u8 {
if !self.issues.is_empty() {
2
} else if self.is_equal() {
0
} else {
1
}
}
/// Limit presentation only: the complete comparison has already been made.
pub fn text(&self, max_objects: usize) -> String {
self.text_filtered(max_objects, &DisplayFilter::default())
}
pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String {
let mut out = format!("profile {PROFILE}\n{}\n", self.scope.description());
if show.cell.is_some() || show.layer.is_some() {
writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap();
}
for change in &self.library_fields {
writeln!(
out,
"library {}: {:?} -> {:?}",
change.field, change.old, change.new
)
.unwrap();
}
if self.library_fields.iter().any(|c| c.field == "UNITS") {
out.push_str("UNITS changed; coordinates remain in each file's database units\n");
}
for change in &self.layer_names {
writeln!(
out,
"layer {}/{} name: {:?} -> {:?}",
change.layer.0, change.layer.1, change.old, change.new
)
.unwrap();
}
let changed_layers: BTreeSet<_> = self
.cells
.iter()
.filter(|cell| show.cell.as_ref().is_none_or(|name| name == &cell.name))
.flat_map(|cell| cell.removed.iter().chain(&cell.added))
.filter_map(|object| object.layer)
.filter(|layer| show.layer.is_none_or(|selected| selected == *layer))
.collect();
for layer in changed_layers {
if let Some(name) = self.layer_legend.get(&layer) {
writeln!(out, "layer {}/{} name={name:?}", layer.0, layer.1).unwrap();
}
}
let total: usize = self
.cells
.iter()
.map(|c| c.removed.len() + c.added.len())
.sum();
let mut shown: usize = 0;
for cell in &self.cells {
if show.cell.as_ref().is_some_and(|name| name != &cell.name) {
continue;
}
writeln!(
out,
"cell {:?} old@{:?} new@{:?}{}",
cell.name,
cell.old_offset,
cell.new_offset,
if cell.attributes_changed {
" [cell/header changed]"
} else {
""
}
)
.unwrap();
let mut paired_old = BTreeSet::new();
let mut paired_new = BTreeSet::new();
for replacement in &cell.replacements {
let a = &cell.removed[replacement.removed];
let b = &cell.added[replacement.added];
if shown.saturating_add(2) <= max_objects
&& show
.layer
.is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer))
{
writeln!(out, " ~ {}", replacement.explanation).unwrap();
writeln!(
out,
" - @{} bbox={:?} {}",
a.offset, a.bounds, a.description
)
.unwrap();
writeln!(
out,
" + @{} bbox={:?} {}",
b.offset, b.bounds, b.description
)
.unwrap();
paired_old.insert(replacement.removed);
paired_new.insert(replacement.added);
shown += 2;
}
}
for (sign, objects, paired) in [
("-", &cell.removed, &paired_old),
("+", &cell.added, &paired_new),
] {
for (index, object) in objects.iter().enumerate() {
if paired.contains(&index) {
continue;
}
if show.layer.is_some_and(|layer| object.layer != Some(layer)) {
continue;
}
if shown < max_objects {
writeln!(
out,
" {sign} @{} bbox={:?} {}",
object.offset, object.bounds, object.description
)
.unwrap();
shown += 1;
}
}
}
}
writeln!(
out,
"{} changed cells; {shown} of {total} object changes shown; {} layer-name changes",
self.cells.len(),
self.layer_names.len()
)
.unwrap();
for issue in &self.issues {
writeln!(out, "INCOMPLETE: {issue}").unwrap();
}
if self.is_equal() {
out.push_str("equal under this profile\n");
}
out
}
}
/// The graph joins both revisions, retaining removed branches and old callers.
pub fn hierarchy(
old: &Document<'_>,
new: &Document<'_>,
diff: Option<&Comparison>,
mermaid: bool,
max_depth: usize,
) -> String {
let old_edges = old.edges();
let new_edges = new.edges();
let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect();
let direct: BTreeSet<_> = diff
.into_iter()
.flat_map(|d| d.cells.iter().map(|c| c.name.clone()))
.collect();
let changed_refs: BTreeSet<_> = diff
.into_iter()
.flat_map(|d| &d.cells)
.flat_map(|c| {
c.removed
.iter()
.chain(&c.added)
.filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone())))
})
.collect();
let mut parents = BTreeMap::<&str, Vec<&str>>::new();
for (parent, child) in &edges {
parents.entry(child).or_default().push(parent);
}
let mut affected = direct.clone();
let mut pending: Vec<_> = direct.iter().cloned().collect();
while let Some(child) = pending.pop() {
for parent in parents.get(child.as_str()).into_iter().flatten() {
if affected.insert((*parent).to_string()) {
pending.push((*parent).to_string());
}
}
}
let visible: BTreeSet<_> = if diff.is_some() {
affected
.iter()
.cloned()
.chain(changed_refs.iter().map(|(_, t)| t.clone()))
.collect()
} else {
old.cells
.keys()
.chain(new.cells.keys())
.cloned()
.chain(edges.iter().map(|(_, t)| t.clone()))
.collect()
};
let mut children = BTreeMap::<String, Vec<String>>::new();
let mut targets = BTreeSet::new();
for (parent, target) in &edges {
if visible.contains(parent) && visible.contains(target) {
children
.entry(parent.clone())
.or_default()
.push(target.clone());
targets.insert(target.clone());
}
}
let roots: Vec<_> = visible.difference(&targets).cloned().collect();
// Distinguish rootless cycles from descendants hidden by the depth limit.
let mut reachable = BTreeSet::new();
let mut pending = roots.clone();
while let Some(name) = pending.pop() {
if reachable.insert(name.clone()) {
pending.extend(children.get(&name).into_iter().flatten().cloned());
}
}
let mut included = BTreeSet::new();
let mut rows = Vec::new();
let mut stack: Vec<_> = roots
.iter()
.rev()
.map(|n| (n.clone(), 0, None::<String>))
.collect();
while let Some((name, depth, parent)) = stack.pop() {
let seen = !included.insert(name.clone());
let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]);
rows.push((
name.clone(),
depth,
parent,
seen,
depth >= max_depth && !descendants.is_empty(),
));
if !seen && depth < max_depth {
for child in descendants.iter().rev() {
stack.push((child.clone(), depth + 1, Some(name.clone())));
}
}
}
for name in visible.difference(&reachable) {
rows.push((name.clone(), 0, None, false, true));
}
let status = |name: &String| {
if !old.cells.contains_key(name) && !new.cells.contains_key(name) {
if old.known_cells.contains(name) || new.known_cells.contains(name) {
" [outside scope]"
} else {
" [unresolved]"
}
} else if direct.contains(name) {
" [cell changed]"
} else if affected.contains(name) {
" [affected]"
} else if diff.is_some() {
" [cell unchanged]"
} else {
""
}
};
let edge_label = |parent: &String, target: &String| {
let key = (parent.clone(), target.clone());
let a = old_edges.get(&key).copied().unwrap_or(0);
let b = new_edges.get(&key).copied().unwrap_or(0);
let count = if a == b {
format!("x{b}")
} else {
format!("x{a} -> x{b}")
};
format!(
"{count}{}",
if changed_refs.contains(&key) {
" [reference changed]"
} else {
""
}
)
};
if mermaid {
let ids: BTreeMap<_, _> = rows
.iter()
.map(|r| &r.0)
.collect::<BTreeSet<_>>()
.into_iter()
.enumerate()
.map(|(i, n)| (n, i))
.collect();
let mut out = format!("flowchart TD\n %% {}\n", new.scope.description());
for (name, id) in &ids {
let label = format!("{name}{}", status(name));
let label = label
.replace('&', "&amp;")
.replace('"', "&quot;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace(['\r', '\n'], " ");
writeln!(out, " n{id}[\"{label}\"]").unwrap();
}
for (parent, target) in &edges {
if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) {
writeln!(out, " n{a} -->|\"{}\"| n{b}", edge_label(parent, target)).unwrap();
}
}
if rows.is_empty() {
out.push_str(" empty[\"No changed hierarchy\"]\n");
}
out
} else {
let mut out = format!("{}\n", new.scope.description());
for (name, depth, parent, seen, limited) in rows {
let count = parent
.map(|p| edge_label(&p, &name))
.unwrap_or_else(|| "x1".to_string());
writeln!(
out,
"{}{name:?} {count}{}{}{}",
" ".repeat(depth),
status(&name),
if seen { " [shown above]" } else { "" },
if limited { " [children omitted]" } else { "" }
)
.unwrap();
}
if visible.is_empty() {
out.push_str("No changed hierarchy\n");
}
out
}
}
fn quoted(data: &[u8]) -> String {
let mut out = "\"".to_string();
for byte in data {
for escaped in std::ascii::escape_default(*byte) {
out.push(escaped as char);
}
}
out.push('"');
out
}
fn format_data(tag: u16, data: &[u8]) -> String {
if matches!(
tag,
RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_BOXTYPE | RTAG_NODETYPE
) && data.len() == 2
{
return integer(data).to_string();
}
match tag & 0xff {
0 if data.is_empty() => String::new(),
2 if data.len().is_multiple_of(2) => format!(
"{:?}",
data.as_chunks::<2>()
.0
.iter()
.copied()
.map(i16::from_be_bytes)
.collect::<Vec<_>>()
),
3 if data.len().is_multiple_of(4) => format!(
"{:?}",
data.as_chunks::<4>()
.0
.iter()
.copied()
.map(i32::from_be_bytes)
.collect::<Vec<_>>()
),
5 if data.len().is_multiple_of(8) => format!(
"{:?} raw={}",
data.as_chunks::<8>()
.0
.iter()
.map(|&b| decode_real8(u64::from_be_bytes(b)))
.collect::<Vec<_>>(),
hex(data)
),
6 => quoted(data),
_ => hex(data),
}
}
fn hex(data: &[u8]) -> String {
let mut out = "0x".to_string();
for byte in data {
write!(out, "{byte:02x}").unwrap();
}
out
}
fn kind_name(tag: u16) -> &'static str {
match tag {
RTAG_BOUNDARY => "BOUNDARY",
RTAG_PATH => "PATH",
RTAG_SREF => "SREF",
RTAG_AREF => "AREF",
RTAG_TEXT => "TEXT",
RTAG_NODE => "NODE",
RTAG_BOX => "BOX",
_ => "UNKNOWN",
}
}
fn tag_name(tag: u16) -> String {
let name = match tag {
RTAG_HEADER => "HEADER",
RTAG_BGNLIB => "BGNLIB",
RTAG_LIBNAME => "LIBNAME",
RTAG_UNITS => "UNITS",
RTAG_ENDLIB => "ENDLIB",
RTAG_BGNSTR => "BGNSTR",
RTAG_STRNAME => "STRNAME",
RTAG_ENDSTR => "ENDSTR",
RTAG_ENDEL => "ENDEL",
RTAG_LAYER => "layer",
RTAG_DATATYPE => "datatype",
RTAG_TEXTTYPE => "texttype",
RTAG_XY => "xy",
RTAG_SNAME => "target",
RTAG_COLROW => "counts",
RTAG_STRANS => "strans",
RTAG_MAG => "mag",
RTAG_ANGLE => "angle",
RTAG_WIDTH => "width",
RTAG_PATHTYPE => "pathtype",
RTAG_STRING => "string",
RTAG_PROPATTR => "PROPATTR",
RTAG_PROPVALUE => "PROPVALUE",
RTAG_PRESENTATION => "presentation",
RTAG_BGNEXTN => "bgnextn",
RTAG_ENDEXTN => "endextn",
RTAG_BOXTYPE => "boxtype",
RTAG_NODETYPE => "nodetype",
RTAG_ELFLAGS => "elflags",
RTAG_PLEX => "plex",
RTAG_STRCLASS => "STRCLASS",
tag if is_element(tag) => kind_name(tag),
_ => return format!("record_{tag:04x}"),
};
name.to_string()
}