Simplify and speed up GDSII diffs

Summarize added and removed cells, group references, and keep replacement pairs together at text display limits. Trim hierarchy labels and summarize long unmatched paths.

Stream JSON object evidence and reduce parsing and comparison allocations. Parse large revisions concurrently and adaptively reuse matching cell bodies when most of the layout is shared.
This commit is contained in:
Jan Petykiewicz 2026-11-18 10:58:19 -08:00
commit 8efeb04aab

View file

@ -14,10 +14,13 @@ use std::{
pub mod changes;
pub mod names;
mod pair;
pub mod report;
pub mod selection;
use changes::Replacement;
use pair::CellReuse;
pub use pair::RevisionPair;
use selection::{DisplayFilter, Scope};
/// Input revision associated with a diagnostic.
@ -139,7 +142,7 @@ struct Key<'a> {
properties: Vec<(u16, &'a [u8])>,
}
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct Object<'a> {
pub offset: usize,
pub end_offset: usize,
@ -231,6 +234,8 @@ fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> {
])
}
const POINTS_PER_LINE: usize = 4;
fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result {
let mut points = XY::points(data);
if points.len() == 1 {
@ -240,7 +245,11 @@ fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result {
out.write_str(" xy=[")?;
for (index, [x, y]) in points.enumerate() {
if index != 0 {
out.write_str(if index % 4 == 0 { ",\n " } else { ", " })?;
out.write_str(if index % POINTS_PER_LINE == 0 {
",\n "
} else {
", "
})?;
}
write!(out, "({x}, {y})")?;
}
@ -312,11 +321,10 @@ fn coordinate_bounds(points: impl IntoIterator<Item = [i32; 2]>) -> Bounds {
}
// Booth's least-rotation algorithm avoids quadratic work on repeated vertices.
fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> {
let n = points.len();
fn least_rotation(n: usize, point: impl Fn(usize) -> [i32; 2]) -> usize {
let (mut i, mut j, mut k) = (0, 1, 0);
while i < n && j < n && k < n {
match points[(i + k) % n].cmp(&points[(j + k) % n]) {
match point((i + k) % n).cmp(&point((j + k) % n)) {
std::cmp::Ordering::Equal => k += 1,
std::cmp::Ordering::Greater => {
i += k + 1;
@ -334,21 +342,37 @@ fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> {
}
}
}
let start = i.min(j);
(0..n).map(|k| points[(start + k) % n]).collect()
i.min(j)
}
fn canonical_ring(mut ring: Vec<[i32; 2]>) -> Vec<u8> {
ring.pop(); // The caller has checked closure.
let forward = least_rotation(&ring);
ring.reverse();
let backward = least_rotation(&ring);
let mut ring = forward.min(backward);
ring.push(ring[0]);
ring.into_iter()
.flatten()
.flat_map(i32::to_be_bytes)
.collect()
fn canonical_ring(data: &[u8]) -> Option<Cow<'_, [u8]>> {
let (points, _) = data.as_chunks::<8>();
let (last, ring) = points.split_last()?;
if ring.len() < 3 || ring.first() != Some(last) {
return None;
}
let n = ring.len();
let first = least_rotation(n, |index| XY::point(ring[index]));
let split = n - least_rotation(n, |index| XY::point(ring[n - 1 - index]));
let forward = ring[first..].iter().chain(&ring[..first]);
let backward = ring[..split].iter().rev().chain(ring[split..].iter().rev());
let decode = |point: &[u8; 8]| XY::point(*point);
let use_forward = forward
.clone()
.map(decode)
.cmp(backward.clone().map(decode))
.is_le();
if use_forward && first == 0 {
return Some(Cow::Borrowed(data));
}
let mut result = Vec::with_capacity(data.len());
if use_forward {
result.extend(forward.flatten().copied());
} else {
result.extend(backward.flatten().copied());
}
result.extend_from_within(..8);
Some(Cow::Owned(result))
}
fn validate_payload(record: &Record<'_>) -> Result<(), String> {
@ -474,14 +498,12 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
data = Cow::Borrowed(unpad_ascii(record.data));
}
if record.tag == RTAG_XY && is_polygon(first.tag) {
let p: Vec<_> = XY::points(record.data).collect();
if p.len() < 4 || p.first() != p.last() {
return Err(error(
data = canonical_ring(record.data).ok_or_else(|| {
error(
record.offset,
"polygon must have at least three vertices and be closed",
));
}
data = Cow::Owned(canonical_ring(p));
)
})?;
}
fields.push((record.tag, data));
i += 1;
@ -556,6 +578,13 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
}
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
read_records_with(input, |_, _| {})
}
fn read_records_with<'a>(
input: &'a [u8],
mut observe: impl FnMut(usize, &Record<'a>),
) -> Result<Vec<Record<'a>>, String> {
if input.starts_with(b"\x1f\x8b") {
return Err(error(
0,
@ -574,11 +603,13 @@ fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
ErrType::Incomplete(_) => error(offset, "truncated record header"),
ErrType::Failed(message) => error(offset, message),
})?;
records.push(Record {
let record = Record {
offset,
tag: frame.tag,
data: frame.data,
});
};
observe(records.len(), &record);
records.push(record);
remaining = next;
if frame.tag == RTAG_ENDLIB {
if remaining.iter().any(|b| *b != 0) {
@ -596,7 +627,16 @@ fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
impl<'a> Document<'a> {
pub fn parse(input: &'a [u8]) -> Result<Self, String> {
let records = read_records(input)?;
for record in &records {
let mut doc = Self::parse_records(&records, None)?;
doc.records = records;
Ok(doc)
}
fn parse_records(
records: &[Record<'a>],
reuse: Option<CellReuse<'a, '_>>,
) -> Result<Self, String> {
for record in records {
validate_payload(record)?;
}
if records.first().map(|r| r.tag) != Some(RTAG_HEADER)
@ -606,7 +646,7 @@ impl<'a> Document<'a> {
return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string());
}
let mut doc = Self {
records,
records: Vec::new(),
cells: BTreeMap::new(),
layer_names: BTreeMap::new(),
diagnostics: Vec::new(),
@ -619,9 +659,9 @@ impl<'a> Document<'a> {
let mut i = 2;
let mut global_seen = BTreeSet::new();
let mut cells_started = false;
doc.header.push((RTAG_HEADER, doc.records[0].data));
while i + 1 < doc.records.len() {
let record = &doc.records[i];
doc.header.push((RTAG_HEADER, records[0].data));
while i + 1 < records.len() {
let record = &records[i];
if record.tag != RTAG_BGNSTR {
if cells_started {
return Err(error(
@ -674,8 +714,7 @@ impl<'a> Document<'a> {
cells_started = true;
let offset = record.offset;
i += 1;
let name_record = doc
.records
let name_record = records
.get(i)
.filter(|r| r.tag == RTAG_STRNAME)
.ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?;
@ -685,6 +724,15 @@ impl<'a> Document<'a> {
if name.is_empty() {
return Err(error(name_record.offset, "empty cell name"));
}
if let Some(reuse) = &reuse
&& let Some((cell, end)) = reuse.cell(&name, offset, records)
{
if doc.cells.insert(name.clone(), cell).is_some() {
return Err(error(offset, format!("duplicate cell name {name:?}")));
}
i = end + 1;
continue;
}
i += 1;
let mut cell = Cell {
offset,
@ -694,8 +742,7 @@ impl<'a> Document<'a> {
issues: Vec::new(),
};
loop {
let record = doc
.records
let record = records
.get(i)
.ok_or_else(|| error(offset, "missing ENDSTR"))?;
if record.tag == RTAG_ENDSTR {
@ -706,19 +753,19 @@ impl<'a> Document<'a> {
if is_element(record.tag) {
let start = i;
i += 1;
while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) {
if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR)
|| is_element(doc.records[i].tag)
while records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) {
if matches!(records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR)
|| is_element(records[i].tag)
{
return Err(error(doc.records[i].offset, "missing ENDEL"));
return Err(error(records[i].offset, "missing ENDEL"));
}
i += 1;
}
if i == doc.records.len() {
if i == records.len() {
return Err(error(offset, "missing ENDEL"));
}
let mut issues = Vec::new();
let mut parsed = object(&doc.records[start..=i], &mut issues)?;
let mut parsed = object(&records[start..=i], &mut issues)?;
parsed.issues = issues;
cell.objects.push(parsed);
i += 1;
@ -820,16 +867,21 @@ impl<'a> Document<'a> {
}
pub fn edges(&self) -> BTreeMap<(String, String), u64> {
let mut edges = BTreeMap::new();
for (name, cell) in &self.cells {
for object in &cell.objects {
if let Some(target) = &object.target {
let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64);
*count = count.saturating_add(object.multiplicity);
self.cells
.iter()
.flat_map(|(name, cell)| {
let mut counts = BTreeMap::new();
for object in &cell.objects {
if let Some(target) = object.target.as_deref() {
let count = counts.entry(target).or_insert(0_u64);
*count = count.saturating_add(object.multiplicity);
}
}
}
}
edges
counts
.into_iter()
.map(|(target, count)| ((name.clone(), target.to_owned()), count))
})
.collect()
}
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
@ -1150,8 +1202,62 @@ pub struct Comparison {
pub diagnostics: Vec<Diagnostic>,
}
/// A comparison for text and hierarchy reports. Layered objects in whole-cell
/// changes contribute to the cell's object count.
#[derive(Debug)]
pub struct TextComparison {
comparison: Comparison,
whole_cell_objects: BTreeMap<String, usize>,
}
impl TextComparison {
pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self {
Self::collect(old, new, &BTreeSet::new())
}
fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self {
let mut whole_cell_objects = BTreeMap::new();
let comparison = Comparison::collect(old, new, reused_cells, Some(&mut whole_cell_objects));
Self {
comparison,
whole_cell_objects,
}
}
pub fn render_text(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
) -> String {
self.comparison.render_text_with_counts(
max_objects,
show,
show_library_presence,
&self.whole_cell_objects,
)
}
pub fn outcome(&self) -> Outcome {
self.comparison.outcome()
}
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
self.comparison.hierarchy(mermaid, max_depth)
}
}
impl Comparison {
pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self {
Self::collect(old, new, &BTreeSet::new(), None)
}
fn collect(
old: &Document<'_>,
new: &Document<'_>,
reused_cells: &BTreeSet<&str>,
mut whole_cell_objects: Option<&mut BTreeMap<String, usize>>,
) -> Self {
let mut result = Self {
old: Revision::from_document(old),
new: Revision::from_document(new),
@ -1231,6 +1337,9 @@ impl Comparison {
}
let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect();
for name in names {
if reused_cells.contains(name.as_str()) {
continue;
}
let a = old.cells.get(name);
let b = new.cells.get(name);
let mut change = CellChange {
@ -1241,6 +1350,30 @@ impl Comparison {
added: Vec::new(),
replacements: Vec::new(),
};
if let Some(counts) = whole_cell_objects.as_deref_mut()
&& (a.is_none() || b.is_none())
{
let details = if a.is_some() {
&mut change.removed
} else {
&mut change.added
};
let mut count = 0;
for object in a.or(b).into_iter().flat_map(|cell| &cell.objects) {
if object.metadata {
continue;
}
count += 1;
if object.layer.is_none() || object.target.is_some() {
details.push(object.into());
}
}
if count != details.len() {
counts.insert(name.clone(), count);
}
result.cells.push(change);
continue;
}
let a: Vec<_> = a
.into_iter()
.flat_map(|c| &c.objects)
@ -1295,6 +1428,7 @@ impl Comparison {
}
}
/// Render the affected hierarchy. ASCII output is empty when no cells changed.
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth)
}
@ -1305,10 +1439,35 @@ impl Comparison {
}
pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String {
let (removed, added) = self.cells.iter().fold((0, 0), |(removed, added), cell| {
(removed + cell.removed.len(), added + cell.added.len())
});
let total = removed + added;
self.render_text(max_objects, show, true)
}
/// Render a text report. Set `show_library_presence` to false when a file
/// header already announces the library addition or deletion.
pub fn render_text(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
) -> String {
self.render_text_with_counts(max_objects, show, show_library_presence, &BTreeMap::new())
}
fn render_text_with_counts(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
whole_cell_objects: &BTreeMap<String, usize>,
) -> String {
let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize);
for cell in &self.cells {
match (&cell.old, &cell.new) {
(None, Some(_)) => added += 1,
(Some(_), None) => removed += 1,
_ => modified += 1,
}
}
let mut out = match self.outcome() {
Outcome::Equal => "Equal",
Outcome::Different => "Different",
@ -1316,19 +1475,25 @@ impl Comparison {
Outcome::Failed => "Failed",
}
.to_string();
if !self.cells.is_empty() {
let cells = if self.cells.len() == 1 {
"cell"
} else {
"cells"
};
write!(out, " · {} changed {cells}", self.cells.len()).unwrap();
}
if total != 0 {
write!(out, " · {removed} removed, {added} added").unwrap();
let mut cell_counts = [
(modified, "modified"),
(added, "added"),
(removed, "removed"),
]
.into_iter()
.filter(|(count, _)| *count != 0);
if let Some((count, action)) = cell_counts.next() {
let noun = if count == 1 { "cell" } else { "cells" };
write!(out, " · {count} {noun} {action}").unwrap();
for (count, action) in cell_counts {
write!(out, ", {count} {action}").unwrap();
}
}
let library_changes = self.library_changes.iter().filter(|change| {
show_library_presence || !matches!(change, LibraryChange::Presence { .. })
});
for (count, label) in [
(self.library_changes.len(), "library change"),
(library_changes.clone().count(), "library change"),
(self.layer_names.len(), "layer name change"),
] {
if count != 0 {
@ -1349,7 +1514,7 @@ impl Comparison {
out.push('\n');
}
let notice_start = out.len();
for change in &self.library_changes {
for change in library_changes {
writeln!(out, "library {change}").unwrap();
}
if self.library_changes.iter().any(|change| {
@ -1379,22 +1544,38 @@ impl Comparison {
}
let body_start = out.len();
let mut shown_layers = BTreeSet::new();
let mut detail_total = 0;
let mut selected = 0;
let mut shown: usize = 0;
let mut hidden_cell_changes = 0;
let visible =
|object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer));
let presence = |action, count| {
let objects = if count == 1 { "object" } else { "objects" };
Cow::Owned(format!(" ({action}, {count} {objects})"))
};
for cell in &self.cells {
let object_count = |details| {
whole_cell_objects
.get(&cell.name)
.copied()
.unwrap_or(details)
};
let note = match (&cell.old, &cell.new) {
(None, Some(_)) => presence("added", cell.added.len()),
(Some(_), None) => presence("removed", cell.removed.len()),
(None, Some(_)) => presence("added", object_count(cell.added.len())),
(Some(_), None) => presence("removed", object_count(cell.removed.len())),
_ if cell.attributes_changed() => Cow::Borrowed(" (attributes changed)"),
_ => Cow::Borrowed(""),
};
let whole_cell = cell.old.is_none() || cell.new.is_none();
// The heading accounts for layered objects in a whole-cell change.
detail_total += if whole_cell {
cell.removed
.iter()
.chain(&cell.added)
.filter(|object| object.layer.is_none())
.count()
} else {
cell.removed.len() + cell.added.len()
};
if show.cell.as_ref().is_some_and(|name| name != &cell.name) {
hidden_cell_changes += usize::from(!note.is_empty());
continue;
@ -1402,11 +1583,16 @@ impl Comparison {
let cell_start = out.len();
let previously_shown = shown;
writeln!(out, "cell {:?}{note}", cell.name).unwrap();
let whole_cell = cell.old.is_none() || cell.new.is_none();
let visible = |object: &ObjectChange| {
(!whole_cell || object.layer.is_none())
&& show.layer.is_none_or(|layer| object.layer == Some(layer))
};
let mut polygons = changes::PolygonChanges::default();
let mut paired_old = BTreeSet::new();
let mut paired_new = BTreeSet::new();
for replacement in &cell.replacements {
paired_old.insert(replacement.removed);
paired_new.insert(replacement.added);
let a = &cell.removed[replacement.removed];
let b = &cell.added[replacement.added];
if max_objects - shown >= 2 && visible(a) && visible(b) {
@ -1416,8 +1602,6 @@ impl Comparison {
writeln!(out, " ~ {}", changes::replacement_summary(a, b)).unwrap();
}
shown_layers.extend(a.layer);
paired_old.insert(replacement.removed);
paired_new.insert(replacement.added);
shown += 2;
}
}
@ -1426,7 +1610,7 @@ impl Comparison {
(false, &cell.added, &paired_new),
] {
let sign = if removed { "-" } else { "+" };
let mut references = BTreeMap::<&str, changes::ReferenceCounts>::new();
let mut references = BTreeMap::new();
for (index, object) in objects.iter().enumerate() {
if !visible(object) {
continue;
@ -1435,17 +1619,24 @@ impl Comparison {
if !paired.contains(&index) && shown < max_objects {
if is_polygon(object.kind) {
polygons.unmatched(object, removed);
} else if whole_cell && let Some(target) = object.target.as_deref() {
references.entry(target).or_default().add(object);
} else if let Some(target) = object.target.as_deref() {
let (_, counts) = references
.entry(target)
.or_insert_with(|| (object, changes::ReferenceCounts::default()));
counts.add(object);
} else {
writeln!(out, " {sign} {object}").unwrap();
changes::write_unmatched(&mut out, sign, object);
}
shown_layers.extend(object.layer);
shown += 1;
}
}
for (target, counts) in references {
counts.write(&mut out, sign, target);
for (target, (first, counts)) in references {
if whole_cell || counts.len() > 1 {
counts.write(&mut out, sign, target);
} else {
writeln!(out, " {sign} {first}").unwrap();
}
}
}
polygons.write(&mut out);
@ -1460,31 +1651,23 @@ impl Comparison {
}
}
out.insert_str(body_start, &legend);
let mut notice = String::new();
if shown < total {
let mut hidden = Vec::new();
if total > selected {
hidden.push(format!("{} filtered out", total - selected));
let mut omissions = Vec::new();
for (count, label, reason) in [
(detail_total - selected, "object change", "filtered out"),
(selected - shown, "object change", "over the limit"),
(hidden_cell_changes, "cell-level change", "filtered out"),
] {
if count != 0 {
let plural = if count == 1 { "" } else { "s" };
omissions.push(format!("{count} {label}{plural} {reason}"));
}
if selected > shown {
hidden.push(format!("{} over the limit", selected - shown));
}
writeln!(
notice,
"Showing {shown} of {total} object changes ({})",
hidden.join(", ")
)
.unwrap();
}
if hidden_cell_changes != 0 {
let plural = if hidden_cell_changes == 1 { "" } else { "s" };
writeln!(
notice,
"{hidden_cell_changes} cell-level change{plural} filtered out"
)
.unwrap();
if !omissions.is_empty() {
out.insert_str(
notice_start,
&format!("Omitted · {}\n", omissions.join(" · ")),
);
}
out.insert_str(notice_start, &notice);
// General comparison errors precede revision-specific errors in text.
if !self.diagnostics.is_empty() {
out.push_str("Diagnostics\n");
@ -1563,6 +1746,10 @@ fn render_hierarchy(
mermaid: bool,
max_depth: usize,
) -> String {
if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) {
return String::new();
}
let HierarchyAnalysis {
old_edges,
new_edges,
@ -1639,10 +1826,18 @@ fn render_hierarchy(
" [unresolved]"
}
} else if direct.contains(name) {
" [cell changed]"
if mermaid {
" [cell changed]"
} else if !old.known_cells.contains(name) {
" [added]"
} else if !new.known_cells.contains(name) {
" [removed]"
} else {
" [changed]"
}
} else if affected.contains(name) {
" [affected]"
} else if changes.is_some() {
} else if mermaid && changes.is_some() {
" [cell unchanged]"
} else {
""
@ -1659,7 +1854,7 @@ fn render_hierarchy(
};
format!(
"{count}{}",
if changed_refs.contains(&key) {
if (mermaid || a == b) && changed_refs.contains(&key) {
" [reference changed]"
} else {
""