Simplify and speed up GDSII diffs

Summarize added and removed cells, group references, and keep replacement pairs together at text display limits. Trim hierarchy labels and summarize long unmatched paths.

Stream JSON object evidence and reduce parsing and comparison allocations. Parse large revisions concurrently and adaptively reuse matching cell bodies when most of the layout is shared.
This commit is contained in:
Jan Petykiewicz 2026-11-18 10:58:19 -08:00
commit 8efeb04aab
3 changed files with 1137 additions and 261 deletions

View file

@ -28,6 +28,9 @@ fn group(object: &ObjectChange, bounds: bool) -> Group {
/// Pair only unique candidates in the remaining multisets. Ambiguous groups
/// remain additions/removals; no identity or geometric equivalence is inferred.
pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec<Replacement> {
if old.is_empty() || new.is_empty() {
return Vec::new();
}
let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new());
let mut result = Vec::new();
for bounds in [true, false] {
@ -101,7 +104,7 @@ fn object_heading(object: &ObjectChange, kind: &str) -> String {
}
}
fn polygon_xy(object: &ObjectChange) -> &[u8] {
fn object_xy(object: &ObjectChange) -> &[u8] {
object
.fields
.iter()
@ -109,6 +112,27 @@ fn polygon_xy(object: &ObjectChange) -> &[u8] {
.map_or(&[], |field| field.data.as_slice())
}
pub(super) fn write_unmatched(out: &mut String, sign: &str, object: &ObjectChange) {
if object.kind == RTAG_PATH
&& let Some([xmin, ymin, xmax, ymax]) = object.bounds
{
let vertices = XY::points(object_xy(object)).len();
if vertices > POINTS_PER_LINE {
let heading = object_heading(object, "path");
let width = object
.width
.map_or_else(|| "absent".to_string(), |width| format!("{width} dbu"));
writeln!(
out,
" {sign} {heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu"
)
.unwrap();
return;
}
}
writeln!(out, " {sign} {object}").unwrap();
}
fn vertex_count(xy: &[u8]) -> usize {
XY::points(xy).len().saturating_sub(1)
}
@ -147,7 +171,7 @@ struct ShapeCounts {
impl ShapeCounts {
fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) {
let is_rectangle =
|object: &ObjectChange| XY::closed_rectangle(polygon_xy(object)).is_some();
|object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some();
match object.kind {
RTAG_BOX => self.boxes += 1,
_ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => {
@ -173,7 +197,7 @@ impl PolygonChanges {
pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) {
let counts = self.0.entry(old.layer).or_default();
let moved = old.properties == new.properties
&& translation(polygon_xy(old), polygon_xy(new)).is_some()
&& translation(object_xy(old), object_xy(new)).is_some()
&& field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY);
let shapes = if moved {
&mut counts.moved
@ -237,6 +261,10 @@ pub(super) struct ReferenceCounts {
}
impl ReferenceCounts {
pub fn len(&self) -> usize {
self.singles + self.arrays
}
pub fn add(&mut self, object: &ObjectChange) {
if object.kind == RTAG_AREF {
self.arrays += 1;
@ -272,8 +300,8 @@ impl ReferenceCounts {
}
fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String {
let before = polygon_xy(old);
let after = polygon_xy(new);
let before = object_xy(old);
let after = object_xy(new);
let old_size = rectangle_size(before);
let new_size = rectangle_size(after);
let mut out = polygon_heading(old, old_size.is_some() && new_size.is_some());

218
src/review/pair.rs Normal file
View file

@ -0,0 +1,218 @@
//! Adaptive parsing for unscoped comparisons of two revisions.
use super::*;
use std::{sync::Mutex, thread};
struct CellSpan<'a> {
body: &'a [u8],
end_record: usize,
}
struct FramedRevision<'a> {
records: Vec<Record<'a>>,
spans: Option<BTreeMap<&'a str, CellSpan<'a>>>,
}
impl<'a> FramedRevision<'a> {
fn read(input: &'a [u8]) -> Result<Self, String> {
let mut markers = Vec::new();
let records = read_records_with(input, |index, record| {
if matches!(record.tag, RTAG_BGNSTR | RTAG_ENDSTR) {
markers.push(index);
}
})?;
let spans = index_cells(input, &records, markers);
Ok(Self { records, spans })
}
}
// Failure to index a layout leaves error reporting to the regular parser.
fn index_cells<'a>(
input: &'a [u8],
records: &[Record<'a>],
markers: Vec<usize>,
) -> Option<BTreeMap<&'a str, CellSpan<'a>>> {
let mut cells = BTreeMap::new();
let mut start = None;
for index in markers {
let record = &records[index];
match record.tag {
RTAG_BGNSTR => {
if start.is_some() {
return None;
}
let name_record = records.get(index + 1).filter(|r| r.tag == RTAG_STRNAME)?;
let name = decode_name(name_record.data)
.ok()
.filter(|name| !name.is_empty())?;
start = Some((name, name_record.offset));
}
RTAG_ENDSTR => {
let (name, begin) = start.take()?;
let body = &input[begin..record.offset + 4];
let span = CellSpan {
body,
end_record: index,
};
if cells.insert(name, span).is_some() {
return None;
}
}
_ => (),
}
}
start.is_none().then_some(cells)
}
fn matching_cells<'a>(
old: &FramedRevision<'a>,
new: &FramedRevision<'a>,
input_size: usize,
) -> BTreeMap<&'a str, usize> {
let mut matches = BTreeMap::new();
if let (Some(old), Some(new)) = (&old.spans, &new.spans) {
let mut bytes = 0;
for (&name, cell) in new {
if old.get(name).is_some_and(|old| old.body == cell.body) {
bytes += cell.body.len();
matches.insert(name, cell.end_record);
}
}
// Reuse serializes some parsing work, so require substantial overlap.
if bytes < input_size - input_size / 10 {
matches.clear();
}
}
matches
}
pub(super) struct CellReuse<'a, 'b> {
old: &'b Document<'a>,
matches: &'b BTreeMap<&'a str, usize>,
}
impl<'a> CellReuse<'a, '_> {
pub(super) fn cell(
&self,
name: &str,
offset: usize,
records: &[Record<'a>],
) -> Option<(Cell<'a>, usize)> {
let end = *self.matches.get(name)?;
let previous = &self.old.cells[name];
// Re-run library metadata and reference checks with the new offsets.
let objects = previous
.objects
.iter()
.filter(|object| object.target.is_some() || object.kind == RTAG_TEXT)
.cloned()
.map(|mut object| {
object.offset = offset + (object.offset - previous.offset);
object.end_offset = offset + (object.end_offset - previous.offset);
object
})
.collect();
Some((
Cell {
offset,
end_offset: records[end].offset,
objects,
attributes: previous.attributes.clone(),
issues: Vec::new(),
},
end,
))
}
}
/// Two unscoped GDSII revisions prepared for comparison.
#[derive(Debug)]
pub struct RevisionPair<'a> {
old: Document<'a>,
new: Document<'a>,
reused_cells: BTreeSet<&'a str>,
}
impl<'a> RevisionPair<'a> {
pub fn parse(old: &'a [u8], new: &'a [u8]) -> Result<Self, Diagnostic> {
let invalid = |message, side| Diagnostic::new("invalid_input", message, Some(side));
let input_size = old.len().max(new.len());
let parallel = thread::available_parallelism().is_ok_and(|count| count.get() > 1);
let (old, new) = run_together(parallel, old, new, FramedRevision::read);
let old = old.map_err(|message| invalid(message, Side::Old))?;
let new = match new {
Ok(new) => new,
Err(message) => {
Document::parse_records(&old.records, None)
.map_err(|message| invalid(message, Side::Old))?;
return Err(invalid(message, Side::New));
}
};
let mut matches = matching_cells(&old, &new, input_size);
let (old, new) = if matches.is_empty() {
let (old, new) = run_together(parallel, old, new, |revision| {
Document::parse_records(&revision.records, None)
});
(old.map_err(|message| invalid(message, Side::Old))?, new)
} else {
let old_doc = Document::parse_records(&old.records, None)
.map_err(|message| invalid(message, Side::Old))?;
drop(old);
// The regular parser gives unsupported records their own offsets
// and messages, so parse those cells independently on each side.
matches.retain(|name, _| {
old_doc.cells.get(*name).is_some_and(|cell| {
cell.issues.is_empty()
&& cell.objects.iter().all(|object| object.issues.is_empty())
})
});
let reuse = CellReuse {
old: &old_doc,
matches: &matches,
};
let new = Document::parse_records(&new.records, Some(reuse));
(old_doc, new)
};
Ok(Self {
old,
new: new.map_err(|message| invalid(message, Side::New))?,
reused_cells: matches.into_keys().collect(),
})
}
pub fn compare(self) -> Comparison {
Comparison::collect(&self.old, &self.new, &self.reused_cells, None)
}
pub fn compare_text(self) -> TextComparison {
TextComparison::collect(&self.old, &self.new, &self.reused_cells)
}
}
fn run_together<I: Send, T: Send>(
parallel: bool,
old: I,
new: I,
process: impl Fn(I) -> T + Copy + Send,
) -> (T, T) {
if !parallel {
return (process(old), process(new));
}
// Transfer ownership only once a worker starts, so a spawn failure can
// process the same input on the calling thread.
let input = Mutex::new(Some(old));
let run_old = {
let input = &input;
move || {
let old = input.lock().unwrap().take().unwrap();
process(old)
}
};
thread::scope(|scope| {
if let Ok(worker) = thread::Builder::new().spawn_scoped(scope, run_old) {
let new = process(new);
return (worker.join().unwrap(), new);
}
(run_old(), process(new))
})
}

View file

@ -1,6 +1,11 @@
//! Versioned machine-readable diff reports. No consumer needs to parse prose.
use super::*;
use serde::{
Serialize, Serializer,
ser::{SerializeMap, SerializeSeq},
};
use serde_json::{Value, json};
use std::io;
use std::path::{Path, PathBuf};
pub const SCHEMA_VERSION: u32 = 1;
@ -196,10 +201,127 @@ fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec<Value> {
.collect()
}
struct SelectedObjects<'a> {
// Up to the last emitted object. A layer filter can leave gaps in this prefix.
objects: &'a [ObjectChange],
layer: Option<Layer>,
cell_index: usize,
side: &'static str,
}
impl SelectedObjects<'_> {
fn id(&self, index: usize) -> String {
format!("c{}:{}:{index}", self.cell_index, self.side)
}
fn contains(&self, index: usize) -> bool {
self.objects
.get(index)
.is_some_and(|object| self.layer.is_none_or(|layer| object.layer == Some(layer)))
}
}
impl Serialize for SelectedObjects<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut sequence = serializer.serialize_seq(None)?;
for (index, change) in self.objects.iter().enumerate() {
if self.layer.is_none_or(|layer| change.layer == Some(layer)) {
sequence.serialize_element(&object(change, &self.id(index), self.side))?;
}
}
sequence.end()
}
}
struct SelectedCell<'a> {
cell: &'a CellChange,
removed: SelectedObjects<'a>,
added: SelectedObjects<'a>,
}
impl Serialize for SelectedCell<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let revision = |revision: &CellRevision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)});
let mut map = serializer.serialize_map(Some(7))?;
map.serialize_entry("added", &self.added)?;
map.serialize_entry("after", &self.cell.new.as_ref().map(revision))?;
map.serialize_entry("attributes_changed", &self.cell.attributes_changed())?;
map.serialize_entry("before", &self.cell.old.as_ref().map(revision))?;
map.serialize_entry("name", &self.cell.name)?;
map.serialize_entry("removed", &self.removed)?;
map.serialize_entry("replacements", &SelectedReplacements(self))?;
map.end()
}
}
struct SelectedReplacements<'a>(&'a SelectedCell<'a>);
impl Serialize for SelectedReplacements<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let selection = self.0;
let mut sequence = serializer.serialize_seq(None)?;
for replacement in &selection.cell.replacements {
if selection.removed.contains(replacement.removed)
&& selection.added.contains(replacement.added)
{
let old = &selection.cell.removed[replacement.removed];
let new = &selection.cell.added[replacement.added];
sequence.serialize_element(&json!({
"removed": selection.removed.id(replacement.removed),
"added": selection.added.id(replacement.added),
"match_rule": replacement.match_rule,
"explanation": changes::replacement_summary(old, new),
"field_changes": field_changes(old, new),
"properties_changed": old.properties != new.properties
}))?;
}
}
sequence.end()
}
}
struct JsonReport<'a> {
metadata: Value,
cells: Vec<SelectedCell<'a>>,
}
impl Serialize for JsonReport<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let metadata = self
.metadata
.as_object()
.expect("report metadata is an object");
let mut map = serializer.serialize_map(Some(metadata.len()))?;
for (key, value) in metadata {
if key == "cell_changes" {
map.serialize_entry(key, &self.cells)?;
} else {
map.serialize_entry(key, value)?;
}
}
map.end()
}
}
impl Comparison {
/// Render the captured comparison evidence without retaining its source documents.
/// IDs are local to this report, not persistent object identities.
pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value {
serde_json::to_value(self.prepare_report(inputs, options))
.expect("serialize comparison report")
}
/// Stream a pretty-printed report to a writer.
pub fn write_json_report(
&self,
writer: impl io::Write,
inputs: &ReportInputs,
options: &ReportOptions,
) -> serde_json::Result<()> {
serde_json::to_writer_pretty(writer, &self.prepare_report(inputs, options))
}
fn prepare_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> JsonReport<'_> {
let outcome = self.outcome();
let mut report = envelope(
inputs,
@ -257,12 +379,8 @@ impl Comparison {
filtered_objects += cell.removed.len() + cell.added.len();
continue;
}
let (mut removed, mut added) = (Vec::new(), Vec::new());
let mut shown = BTreeSet::new();
for (side, objects, output) in [
("old", &cell.removed, &mut removed),
("new", &cell.added, &mut added),
] {
let mut select = |objects: &[ObjectChange]| {
let mut end = 0;
for (index, change) in objects.iter().enumerate() {
if options
.filter
@ -273,32 +391,29 @@ impl Comparison {
} else if options.max_objects.is_some_and(|max| emitted >= max) {
limited += 1;
} else {
let id = format!("c{cell_index}:{side}:{index}");
output.push(object(change, &id, side));
shown.insert(id);
end = index + 1;
emitted += 1;
}
}
}
let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| {
let a = format!("c{cell_index}:old:{}", r.removed);
let b = format!("c{cell_index}:new:{}", r.added);
(shown.contains(&a) && shown.contains(&b)).then(|| json!({
"removed": a, "added": b, "match_rule": r.match_rule,
"explanation": changes::replacement_summary(&cell.removed[r.removed], &cell.added[r.added]),
"field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]),
"properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties
}))
}).collect();
cells.push(json!({
"name": cell.name,
"before": cell.old.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})),
"after": cell.new.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})),
"attributes_changed": cell.attributes_changed(),
"removed": removed, "added": added, "replacements": replacements
}));
end
};
let layer = options.filter.layer;
cells.push(SelectedCell {
cell,
removed: SelectedObjects {
objects: &cell.removed[..select(&cell.removed)],
layer,
cell_index,
side: "old",
},
added: SelectedObjects {
objects: &cell.added[..select(&cell.added)],
layer,
cell_index,
side: "new",
},
});
}
report["cell_changes"] = json!(cells);
report["report_selection"]["truncated"] =
json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0);
for (key, count) in [
@ -313,7 +428,10 @@ impl Comparison {
if options.hierarchy {
report["hierarchy"] = graph(self);
}
report
JsonReport {
metadata: report,
cells,
}
}
}