Simplify and speed up GDSII diffs

Summarize added and removed cells, group references, and keep replacement pairs together at text display limits. Trim hierarchy labels and summarize long unmatched paths.

Stream JSON object evidence and reduce parsing and comparison allocations. Parse large revisions concurrently and adaptively reuse matching cell bodies when most of the layout is shared.
This commit is contained in:
Jan Petykiewicz 2026-11-18 10:58:19 -08:00
commit 8efeb04aab
6 changed files with 1137 additions and 261 deletions

View file

@ -1,7 +1,7 @@
use flate2::read::MultiGzDecoder;
use pheon::review::report::{ReportInputs, ReportOptions, failure_report};
use pheon::review::{
Comparison, Diagnostic, Document, Outcome, Side, dump_records,
Comparison, Diagnostic, Document, Outcome, RevisionPair, Side, TextComparison, dump_records,
names::{parse_layer, parse_layer_name, write_layer_names},
selection::{DisplayFilter, Scope},
};
@ -12,6 +12,7 @@ use std::{
io::{self, Read, Write},
path::PathBuf,
process::ExitCode,
thread,
};
const HELP: &str = "Usage:
@ -116,17 +117,36 @@ fn stdout(output: &str) -> Result<(), String> {
.map_err(|e| e.to_string())
}
fn json_stdout(report: &serde_json::Value) -> Result<(), String> {
let mut output = io::stdout().lock();
serde_json::to_writer_pretty(&mut output, report).map_err(|e| e.to_string())?;
output.write_all(b"\n").map_err(|e| e.to_string())
fn json_stdout(
write: impl FnOnce(&mut io::BufWriter<io::StdoutLock<'_>>) -> serde_json::Result<()>,
) -> Result<(), String> {
let mut output = io::BufWriter::new(io::stdout().lock());
write(&mut output).map_err(|e| e.to_string())?;
output
.write_all(b"\n")
.and_then(|()| output.flush())
.map_err(|e| e.to_string())
}
fn prepare_comparison(
fn parse_revision(bytes: Option<&[u8]>, side: Side) -> Result<Option<Document<'_>>, Diagnostic> {
bytes
.map(|bytes| {
let mut document = Document::parse(bytes)
.map_err(|message| Diagnostic::new("invalid_input", message, Some(side)))?;
// Comparison uses the parsed cells and their source offsets.
document.records = Vec::new();
Ok(document)
})
.transpose()
}
fn prepare_comparison<T>(
old_path: Option<&OsStr>,
new_path: Option<&OsStr>,
scope: &Scope,
) -> Result<Comparison, Diagnostic> {
compare: impl FnOnce(&Document<'_>, &Document<'_>) -> T,
compare_pair: impl FnOnce(RevisionPair<'_>) -> T,
) -> Result<T, Diagnostic> {
let read = |path: Option<&OsStr>, side| {
path.map(|path| {
read_input(path).map_err(|mut diagnostic| {
@ -138,16 +158,34 @@ fn prepare_comparison(
};
let old_bytes = read(old_path, Side::Old)?;
let new_bytes = read(new_path, Side::New)?;
let old = old_bytes
const PARALLEL_PARSE_MIN_BYTES: usize = 2 * 1024 * 1024;
let large_pair = old_bytes
.as_deref()
.map(Document::parse)
.transpose()
.map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::Old)))?;
let new = new_bytes
.as_deref()
.map(Document::parse)
.transpose()
.map_err(|message| Diagnostic::new("invalid_input", message, Some(Side::New)))?;
.zip(new_bytes.as_deref())
.filter(|(old, new)| old.len().min(new.len()) >= PARALLEL_PARSE_MIN_BYTES);
if scope == &Scope::default()
&& let Some((old, new)) = large_pair
{
return RevisionPair::parse(old, new).map(compare_pair);
}
let parse_old = || parse_revision(old_bytes.as_deref(), Side::Old);
let parse_new = || parse_revision(new_bytes.as_deref(), Side::New);
let parallel = if large_pair.is_some()
&& thread::available_parallelism().is_ok_and(|count| count.get() > 1)
{
thread::scope(|scope| {
let worker = thread::Builder::new().spawn_scoped(scope, parse_old).ok()?;
let new = parse_new();
Some((worker.join().unwrap(), new))
})
} else {
None
};
// Report an old-side parse error first if both inputs are malformed.
let (old, new) = match parallel {
Some((old, new)) => (old?, new?),
None => (parse_old()?, parse_new()?),
};
let (old, new) = match (old, new) {
(Some(old), Some(new)) => (old, new),
(None, Some(new)) => (new.empty_like(), new),
@ -163,6 +201,9 @@ fn prepare_comparison(
));
}
};
if scope == &Scope::default() {
return Ok(compare(&old, &new));
}
scope
.validate_pair(&old, &new)
.map_err(|message| Diagnostic::new("invalid_scope", message, None))?;
@ -172,7 +213,7 @@ fn prepare_comparison(
let new = new
.selected(scope, true)
.map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::New)))?;
Ok(Comparison::between(&old, &new))
Ok(compare(&old, &new))
}
fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result<u8, String> {
@ -180,14 +221,26 @@ fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Resu
old: Some(PathBuf::from(&paths[0])),
new: Some(PathBuf::from(&paths[1])),
};
let (report, outcome) = match prepare_comparison(Some(&paths[0]), Some(&paths[1]), scope) {
Ok(diff) => (diff.json_report(&inputs, options), diff.outcome()),
Err(error) => (
failure_report(&inputs, scope, options, &error),
Outcome::Failed,
let (result, outcome) = match prepare_comparison(
Some(&paths[0]),
Some(&paths[1]),
scope,
Comparison::between,
|pair| pair.compare(),
) {
Ok(diff) => (
json_stdout(|output| diff.write_json_report(output, &inputs, options)),
diff.outcome(),
),
Err(error) => {
let report = failure_report(&inputs, scope, options, &error);
(
json_stdout(|output| serde_json::to_writer_pretty(output, &report)),
Outcome::Failed,
)
}
};
if let Err(error) = json_stdout(&report) {
if let Err(error) = result {
// Once output starts, never append a second report on a write failure.
eprintln!("pheon: {error}");
return Ok(2);
@ -327,25 +380,30 @@ fn run(args: &[OsString]) -> Result<u8, String> {
(!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()),
(!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()),
&scope,
TextComparison::between,
|pair| pair.compare_text(),
)
.map_err(|error| error.message)?;
let mut output = String::new();
if git {
let action = match (old_missing, new_missing) {
(true, false) => " added",
(false, true) => " deleted",
_ => "",
};
if paths.len() == 9 {
output.push_str(&format!("layout {:?} -> {:?}\n", paths[0], paths[7]));
output.push_str(&format!(
"layout {:?} -> {:?}{action}\n",
paths[0], paths[7]
));
} else {
output.push_str(&format!("layout {:?}\n", paths[0]));
output.push_str(&format!("layout {:?}{action}\n", paths[0]));
}
if old_missing {
output.push_str("layout added\n");
} else if new_missing {
output.push_str("layout deleted\n");
}
if paths[3] != paths[6] {
if !old_missing && !new_missing && paths[3] != paths[6] {
output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6]));
}
}
output.push_str(&diff.text_filtered(max_changes, &show));
output.push_str(&diff.render_text(max_changes, &show, !old_missing && !new_missing));
if format != "none" {
if format == "mermaid" {
output.push_str("Hierarchy\n```mermaid\n");
@ -402,7 +460,9 @@ fn main() -> ExitCode {
&ReportOptions::default(),
&Diagnostic::new("command_error", message, None),
);
if let Err(error) = json_stdout(&report) {
if let Err(error) =
json_stdout(|output| serde_json::to_writer_pretty(output, &report))
{
eprintln!("pheon: {error}");
}
} else {

View file

@ -182,17 +182,20 @@ impl Record<Int4Array> for XY {
}
impl XY {
pub(crate) fn point([x0, x1, x2, x3, y0, y1, y2, y3]: [u8; 8]) -> [i32; 2] {
[
i32::from_be_bytes([x0, x1, x2, x3]),
i32::from_be_bytes([y0, y1, y2, y3]),
]
}
/// Iterate over a validated XY payload.
pub(crate) fn points(data: &[u8]) -> impl ExactSizeIterator<Item = [i32; 2]> + '_ {
data.as_chunks::<8>()
.0
.iter()
.map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| {
[
i32::from_be_bytes([x0, x1, x2, x3]),
i32::from_be_bytes([y0, y1, y2, y3]),
]
})
.copied()
.map(Self::point)
}
/// Bounds of a rectangle encoded as a closed XY ring.

View file

@ -14,10 +14,13 @@ use std::{
pub mod changes;
pub mod names;
mod pair;
pub mod report;
pub mod selection;
use changes::Replacement;
use pair::CellReuse;
pub use pair::RevisionPair;
use selection::{DisplayFilter, Scope};
/// Input revision associated with a diagnostic.
@ -139,7 +142,7 @@ struct Key<'a> {
properties: Vec<(u16, &'a [u8])>,
}
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct Object<'a> {
pub offset: usize,
pub end_offset: usize,
@ -231,6 +234,8 @@ fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> {
])
}
const POINTS_PER_LINE: usize = 4;
fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result {
let mut points = XY::points(data);
if points.len() == 1 {
@ -240,7 +245,11 @@ fn write_points(out: &mut fmt::Formatter<'_>, data: &[u8]) -> fmt::Result {
out.write_str(" xy=[")?;
for (index, [x, y]) in points.enumerate() {
if index != 0 {
out.write_str(if index % 4 == 0 { ",\n " } else { ", " })?;
out.write_str(if index % POINTS_PER_LINE == 0 {
",\n "
} else {
", "
})?;
}
write!(out, "({x}, {y})")?;
}
@ -312,11 +321,10 @@ fn coordinate_bounds(points: impl IntoIterator<Item = [i32; 2]>) -> Bounds {
}
// Booth's least-rotation algorithm avoids quadratic work on repeated vertices.
fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> {
let n = points.len();
fn least_rotation(n: usize, point: impl Fn(usize) -> [i32; 2]) -> usize {
let (mut i, mut j, mut k) = (0, 1, 0);
while i < n && j < n && k < n {
match points[(i + k) % n].cmp(&points[(j + k) % n]) {
match point((i + k) % n).cmp(&point((j + k) % n)) {
std::cmp::Ordering::Equal => k += 1,
std::cmp::Ordering::Greater => {
i += k + 1;
@ -334,21 +342,37 @@ fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> {
}
}
}
let start = i.min(j);
(0..n).map(|k| points[(start + k) % n]).collect()
i.min(j)
}
fn canonical_ring(mut ring: Vec<[i32; 2]>) -> Vec<u8> {
ring.pop(); // The caller has checked closure.
let forward = least_rotation(&ring);
ring.reverse();
let backward = least_rotation(&ring);
let mut ring = forward.min(backward);
ring.push(ring[0]);
ring.into_iter()
.flatten()
.flat_map(i32::to_be_bytes)
.collect()
fn canonical_ring(data: &[u8]) -> Option<Cow<'_, [u8]>> {
let (points, _) = data.as_chunks::<8>();
let (last, ring) = points.split_last()?;
if ring.len() < 3 || ring.first() != Some(last) {
return None;
}
let n = ring.len();
let first = least_rotation(n, |index| XY::point(ring[index]));
let split = n - least_rotation(n, |index| XY::point(ring[n - 1 - index]));
let forward = ring[first..].iter().chain(&ring[..first]);
let backward = ring[..split].iter().rev().chain(ring[split..].iter().rev());
let decode = |point: &[u8; 8]| XY::point(*point);
let use_forward = forward
.clone()
.map(decode)
.cmp(backward.clone().map(decode))
.is_le();
if use_forward && first == 0 {
return Some(Cow::Borrowed(data));
}
let mut result = Vec::with_capacity(data.len());
if use_forward {
result.extend(forward.flatten().copied());
} else {
result.extend(backward.flatten().copied());
}
result.extend_from_within(..8);
Some(Cow::Owned(result))
}
fn validate_payload(record: &Record<'_>) -> Result<(), String> {
@ -474,14 +498,12 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
data = Cow::Borrowed(unpad_ascii(record.data));
}
if record.tag == RTAG_XY && is_polygon(first.tag) {
let p: Vec<_> = XY::points(record.data).collect();
if p.len() < 4 || p.first() != p.last() {
return Err(error(
data = canonical_ring(record.data).ok_or_else(|| {
error(
record.offset,
"polygon must have at least three vertices and be closed",
));
}
data = Cow::Owned(canonical_ring(p));
)
})?;
}
fields.push((record.tag, data));
i += 1;
@ -556,6 +578,13 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
}
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
read_records_with(input, |_, _| {})
}
fn read_records_with<'a>(
input: &'a [u8],
mut observe: impl FnMut(usize, &Record<'a>),
) -> Result<Vec<Record<'a>>, String> {
if input.starts_with(b"\x1f\x8b") {
return Err(error(
0,
@ -574,11 +603,13 @@ fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
ErrType::Incomplete(_) => error(offset, "truncated record header"),
ErrType::Failed(message) => error(offset, message),
})?;
records.push(Record {
let record = Record {
offset,
tag: frame.tag,
data: frame.data,
});
};
observe(records.len(), &record);
records.push(record);
remaining = next;
if frame.tag == RTAG_ENDLIB {
if remaining.iter().any(|b| *b != 0) {
@ -596,7 +627,16 @@ fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
impl<'a> Document<'a> {
pub fn parse(input: &'a [u8]) -> Result<Self, String> {
let records = read_records(input)?;
for record in &records {
let mut doc = Self::parse_records(&records, None)?;
doc.records = records;
Ok(doc)
}
fn parse_records(
records: &[Record<'a>],
reuse: Option<CellReuse<'a, '_>>,
) -> Result<Self, String> {
for record in records {
validate_payload(record)?;
}
if records.first().map(|r| r.tag) != Some(RTAG_HEADER)
@ -606,7 +646,7 @@ impl<'a> Document<'a> {
return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string());
}
let mut doc = Self {
records,
records: Vec::new(),
cells: BTreeMap::new(),
layer_names: BTreeMap::new(),
diagnostics: Vec::new(),
@ -619,9 +659,9 @@ impl<'a> Document<'a> {
let mut i = 2;
let mut global_seen = BTreeSet::new();
let mut cells_started = false;
doc.header.push((RTAG_HEADER, doc.records[0].data));
while i + 1 < doc.records.len() {
let record = &doc.records[i];
doc.header.push((RTAG_HEADER, records[0].data));
while i + 1 < records.len() {
let record = &records[i];
if record.tag != RTAG_BGNSTR {
if cells_started {
return Err(error(
@ -674,8 +714,7 @@ impl<'a> Document<'a> {
cells_started = true;
let offset = record.offset;
i += 1;
let name_record = doc
.records
let name_record = records
.get(i)
.filter(|r| r.tag == RTAG_STRNAME)
.ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?;
@ -685,6 +724,15 @@ impl<'a> Document<'a> {
if name.is_empty() {
return Err(error(name_record.offset, "empty cell name"));
}
if let Some(reuse) = &reuse
&& let Some((cell, end)) = reuse.cell(&name, offset, records)
{
if doc.cells.insert(name.clone(), cell).is_some() {
return Err(error(offset, format!("duplicate cell name {name:?}")));
}
i = end + 1;
continue;
}
i += 1;
let mut cell = Cell {
offset,
@ -694,8 +742,7 @@ impl<'a> Document<'a> {
issues: Vec::new(),
};
loop {
let record = doc
.records
let record = records
.get(i)
.ok_or_else(|| error(offset, "missing ENDSTR"))?;
if record.tag == RTAG_ENDSTR {
@ -706,19 +753,19 @@ impl<'a> Document<'a> {
if is_element(record.tag) {
let start = i;
i += 1;
while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) {
if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR)
|| is_element(doc.records[i].tag)
while records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) {
if matches!(records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR)
|| is_element(records[i].tag)
{
return Err(error(doc.records[i].offset, "missing ENDEL"));
return Err(error(records[i].offset, "missing ENDEL"));
}
i += 1;
}
if i == doc.records.len() {
if i == records.len() {
return Err(error(offset, "missing ENDEL"));
}
let mut issues = Vec::new();
let mut parsed = object(&doc.records[start..=i], &mut issues)?;
let mut parsed = object(&records[start..=i], &mut issues)?;
parsed.issues = issues;
cell.objects.push(parsed);
i += 1;
@ -820,16 +867,21 @@ impl<'a> Document<'a> {
}
pub fn edges(&self) -> BTreeMap<(String, String), u64> {
let mut edges = BTreeMap::new();
for (name, cell) in &self.cells {
for object in &cell.objects {
if let Some(target) = &object.target {
let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64);
*count = count.saturating_add(object.multiplicity);
self.cells
.iter()
.flat_map(|(name, cell)| {
let mut counts = BTreeMap::new();
for object in &cell.objects {
if let Some(target) = object.target.as_deref() {
let count = counts.entry(target).or_insert(0_u64);
*count = count.saturating_add(object.multiplicity);
}
}
}
}
edges
counts
.into_iter()
.map(|(target, count)| ((name.clone(), target.to_owned()), count))
})
.collect()
}
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
@ -1150,8 +1202,62 @@ pub struct Comparison {
pub diagnostics: Vec<Diagnostic>,
}
/// A comparison for text and hierarchy reports. Layered objects in whole-cell
/// changes contribute to the cell's object count.
#[derive(Debug)]
pub struct TextComparison {
comparison: Comparison,
whole_cell_objects: BTreeMap<String, usize>,
}
impl TextComparison {
pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self {
Self::collect(old, new, &BTreeSet::new())
}
fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self {
let mut whole_cell_objects = BTreeMap::new();
let comparison = Comparison::collect(old, new, reused_cells, Some(&mut whole_cell_objects));
Self {
comparison,
whole_cell_objects,
}
}
pub fn render_text(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
) -> String {
self.comparison.render_text_with_counts(
max_objects,
show,
show_library_presence,
&self.whole_cell_objects,
)
}
pub fn outcome(&self) -> Outcome {
self.comparison.outcome()
}
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
self.comparison.hierarchy(mermaid, max_depth)
}
}
impl Comparison {
pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self {
Self::collect(old, new, &BTreeSet::new(), None)
}
fn collect(
old: &Document<'_>,
new: &Document<'_>,
reused_cells: &BTreeSet<&str>,
mut whole_cell_objects: Option<&mut BTreeMap<String, usize>>,
) -> Self {
let mut result = Self {
old: Revision::from_document(old),
new: Revision::from_document(new),
@ -1231,6 +1337,9 @@ impl Comparison {
}
let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect();
for name in names {
if reused_cells.contains(name.as_str()) {
continue;
}
let a = old.cells.get(name);
let b = new.cells.get(name);
let mut change = CellChange {
@ -1241,6 +1350,30 @@ impl Comparison {
added: Vec::new(),
replacements: Vec::new(),
};
if let Some(counts) = whole_cell_objects.as_deref_mut()
&& (a.is_none() || b.is_none())
{
let details = if a.is_some() {
&mut change.removed
} else {
&mut change.added
};
let mut count = 0;
for object in a.or(b).into_iter().flat_map(|cell| &cell.objects) {
if object.metadata {
continue;
}
count += 1;
if object.layer.is_none() || object.target.is_some() {
details.push(object.into());
}
}
if count != details.len() {
counts.insert(name.clone(), count);
}
result.cells.push(change);
continue;
}
let a: Vec<_> = a
.into_iter()
.flat_map(|c| &c.objects)
@ -1295,6 +1428,7 @@ impl Comparison {
}
}
/// Render the affected hierarchy. ASCII output is empty when no cells changed.
pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String {
render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth)
}
@ -1305,10 +1439,35 @@ impl Comparison {
}
pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String {
let (removed, added) = self.cells.iter().fold((0, 0), |(removed, added), cell| {
(removed + cell.removed.len(), added + cell.added.len())
});
let total = removed + added;
self.render_text(max_objects, show, true)
}
/// Render a text report. Set `show_library_presence` to false when a file
/// header already announces the library addition or deletion.
pub fn render_text(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
) -> String {
self.render_text_with_counts(max_objects, show, show_library_presence, &BTreeMap::new())
}
fn render_text_with_counts(
&self,
max_objects: usize,
show: &DisplayFilter,
show_library_presence: bool,
whole_cell_objects: &BTreeMap<String, usize>,
) -> String {
let (mut modified, mut added, mut removed) = (0_usize, 0_usize, 0_usize);
for cell in &self.cells {
match (&cell.old, &cell.new) {
(None, Some(_)) => added += 1,
(Some(_), None) => removed += 1,
_ => modified += 1,
}
}
let mut out = match self.outcome() {
Outcome::Equal => "Equal",
Outcome::Different => "Different",
@ -1316,19 +1475,25 @@ impl Comparison {
Outcome::Failed => "Failed",
}
.to_string();
if !self.cells.is_empty() {
let cells = if self.cells.len() == 1 {
"cell"
} else {
"cells"
};
write!(out, " · {} changed {cells}", self.cells.len()).unwrap();
}
if total != 0 {
write!(out, " · {removed} removed, {added} added").unwrap();
let mut cell_counts = [
(modified, "modified"),
(added, "added"),
(removed, "removed"),
]
.into_iter()
.filter(|(count, _)| *count != 0);
if let Some((count, action)) = cell_counts.next() {
let noun = if count == 1 { "cell" } else { "cells" };
write!(out, " · {count} {noun} {action}").unwrap();
for (count, action) in cell_counts {
write!(out, ", {count} {action}").unwrap();
}
}
let library_changes = self.library_changes.iter().filter(|change| {
show_library_presence || !matches!(change, LibraryChange::Presence { .. })
});
for (count, label) in [
(self.library_changes.len(), "library change"),
(library_changes.clone().count(), "library change"),
(self.layer_names.len(), "layer name change"),
] {
if count != 0 {
@ -1349,7 +1514,7 @@ impl Comparison {
out.push('\n');
}
let notice_start = out.len();
for change in &self.library_changes {
for change in library_changes {
writeln!(out, "library {change}").unwrap();
}
if self.library_changes.iter().any(|change| {
@ -1379,22 +1544,38 @@ impl Comparison {
}
let body_start = out.len();
let mut shown_layers = BTreeSet::new();
let mut detail_total = 0;
let mut selected = 0;
let mut shown: usize = 0;
let mut hidden_cell_changes = 0;
let visible =
|object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer));
let presence = |action, count| {
let objects = if count == 1 { "object" } else { "objects" };
Cow::Owned(format!(" ({action}, {count} {objects})"))
};
for cell in &self.cells {
let object_count = |details| {
whole_cell_objects
.get(&cell.name)
.copied()
.unwrap_or(details)
};
let note = match (&cell.old, &cell.new) {
(None, Some(_)) => presence("added", cell.added.len()),
(Some(_), None) => presence("removed", cell.removed.len()),
(None, Some(_)) => presence("added", object_count(cell.added.len())),
(Some(_), None) => presence("removed", object_count(cell.removed.len())),
_ if cell.attributes_changed() => Cow::Borrowed(" (attributes changed)"),
_ => Cow::Borrowed(""),
};
let whole_cell = cell.old.is_none() || cell.new.is_none();
// The heading accounts for layered objects in a whole-cell change.
detail_total += if whole_cell {
cell.removed
.iter()
.chain(&cell.added)
.filter(|object| object.layer.is_none())
.count()
} else {
cell.removed.len() + cell.added.len()
};
if show.cell.as_ref().is_some_and(|name| name != &cell.name) {
hidden_cell_changes += usize::from(!note.is_empty());
continue;
@ -1402,11 +1583,16 @@ impl Comparison {
let cell_start = out.len();
let previously_shown = shown;
writeln!(out, "cell {:?}{note}", cell.name).unwrap();
let whole_cell = cell.old.is_none() || cell.new.is_none();
let visible = |object: &ObjectChange| {
(!whole_cell || object.layer.is_none())
&& show.layer.is_none_or(|layer| object.layer == Some(layer))
};
let mut polygons = changes::PolygonChanges::default();
let mut paired_old = BTreeSet::new();
let mut paired_new = BTreeSet::new();
for replacement in &cell.replacements {
paired_old.insert(replacement.removed);
paired_new.insert(replacement.added);
let a = &cell.removed[replacement.removed];
let b = &cell.added[replacement.added];
if max_objects - shown >= 2 && visible(a) && visible(b) {
@ -1416,8 +1602,6 @@ impl Comparison {
writeln!(out, " ~ {}", changes::replacement_summary(a, b)).unwrap();
}
shown_layers.extend(a.layer);
paired_old.insert(replacement.removed);
paired_new.insert(replacement.added);
shown += 2;
}
}
@ -1426,7 +1610,7 @@ impl Comparison {
(false, &cell.added, &paired_new),
] {
let sign = if removed { "-" } else { "+" };
let mut references = BTreeMap::<&str, changes::ReferenceCounts>::new();
let mut references = BTreeMap::new();
for (index, object) in objects.iter().enumerate() {
if !visible(object) {
continue;
@ -1435,17 +1619,24 @@ impl Comparison {
if !paired.contains(&index) && shown < max_objects {
if is_polygon(object.kind) {
polygons.unmatched(object, removed);
} else if whole_cell && let Some(target) = object.target.as_deref() {
references.entry(target).or_default().add(object);
} else if let Some(target) = object.target.as_deref() {
let (_, counts) = references
.entry(target)
.or_insert_with(|| (object, changes::ReferenceCounts::default()));
counts.add(object);
} else {
writeln!(out, " {sign} {object}").unwrap();
changes::write_unmatched(&mut out, sign, object);
}
shown_layers.extend(object.layer);
shown += 1;
}
}
for (target, counts) in references {
counts.write(&mut out, sign, target);
for (target, (first, counts)) in references {
if whole_cell || counts.len() > 1 {
counts.write(&mut out, sign, target);
} else {
writeln!(out, " {sign} {first}").unwrap();
}
}
}
polygons.write(&mut out);
@ -1460,31 +1651,23 @@ impl Comparison {
}
}
out.insert_str(body_start, &legend);
let mut notice = String::new();
if shown < total {
let mut hidden = Vec::new();
if total > selected {
hidden.push(format!("{} filtered out", total - selected));
let mut omissions = Vec::new();
for (count, label, reason) in [
(detail_total - selected, "object change", "filtered out"),
(selected - shown, "object change", "over the limit"),
(hidden_cell_changes, "cell-level change", "filtered out"),
] {
if count != 0 {
let plural = if count == 1 { "" } else { "s" };
omissions.push(format!("{count} {label}{plural} {reason}"));
}
if selected > shown {
hidden.push(format!("{} over the limit", selected - shown));
}
writeln!(
notice,
"Showing {shown} of {total} object changes ({})",
hidden.join(", ")
)
.unwrap();
}
if hidden_cell_changes != 0 {
let plural = if hidden_cell_changes == 1 { "" } else { "s" };
writeln!(
notice,
"{hidden_cell_changes} cell-level change{plural} filtered out"
)
.unwrap();
if !omissions.is_empty() {
out.insert_str(
notice_start,
&format!("Omitted · {}\n", omissions.join(" · ")),
);
}
out.insert_str(notice_start, &notice);
// General comparison errors precede revision-specific errors in text.
if !self.diagnostics.is_empty() {
out.push_str("Diagnostics\n");
@ -1563,6 +1746,10 @@ fn render_hierarchy(
mermaid: bool,
max_depth: usize,
) -> String {
if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) {
return String::new();
}
let HierarchyAnalysis {
old_edges,
new_edges,
@ -1639,10 +1826,18 @@ fn render_hierarchy(
" [unresolved]"
}
} else if direct.contains(name) {
" [cell changed]"
if mermaid {
" [cell changed]"
} else if !old.known_cells.contains(name) {
" [added]"
} else if !new.known_cells.contains(name) {
" [removed]"
} else {
" [changed]"
}
} else if affected.contains(name) {
" [affected]"
} else if changes.is_some() {
} else if mermaid && changes.is_some() {
" [cell unchanged]"
} else {
""
@ -1659,7 +1854,7 @@ fn render_hierarchy(
};
format!(
"{count}{}",
if changed_refs.contains(&key) {
if (mermaid || a == b) && changed_refs.contains(&key) {
" [reference changed]"
} else {
""

View file

@ -28,6 +28,9 @@ fn group(object: &ObjectChange, bounds: bool) -> Group {
/// Pair only unique candidates in the remaining multisets. Ambiguous groups
/// remain additions/removals; no identity or geometric equivalence is inferred.
pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec<Replacement> {
if old.is_empty() || new.is_empty() {
return Vec::new();
}
let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new());
let mut result = Vec::new();
for bounds in [true, false] {
@ -101,7 +104,7 @@ fn object_heading(object: &ObjectChange, kind: &str) -> String {
}
}
fn polygon_xy(object: &ObjectChange) -> &[u8] {
fn object_xy(object: &ObjectChange) -> &[u8] {
object
.fields
.iter()
@ -109,6 +112,27 @@ fn polygon_xy(object: &ObjectChange) -> &[u8] {
.map_or(&[], |field| field.data.as_slice())
}
pub(super) fn write_unmatched(out: &mut String, sign: &str, object: &ObjectChange) {
if object.kind == RTAG_PATH
&& let Some([xmin, ymin, xmax, ymax]) = object.bounds
{
let vertices = XY::points(object_xy(object)).len();
if vertices > POINTS_PER_LINE {
let heading = object_heading(object, "path");
let width = object
.width
.map_or_else(|| "absent".to_string(), |width| format!("{width} dbu"));
writeln!(
out,
" {sign} {heading} · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu"
)
.unwrap();
return;
}
}
writeln!(out, " {sign} {object}").unwrap();
}
fn vertex_count(xy: &[u8]) -> usize {
XY::points(xy).len().saturating_sub(1)
}
@ -147,7 +171,7 @@ struct ShapeCounts {
impl ShapeCounts {
fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) {
let is_rectangle =
|object: &ObjectChange| XY::closed_rectangle(polygon_xy(object)).is_some();
|object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some();
match object.kind {
RTAG_BOX => self.boxes += 1,
_ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => {
@ -173,7 +197,7 @@ impl PolygonChanges {
pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) {
let counts = self.0.entry(old.layer).or_default();
let moved = old.properties == new.properties
&& translation(polygon_xy(old), polygon_xy(new)).is_some()
&& translation(object_xy(old), object_xy(new)).is_some()
&& field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY);
let shapes = if moved {
&mut counts.moved
@ -237,6 +261,10 @@ pub(super) struct ReferenceCounts {
}
impl ReferenceCounts {
pub fn len(&self) -> usize {
self.singles + self.arrays
}
pub fn add(&mut self, object: &ObjectChange) {
if object.kind == RTAG_AREF {
self.arrays += 1;
@ -272,8 +300,8 @@ impl ReferenceCounts {
}
fn polygon_replacement(old: &ObjectChange, new: &ObjectChange) -> String {
let before = polygon_xy(old);
let after = polygon_xy(new);
let before = object_xy(old);
let after = object_xy(new);
let old_size = rectangle_size(before);
let new_size = rectangle_size(after);
let mut out = polygon_heading(old, old_size.is_some() && new_size.is_some());

218
src/review/pair.rs Normal file
View file

@ -0,0 +1,218 @@
//! Adaptive parsing for unscoped comparisons of two revisions.
use super::*;
use std::{sync::Mutex, thread};
struct CellSpan<'a> {
body: &'a [u8],
end_record: usize,
}
struct FramedRevision<'a> {
records: Vec<Record<'a>>,
spans: Option<BTreeMap<&'a str, CellSpan<'a>>>,
}
impl<'a> FramedRevision<'a> {
fn read(input: &'a [u8]) -> Result<Self, String> {
let mut markers = Vec::new();
let records = read_records_with(input, |index, record| {
if matches!(record.tag, RTAG_BGNSTR | RTAG_ENDSTR) {
markers.push(index);
}
})?;
let spans = index_cells(input, &records, markers);
Ok(Self { records, spans })
}
}
// Failure to index a layout leaves error reporting to the regular parser.
fn index_cells<'a>(
input: &'a [u8],
records: &[Record<'a>],
markers: Vec<usize>,
) -> Option<BTreeMap<&'a str, CellSpan<'a>>> {
let mut cells = BTreeMap::new();
let mut start = None;
for index in markers {
let record = &records[index];
match record.tag {
RTAG_BGNSTR => {
if start.is_some() {
return None;
}
let name_record = records.get(index + 1).filter(|r| r.tag == RTAG_STRNAME)?;
let name = decode_name(name_record.data)
.ok()
.filter(|name| !name.is_empty())?;
start = Some((name, name_record.offset));
}
RTAG_ENDSTR => {
let (name, begin) = start.take()?;
let body = &input[begin..record.offset + 4];
let span = CellSpan {
body,
end_record: index,
};
if cells.insert(name, span).is_some() {
return None;
}
}
_ => (),
}
}
start.is_none().then_some(cells)
}
fn matching_cells<'a>(
old: &FramedRevision<'a>,
new: &FramedRevision<'a>,
input_size: usize,
) -> BTreeMap<&'a str, usize> {
let mut matches = BTreeMap::new();
if let (Some(old), Some(new)) = (&old.spans, &new.spans) {
let mut bytes = 0;
for (&name, cell) in new {
if old.get(name).is_some_and(|old| old.body == cell.body) {
bytes += cell.body.len();
matches.insert(name, cell.end_record);
}
}
// Reuse serializes some parsing work, so require substantial overlap.
if bytes < input_size - input_size / 10 {
matches.clear();
}
}
matches
}
pub(super) struct CellReuse<'a, 'b> {
old: &'b Document<'a>,
matches: &'b BTreeMap<&'a str, usize>,
}
impl<'a> CellReuse<'a, '_> {
pub(super) fn cell(
&self,
name: &str,
offset: usize,
records: &[Record<'a>],
) -> Option<(Cell<'a>, usize)> {
let end = *self.matches.get(name)?;
let previous = &self.old.cells[name];
// Re-run library metadata and reference checks with the new offsets.
let objects = previous
.objects
.iter()
.filter(|object| object.target.is_some() || object.kind == RTAG_TEXT)
.cloned()
.map(|mut object| {
object.offset = offset + (object.offset - previous.offset);
object.end_offset = offset + (object.end_offset - previous.offset);
object
})
.collect();
Some((
Cell {
offset,
end_offset: records[end].offset,
objects,
attributes: previous.attributes.clone(),
issues: Vec::new(),
},
end,
))
}
}
/// Two unscoped GDSII revisions prepared for comparison.
#[derive(Debug)]
pub struct RevisionPair<'a> {
old: Document<'a>,
new: Document<'a>,
reused_cells: BTreeSet<&'a str>,
}
impl<'a> RevisionPair<'a> {
pub fn parse(old: &'a [u8], new: &'a [u8]) -> Result<Self, Diagnostic> {
let invalid = |message, side| Diagnostic::new("invalid_input", message, Some(side));
let input_size = old.len().max(new.len());
let parallel = thread::available_parallelism().is_ok_and(|count| count.get() > 1);
let (old, new) = run_together(parallel, old, new, FramedRevision::read);
let old = old.map_err(|message| invalid(message, Side::Old))?;
let new = match new {
Ok(new) => new,
Err(message) => {
Document::parse_records(&old.records, None)
.map_err(|message| invalid(message, Side::Old))?;
return Err(invalid(message, Side::New));
}
};
let mut matches = matching_cells(&old, &new, input_size);
let (old, new) = if matches.is_empty() {
let (old, new) = run_together(parallel, old, new, |revision| {
Document::parse_records(&revision.records, None)
});
(old.map_err(|message| invalid(message, Side::Old))?, new)
} else {
let old_doc = Document::parse_records(&old.records, None)
.map_err(|message| invalid(message, Side::Old))?;
drop(old);
// The regular parser gives unsupported records their own offsets
// and messages, so parse those cells independently on each side.
matches.retain(|name, _| {
old_doc.cells.get(*name).is_some_and(|cell| {
cell.issues.is_empty()
&& cell.objects.iter().all(|object| object.issues.is_empty())
})
});
let reuse = CellReuse {
old: &old_doc,
matches: &matches,
};
let new = Document::parse_records(&new.records, Some(reuse));
(old_doc, new)
};
Ok(Self {
old,
new: new.map_err(|message| invalid(message, Side::New))?,
reused_cells: matches.into_keys().collect(),
})
}
pub fn compare(self) -> Comparison {
Comparison::collect(&self.old, &self.new, &self.reused_cells, None)
}
pub fn compare_text(self) -> TextComparison {
TextComparison::collect(&self.old, &self.new, &self.reused_cells)
}
}
fn run_together<I: Send, T: Send>(
parallel: bool,
old: I,
new: I,
process: impl Fn(I) -> T + Copy + Send,
) -> (T, T) {
if !parallel {
return (process(old), process(new));
}
// Transfer ownership only once a worker starts, so a spawn failure can
// process the same input on the calling thread.
let input = Mutex::new(Some(old));
let run_old = {
let input = &input;
move || {
let old = input.lock().unwrap().take().unwrap();
process(old)
}
};
thread::scope(|scope| {
if let Ok(worker) = thread::Builder::new().spawn_scoped(scope, run_old) {
let new = process(new);
return (worker.join().unwrap(), new);
}
(run_old(), process(new))
})
}

View file

@ -1,6 +1,11 @@
//! Versioned machine-readable diff reports. No consumer needs to parse prose.
use super::*;
use serde::{
Serialize, Serializer,
ser::{SerializeMap, SerializeSeq},
};
use serde_json::{Value, json};
use std::io;
use std::path::{Path, PathBuf};
pub const SCHEMA_VERSION: u32 = 1;
@ -196,10 +201,127 @@ fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec<Value> {
.collect()
}
struct SelectedObjects<'a> {
// Up to the last emitted object. A layer filter can leave gaps in this prefix.
objects: &'a [ObjectChange],
layer: Option<Layer>,
cell_index: usize,
side: &'static str,
}
impl SelectedObjects<'_> {
fn id(&self, index: usize) -> String {
format!("c{}:{}:{index}", self.cell_index, self.side)
}
fn contains(&self, index: usize) -> bool {
self.objects
.get(index)
.is_some_and(|object| self.layer.is_none_or(|layer| object.layer == Some(layer)))
}
}
impl Serialize for SelectedObjects<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut sequence = serializer.serialize_seq(None)?;
for (index, change) in self.objects.iter().enumerate() {
if self.layer.is_none_or(|layer| change.layer == Some(layer)) {
sequence.serialize_element(&object(change, &self.id(index), self.side))?;
}
}
sequence.end()
}
}
struct SelectedCell<'a> {
cell: &'a CellChange,
removed: SelectedObjects<'a>,
added: SelectedObjects<'a>,
}
impl Serialize for SelectedCell<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let revision = |revision: &CellRevision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)});
let mut map = serializer.serialize_map(Some(7))?;
map.serialize_entry("added", &self.added)?;
map.serialize_entry("after", &self.cell.new.as_ref().map(revision))?;
map.serialize_entry("attributes_changed", &self.cell.attributes_changed())?;
map.serialize_entry("before", &self.cell.old.as_ref().map(revision))?;
map.serialize_entry("name", &self.cell.name)?;
map.serialize_entry("removed", &self.removed)?;
map.serialize_entry("replacements", &SelectedReplacements(self))?;
map.end()
}
}
struct SelectedReplacements<'a>(&'a SelectedCell<'a>);
impl Serialize for SelectedReplacements<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let selection = self.0;
let mut sequence = serializer.serialize_seq(None)?;
for replacement in &selection.cell.replacements {
if selection.removed.contains(replacement.removed)
&& selection.added.contains(replacement.added)
{
let old = &selection.cell.removed[replacement.removed];
let new = &selection.cell.added[replacement.added];
sequence.serialize_element(&json!({
"removed": selection.removed.id(replacement.removed),
"added": selection.added.id(replacement.added),
"match_rule": replacement.match_rule,
"explanation": changes::replacement_summary(old, new),
"field_changes": field_changes(old, new),
"properties_changed": old.properties != new.properties
}))?;
}
}
sequence.end()
}
}
struct JsonReport<'a> {
metadata: Value,
cells: Vec<SelectedCell<'a>>,
}
impl Serialize for JsonReport<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let metadata = self
.metadata
.as_object()
.expect("report metadata is an object");
let mut map = serializer.serialize_map(Some(metadata.len()))?;
for (key, value) in metadata {
if key == "cell_changes" {
map.serialize_entry(key, &self.cells)?;
} else {
map.serialize_entry(key, value)?;
}
}
map.end()
}
}
impl Comparison {
/// Render the captured comparison evidence without retaining its source documents.
/// IDs are local to this report, not persistent object identities.
pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value {
serde_json::to_value(self.prepare_report(inputs, options))
.expect("serialize comparison report")
}
/// Stream a pretty-printed report to a writer.
pub fn write_json_report(
&self,
writer: impl io::Write,
inputs: &ReportInputs,
options: &ReportOptions,
) -> serde_json::Result<()> {
serde_json::to_writer_pretty(writer, &self.prepare_report(inputs, options))
}
fn prepare_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> JsonReport<'_> {
let outcome = self.outcome();
let mut report = envelope(
inputs,
@ -257,12 +379,8 @@ impl Comparison {
filtered_objects += cell.removed.len() + cell.added.len();
continue;
}
let (mut removed, mut added) = (Vec::new(), Vec::new());
let mut shown = BTreeSet::new();
for (side, objects, output) in [
("old", &cell.removed, &mut removed),
("new", &cell.added, &mut added),
] {
let mut select = |objects: &[ObjectChange]| {
let mut end = 0;
for (index, change) in objects.iter().enumerate() {
if options
.filter
@ -273,32 +391,29 @@ impl Comparison {
} else if options.max_objects.is_some_and(|max| emitted >= max) {
limited += 1;
} else {
let id = format!("c{cell_index}:{side}:{index}");
output.push(object(change, &id, side));
shown.insert(id);
end = index + 1;
emitted += 1;
}
}
}
let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| {
let a = format!("c{cell_index}:old:{}", r.removed);
let b = format!("c{cell_index}:new:{}", r.added);
(shown.contains(&a) && shown.contains(&b)).then(|| json!({
"removed": a, "added": b, "match_rule": r.match_rule,
"explanation": changes::replacement_summary(&cell.removed[r.removed], &cell.added[r.added]),
"field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]),
"properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties
}))
}).collect();
cells.push(json!({
"name": cell.name,
"before": cell.old.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})),
"after": cell.new.as_ref().map(|revision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)})),
"attributes_changed": cell.attributes_changed(),
"removed": removed, "added": added, "replacements": replacements
}));
end
};
let layer = options.filter.layer;
cells.push(SelectedCell {
cell,
removed: SelectedObjects {
objects: &cell.removed[..select(&cell.removed)],
layer,
cell_index,
side: "old",
},
added: SelectedObjects {
objects: &cell.added[..select(&cell.added)],
layer,
cell_index,
side: "new",
},
});
}
report["cell_changes"] = json!(cells);
report["report_selection"]["truncated"] =
json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0);
for (key, count) in [
@ -313,7 +428,10 @@ impl Comparison {
if options.hierarchy {
report["hierarchy"] = graph(self);
}
report
JsonReport {
metadata: report,
cells,
}
}
}