From 6e4b37104669ee25e12d7e3ab5e0ba6c2f937df8 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 17 Nov 2026 16:14:53 -0800 Subject: [PATCH] Read gzip-compressed GDSII directly in the CLI Detect gzip headers on every CLI input and decode all members before parsing. Keep report offsets relative to the decompressed stream and report corrupt archives as invalid input. Recognize OASIS headers and explain how to provide GDSII data. Cover mixed compressed and plain comparisons, corrupt trailers, and layer-name output from compressed input. --- Cargo.lock | 42 ++++++++++++++++++++ Cargo.toml | 1 + docs/cli.md | 3 +- docs/diff-reports.md | 3 +- docs/layer-names.md | 2 + src/bin/pheon.rs | 39 +++++++++++++++--- src/review.rs | 10 +++++ tests/review.rs | 94 +++++++++++++++++++++++++++++++++++++++++++- 8 files changed, 185 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a525b0..4220cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.11" @@ -156,12 +162,31 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -254,6 +279,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num" version = "0.4.3" @@ -340,6 +375,7 @@ version = "0.3.0" dependencies = [ "arrow-array", "arrow-schema", + "flate2", "serde_json", ] @@ -411,6 +447,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "syn" version = "2.0.100" diff --git a/Cargo.toml b/Cargo.toml index 49e7ad5..c40323a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] arrow-array = {version = "^54", features = ["ffi"]} arrow-schema = "^54" +flate2 = "1" serde_json = "1" diff --git a/docs/cli.md b/docs/cli.md index e01228d..95ea2cf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,8 @@ # Command-line tools Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. -Use uncompressed `.gds` files with the CLI. It runs independently of Python. +The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets +refer to the decompressed layout. The CLI runs independently of Python. ## Install diff --git a/docs/diff-reports.md b/docs/diff-reports.md index 3408ea5..ef870a5 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -86,7 +86,8 @@ hex preserves their bytes. Evidence follows the comparison profile: boundary/box rings are canonicalized, string padding is removed, and fields/properties are sorted. REAL8 encodings and -explicit defaults remain significant. Source offsets locate original records. +explicit defaults remain significant. For gzip inputs, source offsets locate +records in the decompressed layout. Replacements link report-local removed/added IDs with `match_rule`, `field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, diff --git a/docs/layer-names.md b/docs/layer-names.md index d006037..505b08d 100644 --- a/docs/layer-names.md +++ b/docs/layer-names.md @@ -16,6 +16,8 @@ cell as the carrier. The carrier must contain no ordinary design objects. The output path must be new; Pheon refuses to overwrite any existing file, including the input. +The output is uncompressed GDSII, including when the input is gzip-compressed. + The command consolidates the legend into that cell. Existing mappings remain unless you override them with `--name`. It creates no cells or references, and copies other records, timestamps, unknown records, and trailing padding byte for diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 1d514b7..80145b4 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,3 +1,4 @@ +use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ Comparison, Diagnostic, Document, Outcome, Side, dump_records, @@ -8,7 +9,7 @@ use std::{ collections::BTreeMap, ffi::{OsStr, OsString}, fs, - io::{self, Write}, + io::{self, Read, Write}, path::PathBuf, process::ExitCode, }; @@ -21,9 +22,36 @@ const HELP: &str = "Usage: Structural comparison: ignore timestamps and element order, normalize polygon start/winding, preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Gzip-compressed GDSII input is recognized by its header. Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. "; +fn read_input(path: &OsStr) -> Result, Diagnostic> { + let io_error = + |error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None); + let mut file = fs::File::open(path).map_err(io_error)?; + let mut bytes = Vec::new(); + (&mut file) + .take(2) + .read_to_end(&mut bytes) + .map_err(io_error)?; + if bytes == b"\x1f\x8b" { + let mut decoded = Vec::new(); + MultiGzDecoder::new(bytes.as_slice().chain(file)) + .read_to_end(&mut decoded) + .map_err(|error| { + Diagnostic::new( + "invalid_input", + format!("{path:?}: gzip decompression failed: {error}"), + None, + ) + })?; + return Ok(decoded); + } + file.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + fn value(args: &[OsString], index: &mut usize) -> Result { *index += 1; args.get(*index) @@ -61,7 +89,7 @@ fn name_layers(args: &[OsString]) -> Result { if names.is_empty() { return Err("provide at least one --name".to_string()); } - let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; + let input = read_input(&args[0]).map_err(|error| error.message)?; let output = write_layer_names( &input, &cell.ok_or("provide --cell")?, @@ -101,8 +129,9 @@ fn prepare_comparison( ) -> Result { let read = |path: Option<&OsStr>, side| { path.map(|path| { - fs::read(path).map_err(|error| { - Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side)) + read_input(path).map_err(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic }) }) .transpose() @@ -267,7 +296,7 @@ fn run(args: &[OsString]) -> Result { { return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); } - let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); + let read = |path: &OsString| read_input(path).map_err(|error| error.message); if raw { if scope != Scope::default() { return Err("record dumps do not accept object scope selectors".to_string()); diff --git a/src/review.rs b/src/review.rs index 8a63291..9bb17a8 100644 --- a/src/review.rs +++ b/src/review.rs @@ -500,6 +500,16 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result Result>, String> { + if input.starts_with(b"\x1f\x8b") { + return Err(error( + 0, + "gzip-compressed input. Decompress it before reading GDSII", + )); + } + if input.starts_with(b"%SEMI-OASIS\r\n") { + return Err(error(0, "OASIS input. Export the layout as GDSII")); + } + let mut records = Vec::new(); let mut offset = 0; while offset < input.len() { diff --git a/tests/review.rs b/tests/review.rs index 3f49221..979c391 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -43,6 +43,14 @@ fn layout(cells: &[(&str, Vec>)]) -> Vec { bytes } +fn gzip(data: &[u8]) -> Vec { + use std::io::Write; + + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + fn polygon(points: &[[i32; 2]]) -> Vec { let mut bytes = Vec::new(); record(&mut bytes, RTAG_BOUNDARY, &[]); @@ -283,6 +291,21 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { } } +#[test] +fn review_identifies_gzip_and_oasis_input() { + for (bytes, expected) in [ + (&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"), + (&b"%SEMI-OASIS\r\n"[..], "OASIS input"), + ] { + let error = Document::parse(bytes).unwrap_err(); + assert!( + error.starts_with(&format!("at byte 0: {expected}")), + "{error}" + ); + assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error); + } +} + #[test] fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { let bytes = layout(&[("CHIP", vec![square()])]); @@ -691,6 +714,73 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { ); } +#[test] +fn cli_decodes_gzip_by_header_and_checks_the_trailer() { + let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let plain = directory.join("plain.gds"); + let compressed = directory.join("compressed.gds"); + let data = shared(square()); + // Splitting a record between gzip members must preserve the decompressed layout. + let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat(); + fs::write(&plain, &data).unwrap(); + fs::write(&compressed, &encoded).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let dump = |path: &PathBuf| { + let output = command() + .arg("dump") + .arg(path) + .arg("--records") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout + }; + assert_eq!(dump(&plain), dump(&compressed)); + let compare = || { + let output = command() + .arg("diff") + .arg(&plain) + .arg(&compressed) + .args(["--format", "json"]) + .output() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + (output.status.code(), report) + }; + let (code, report) = compare(); + assert_eq!(code, Some(0)); + assert_eq!(report["result"], "equal"); + + // Corrupt the last CRC after all the GDSII bytes have already been decoded. + let crc = encoded.len() - 8; + encoded[crc] ^= 0xff; + fs::write(&compressed, encoded).unwrap(); + let (code, report) = compare(); + assert_eq!(code, Some(2)); + assert_eq!(report["status"], "failed"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "invalid_input"); + assert_eq!(diagnostic["side"], "new"); + assert!( + diagnostic["message"] + .as_str() + .unwrap() + .contains("gzip decompression failed") + ); +} + #[test] fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { use pheon::review::names::write_layer_names; @@ -776,9 +866,9 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); - let source = directory.join("source.gds"); + let source = directory.join("source.gds.gz"); let output = directory.join("output.gds"); - let bytes = shared(square()); + let bytes = gzip(&shared(square())); fs::write(&source, &bytes).unwrap(); let _ = fs::remove_file(&output); let mut command = Command::new(env!("CARGO_BIN_EXE_pheon"));