diff --git a/Cargo.lock b/Cargo.lock index 3a525b0..4220cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.11" @@ -156,12 +162,31 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -254,6 +279,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num" version = "0.4.3" @@ -340,6 +375,7 @@ version = "0.3.0" dependencies = [ "arrow-array", "arrow-schema", + "flate2", "serde_json", ] @@ -411,6 +447,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "syn" version = "2.0.100" diff --git a/Cargo.toml b/Cargo.toml index 49e7ad5..c40323a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] arrow-array = {version = "^54", features = ["ffi"]} arrow-schema = "^54" +flate2 = "1" serde_json = "1" diff --git a/docs/cli.md b/docs/cli.md index e01228d..95ea2cf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,8 @@ # Command-line tools Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. -Use uncompressed `.gds` files with the CLI. It runs independently of Python. +The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets +refer to the decompressed layout. The CLI runs independently of Python. ## Install diff --git a/docs/diff-reports.md b/docs/diff-reports.md index 3408ea5..ef870a5 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -86,7 +86,8 @@ hex preserves their bytes. Evidence follows the comparison profile: boundary/box rings are canonicalized, string padding is removed, and fields/properties are sorted. REAL8 encodings and -explicit defaults remain significant. Source offsets locate original records. +explicit defaults remain significant. For gzip inputs, source offsets locate +records in the decompressed layout. Replacements link report-local removed/added IDs with `match_rule`, `field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, diff --git a/docs/layer-names.md b/docs/layer-names.md index d006037..505b08d 100644 --- a/docs/layer-names.md +++ b/docs/layer-names.md @@ -16,6 +16,8 @@ cell as the carrier. The carrier must contain no ordinary design objects. The output path must be new; Pheon refuses to overwrite any existing file, including the input. +The output is uncompressed GDSII, including when the input is gzip-compressed. + The command consolidates the legend into that cell. Existing mappings remain unless you override them with `--name`. It creates no cells or references, and copies other records, timestamps, unknown records, and trailing padding byte for diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs index 1d514b7..80145b4 100644 --- a/src/bin/pheon.rs +++ b/src/bin/pheon.rs @@ -1,3 +1,4 @@ +use flate2::read::MultiGzDecoder; use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::{ Comparison, Diagnostic, Document, Outcome, Side, dump_records, @@ -8,7 +9,7 @@ use std::{ collections::BTreeMap, ffi::{OsStr, OsString}, fs, - io::{self, Write}, + io::{self, Read, Write}, path::PathBuf, process::ExitCode, }; @@ -21,9 +22,36 @@ const HELP: &str = "Usage: Structural comparison: ignore timestamps and element order, normalize polygon start/winding, preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Gzip-compressed GDSII input is recognized by its header. Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. "; +fn read_input(path: &OsStr) -> Result, Diagnostic> { + let io_error = + |error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None); + let mut file = fs::File::open(path).map_err(io_error)?; + let mut bytes = Vec::new(); + (&mut file) + .take(2) + .read_to_end(&mut bytes) + .map_err(io_error)?; + if bytes == b"\x1f\x8b" { + let mut decoded = Vec::new(); + MultiGzDecoder::new(bytes.as_slice().chain(file)) + .read_to_end(&mut decoded) + .map_err(|error| { + Diagnostic::new( + "invalid_input", + format!("{path:?}: gzip decompression failed: {error}"), + None, + ) + })?; + return Ok(decoded); + } + file.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + fn value(args: &[OsString], index: &mut usize) -> Result { *index += 1; args.get(*index) @@ -61,7 +89,7 @@ fn name_layers(args: &[OsString]) -> Result { if names.is_empty() { return Err("provide at least one --name".to_string()); } - let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; + let input = read_input(&args[0]).map_err(|error| error.message)?; let output = write_layer_names( &input, &cell.ok_or("provide --cell")?, @@ -101,8 +129,9 @@ fn prepare_comparison( ) -> Result { let read = |path: Option<&OsStr>, side| { path.map(|path| { - fs::read(path).map_err(|error| { - Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side)) + read_input(path).map_err(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic }) }) .transpose() @@ -267,7 +296,7 @@ fn run(args: &[OsString]) -> Result { { return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); } - let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); + let read = |path: &OsString| read_input(path).map_err(|error| error.message); if raw { if scope != Scope::default() { return Err("record dumps do not accept object scope selectors".to_string()); diff --git a/src/review.rs b/src/review.rs index 8a63291..9bb17a8 100644 --- a/src/review.rs +++ b/src/review.rs @@ -500,6 +500,16 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result Result>, String> { + if input.starts_with(b"\x1f\x8b") { + return Err(error( + 0, + "gzip-compressed input. Decompress it before reading GDSII", + )); + } + if input.starts_with(b"%SEMI-OASIS\r\n") { + return Err(error(0, "OASIS input. Export the layout as GDSII")); + } + let mut records = Vec::new(); let mut offset = 0; while offset < input.len() { diff --git a/tests/review.rs b/tests/review.rs index 3f49221..979c391 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -43,6 +43,14 @@ fn layout(cells: &[(&str, Vec>)]) -> Vec { bytes } +fn gzip(data: &[u8]) -> Vec { + use std::io::Write; + + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + fn polygon(points: &[[i32; 2]]) -> Vec { let mut bytes = Vec::new(); record(&mut bytes, RTAG_BOUNDARY, &[]); @@ -283,6 +291,21 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { } } +#[test] +fn review_identifies_gzip_and_oasis_input() { + for (bytes, expected) in [ + (&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"), + (&b"%SEMI-OASIS\r\n"[..], "OASIS input"), + ] { + let error = Document::parse(bytes).unwrap_err(); + assert!( + error.starts_with(&format!("at byte 0: {expected}")), + "{error}" + ); + assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error); + } +} + #[test] fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { let bytes = layout(&[("CHIP", vec![square()])]); @@ -691,6 +714,73 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { ); } +#[test] +fn cli_decodes_gzip_by_header_and_checks_the_trailer() { + let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let plain = directory.join("plain.gds"); + let compressed = directory.join("compressed.gds"); + let data = shared(square()); + // Splitting a record between gzip members must preserve the decompressed layout. + let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat(); + fs::write(&plain, &data).unwrap(); + fs::write(&compressed, &encoded).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let dump = |path: &PathBuf| { + let output = command() + .arg("dump") + .arg(path) + .arg("--records") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout + }; + assert_eq!(dump(&plain), dump(&compressed)); + let compare = || { + let output = command() + .arg("diff") + .arg(&plain) + .arg(&compressed) + .args(["--format", "json"]) + .output() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + (output.status.code(), report) + }; + let (code, report) = compare(); + assert_eq!(code, Some(0)); + assert_eq!(report["result"], "equal"); + + // Corrupt the last CRC after all the GDSII bytes have already been decoded. + let crc = encoded.len() - 8; + encoded[crc] ^= 0xff; + fs::write(&compressed, encoded).unwrap(); + let (code, report) = compare(); + assert_eq!(code, Some(2)); + assert_eq!(report["status"], "failed"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "invalid_input"); + assert_eq!(diagnostic["side"], "new"); + assert!( + diagnostic["message"] + .as_str() + .unwrap() + .contains("gzip decompression failed") + ); +} + #[test] fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { use pheon::review::names::write_layer_names; @@ -776,9 +866,9 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); - let source = directory.join("source.gds"); + let source = directory.join("source.gds.gz"); let output = directory.join("output.gds"); - let bytes = shared(square()); + let bytes = gzip(&shared(square())); fs::write(&source, &bytes).unwrap(); let _ = fs::remove_file(&output); let mut command = Command::new(env!("CARGO_BIN_EXE_pheon"));