Read gzip-compressed GDSII directly in the CLI

Detect gzip headers on every CLI input and decode all members before parsing.
Keep report offsets relative to the decompressed stream and report corrupt
archives as invalid input. Recognize OASIS headers and explain how to provide
GDSII data.

Cover mixed compressed and plain comparisons, corrupt trailers, and layer-name
output from compressed input.
This commit is contained in:
Jan Petykiewicz 2026-11-17 16:14:53 -08:00
commit 6e4b371046
8 changed files with 185 additions and 9 deletions

42
Cargo.lock generated
View file

@ -2,6 +2,12 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 4 version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]] [[package]]
name = "ahash" name = "ahash"
version = "0.8.11" version = "0.8.11"
@ -156,12 +162,31 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "crunchy" name = "crunchy"
version = "0.2.3" version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.15" version = "0.2.15"
@ -254,6 +279,16 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]] [[package]]
name = "num" name = "num"
version = "0.4.3" version = "0.4.3"
@ -340,6 +375,7 @@ version = "0.3.0"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-schema", "arrow-schema",
"flate2",
"serde_json", "serde_json",
] ]
@ -411,6 +447,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "simd-adler32"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.100" version = "2.0.100"

View file

@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies] [dependencies]
arrow-array = {version = "^54", features = ["ffi"]} arrow-array = {version = "^54", features = ["ffi"]}
arrow-schema = "^54" arrow-schema = "^54"
flate2 = "1"
serde_json = "1" serde_json = "1"

View file

@ -1,7 +1,8 @@
# Command-line tools # Command-line tools
Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references.
Use uncompressed `.gds` files with the CLI. It runs independently of Python. The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets
refer to the decompressed layout. The CLI runs independently of Python.
## Install ## Install

View file

@ -86,7 +86,8 @@ hex preserves their bytes.
Evidence follows the comparison profile: boundary/box rings are canonicalized, Evidence follows the comparison profile: boundary/box rings are canonicalized,
string padding is removed, and fields/properties are sorted. REAL8 encodings and string padding is removed, and fields/properties are sorted. REAL8 encodings and
explicit defaults remain significant. Source offsets locate original records. explicit defaults remain significant. For gzip inputs, source offsets locate
records in the decompressed layout.
Replacements link report-local removed/added IDs with `match_rule`, Replacements link report-local removed/added IDs with `match_rule`,
`field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`, `field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`,

View file

@ -16,6 +16,8 @@ cell as the carrier. The carrier must contain no ordinary design objects.
The output path must be new; Pheon refuses to overwrite any existing file, The output path must be new; Pheon refuses to overwrite any existing file,
including the input. including the input.
The output is uncompressed GDSII, including when the input is gzip-compressed.
The command consolidates the legend into that cell. Existing mappings remain The command consolidates the legend into that cell. Existing mappings remain
unless you override them with `--name`. It creates no cells or references, and unless you override them with `--name`. It creates no cells or references, and
copies other records, timestamps, unknown records, and trailing padding byte for copies other records, timestamps, unknown records, and trailing padding byte for

View file

@ -1,3 +1,4 @@
use flate2::read::MultiGzDecoder;
use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; use pheon::review::report::{ReportInputs, ReportOptions, failure_report};
use pheon::review::{ use pheon::review::{
Comparison, Diagnostic, Document, Outcome, Side, dump_records, Comparison, Diagnostic, Document, Outcome, Side, dump_records,
@ -8,7 +9,7 @@ use std::{
collections::BTreeMap, collections::BTreeMap,
ffi::{OsStr, OsString}, ffi::{OsStr, OsString},
fs, fs,
io::{self, Write}, io::{self, Read, Write},
path::PathBuf, path::PathBuf,
process::ExitCode, process::ExitCode,
}; };
@ -21,9 +22,36 @@ const HELP: &str = "Usage:
Structural comparison: ignore timestamps and element order, normalize polygon start/winding, Structural comparison: ignore timestamps and element order, normalize polygon start/winding,
preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance.
Gzip-compressed GDSII input is recognized by its header.
Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete.
"; ";
fn read_input(path: &OsStr) -> Result<Vec<u8>, Diagnostic> {
let io_error =
|error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None);
let mut file = fs::File::open(path).map_err(io_error)?;
let mut bytes = Vec::new();
(&mut file)
.take(2)
.read_to_end(&mut bytes)
.map_err(io_error)?;
if bytes == b"\x1f\x8b" {
let mut decoded = Vec::new();
MultiGzDecoder::new(bytes.as_slice().chain(file))
.read_to_end(&mut decoded)
.map_err(|error| {
Diagnostic::new(
"invalid_input",
format!("{path:?}: gzip decompression failed: {error}"),
None,
)
})?;
return Ok(decoded);
}
file.read_to_end(&mut bytes).map_err(io_error)?;
Ok(bytes)
}
fn value(args: &[OsString], index: &mut usize) -> Result<String, String> { fn value(args: &[OsString], index: &mut usize) -> Result<String, String> {
*index += 1; *index += 1;
args.get(*index) args.get(*index)
@ -61,7 +89,7 @@ fn name_layers(args: &[OsString]) -> Result<u8, String> {
if names.is_empty() { if names.is_empty() {
return Err("provide at least one --name".to_string()); return Err("provide at least one --name".to_string());
} }
let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; let input = read_input(&args[0]).map_err(|error| error.message)?;
let output = write_layer_names( let output = write_layer_names(
&input, &input,
&cell.ok_or("provide --cell")?, &cell.ok_or("provide --cell")?,
@ -101,8 +129,9 @@ fn prepare_comparison(
) -> Result<Comparison, Diagnostic> { ) -> Result<Comparison, Diagnostic> {
let read = |path: Option<&OsStr>, side| { let read = |path: Option<&OsStr>, side| {
path.map(|path| { path.map(|path| {
fs::read(path).map_err(|error| { read_input(path).map_err(|mut diagnostic| {
Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side)) diagnostic.side = Some(side);
diagnostic
}) })
}) })
.transpose() .transpose()
@ -267,7 +296,7 @@ fn run(args: &[OsString]) -> Result<u8, String> {
{ {
return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); return Err("format must be ascii or mermaid (diff also accepts none)".to_string());
} }
let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); let read = |path: &OsString| read_input(path).map_err(|error| error.message);
if raw { if raw {
if scope != Scope::default() { if scope != Scope::default() {
return Err("record dumps do not accept object scope selectors".to_string()); return Err("record dumps do not accept object scope selectors".to_string());

View file

@ -500,6 +500,16 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
} }
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> { fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
if input.starts_with(b"\x1f\x8b") {
return Err(error(
0,
"gzip-compressed input. Decompress it before reading GDSII",
));
}
if input.starts_with(b"%SEMI-OASIS\r\n") {
return Err(error(0, "OASIS input. Export the layout as GDSII"));
}
let mut records = Vec::new(); let mut records = Vec::new();
let mut offset = 0; let mut offset = 0;
while offset < input.len() { while offset < input.len() {

View file

@ -43,6 +43,14 @@ fn layout(cells: &[(&str, Vec<Vec<u8>>)]) -> Vec<u8> {
bytes bytes
} }
fn gzip(data: &[u8]) -> Vec<u8> {
use std::io::Write;
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
fn polygon(points: &[[i32; 2]]) -> Vec<u8> { fn polygon(points: &[[i32; 2]]) -> Vec<u8> {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
record(&mut bytes, RTAG_BOUNDARY, &[]); record(&mut bytes, RTAG_BOUNDARY, &[]);
@ -283,6 +291,21 @@ fn ten_fixture_pairs_have_explicit_expected_changes() {
} }
} }
#[test]
fn review_identifies_gzip_and_oasis_input() {
for (bytes, expected) in [
(&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"),
(&b"%SEMI-OASIS\r\n"[..], "OASIS input"),
] {
let error = Document::parse(bytes).unwrap_err();
assert!(
error.starts_with(&format!("at byte 0: {expected}")),
"{error}"
);
assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error);
}
}
#[test] #[test]
fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { fn canonical_polygon_dump_is_readable_and_offsets_are_separate() {
let bytes = layout(&[("CHIP", vec![square()])]); let bytes = layout(&[("CHIP", vec![square()])]);
@ -691,6 +714,73 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() {
); );
} }
#[test]
fn cli_decodes_gzip_by_header_and_checks_the_trailer() {
let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id()));
fs::create_dir_all(&directory).unwrap();
struct Cleanup(PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(directory.clone());
let plain = directory.join("plain.gds");
let compressed = directory.join("compressed.gds");
let data = shared(square());
// Splitting a record between gzip members must preserve the decompressed layout.
let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat();
fs::write(&plain, &data).unwrap();
fs::write(&compressed, &encoded).unwrap();
let command = || Command::new(env!("CARGO_BIN_EXE_pheon"));
let dump = |path: &PathBuf| {
let output = command()
.arg("dump")
.arg(path)
.arg("--records")
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
output.stdout
};
assert_eq!(dump(&plain), dump(&compressed));
let compare = || {
let output = command()
.arg("diff")
.arg(&plain)
.arg(&compressed)
.args(["--format", "json"])
.output()
.unwrap();
let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
(output.status.code(), report)
};
let (code, report) = compare();
assert_eq!(code, Some(0));
assert_eq!(report["result"], "equal");
// Corrupt the last CRC after all the GDSII bytes have already been decoded.
let crc = encoded.len() - 8;
encoded[crc] ^= 0xff;
fs::write(&compressed, encoded).unwrap();
let (code, report) = compare();
assert_eq!(code, Some(2));
assert_eq!(report["status"], "failed");
let diagnostic = &report["diagnostics"][0];
assert_eq!(diagnostic["code"], "invalid_input");
assert_eq!(diagnostic["side"], "new");
assert!(
diagnostic["message"]
.as_str()
.unwrap()
.contains("gzip decompression failed")
);
}
#[test] #[test]
fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() {
use pheon::review::names::write_layer_names; use pheon::review::names::write_layer_names;
@ -776,9 +866,9 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() {
fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() {
let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id()));
fs::create_dir_all(&directory).unwrap(); fs::create_dir_all(&directory).unwrap();
let source = directory.join("source.gds"); let source = directory.join("source.gds.gz");
let output = directory.join("output.gds"); let output = directory.join("output.gds");
let bytes = shared(square()); let bytes = gzip(&shared(square()));
fs::write(&source, &bytes).unwrap(); fs::write(&source, &bytes).unwrap();
let _ = fs::remove_file(&output); let _ = fs::remove_file(&output);
let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); let mut command = Command::new(env!("CARGO_BIN_EXE_pheon"));