Read gzip-compressed GDSII directly in the CLI
Detect gzip headers on every CLI input and decode all members before parsing. Keep report offsets relative to the decompressed stream and report corrupt archives as invalid input. Recognize OASIS headers and explain how to provide GDSII data. Cover mixed compressed and plain comparisons, corrupt trailers, and layer-name output from compressed input.
This commit is contained in:
parent
58adf0cc99
commit
6e4b371046
8 changed files with 185 additions and 9 deletions
42
Cargo.lock
generated
42
Cargo.lock
generated
|
|
@ -2,6 +2,12 @@
|
||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "adler2"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ahash"
|
name = "ahash"
|
||||||
version = "0.8.11"
|
version = "0.8.11"
|
||||||
|
|
@ -156,12 +162,31 @@ version = "0.8.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc32fast"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crunchy"
|
name = "crunchy"
|
||||||
version = "0.2.3"
|
version = "0.2.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
|
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "flate2"
|
||||||
|
version = "1.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"miniz_oxide",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "getrandom"
|
name = "getrandom"
|
||||||
version = "0.2.15"
|
version = "0.2.15"
|
||||||
|
|
@ -254,6 +279,16 @@ version = "2.7.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.8.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||||
|
dependencies = [
|
||||||
|
"adler2",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num"
|
name = "num"
|
||||||
version = "0.4.3"
|
version = "0.4.3"
|
||||||
|
|
@ -340,6 +375,7 @@ version = "0.3.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
|
"flate2",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -411,6 +447,12 @@ version = "1.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "simd-adler32"
|
||||||
|
version = "0.3.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "2.0.100"
|
version = "2.0.100"
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"]
|
||||||
[dependencies]
|
[dependencies]
|
||||||
arrow-array = {version = "^54", features = ["ffi"]}
|
arrow-array = {version = "^54", features = ["ffi"]}
|
||||||
arrow-schema = "^54"
|
arrow-schema = "^54"
|
||||||
|
flate2 = "1"
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
# Command-line tools
|
# Command-line tools
|
||||||
|
|
||||||
Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references.
|
Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references.
|
||||||
Use uncompressed `.gds` files with the CLI. It runs independently of Python.
|
The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets
|
||||||
|
refer to the decompressed layout. The CLI runs independently of Python.
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,8 @@ hex preserves their bytes.
|
||||||
|
|
||||||
Evidence follows the comparison profile: boundary/box rings are canonicalized,
|
Evidence follows the comparison profile: boundary/box rings are canonicalized,
|
||||||
string padding is removed, and fields/properties are sorted. REAL8 encodings and
|
string padding is removed, and fields/properties are sorted. REAL8 encodings and
|
||||||
explicit defaults remain significant. Source offsets locate original records.
|
explicit defaults remain significant. For gzip inputs, source offsets locate
|
||||||
|
records in the decompressed layout.
|
||||||
|
|
||||||
Replacements link report-local removed/added IDs with `match_rule`,
|
Replacements link report-local removed/added IDs with `match_rule`,
|
||||||
`field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`,
|
`field_changes`, and `properties_changed`. Rules are `unique_kind_layer_bounds`,
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@ cell as the carrier. The carrier must contain no ordinary design objects.
|
||||||
The output path must be new; Pheon refuses to overwrite any existing file,
|
The output path must be new; Pheon refuses to overwrite any existing file,
|
||||||
including the input.
|
including the input.
|
||||||
|
|
||||||
|
The output is uncompressed GDSII, including when the input is gzip-compressed.
|
||||||
|
|
||||||
The command consolidates the legend into that cell. Existing mappings remain
|
The command consolidates the legend into that cell. Existing mappings remain
|
||||||
unless you override them with `--name`. It creates no cells or references, and
|
unless you override them with `--name`. It creates no cells or references, and
|
||||||
copies other records, timestamps, unknown records, and trailing padding byte for
|
copies other records, timestamps, unknown records, and trailing padding byte for
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
use flate2::read::MultiGzDecoder;
|
||||||
use pheon::review::report::{ReportInputs, ReportOptions, failure_report};
|
use pheon::review::report::{ReportInputs, ReportOptions, failure_report};
|
||||||
use pheon::review::{
|
use pheon::review::{
|
||||||
Comparison, Diagnostic, Document, Outcome, Side, dump_records,
|
Comparison, Diagnostic, Document, Outcome, Side, dump_records,
|
||||||
|
|
@ -8,7 +9,7 @@ use std::{
|
||||||
collections::BTreeMap,
|
collections::BTreeMap,
|
||||||
ffi::{OsStr, OsString},
|
ffi::{OsStr, OsString},
|
||||||
fs,
|
fs,
|
||||||
io::{self, Write},
|
io::{self, Read, Write},
|
||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
process::ExitCode,
|
process::ExitCode,
|
||||||
};
|
};
|
||||||
|
|
@ -21,9 +22,36 @@ const HELP: &str = "Usage:
|
||||||
|
|
||||||
Structural comparison: ignore timestamps and element order, normalize polygon start/winding,
|
Structural comparison: ignore timestamps and element order, normalize polygon start/winding,
|
||||||
preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance.
|
preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance.
|
||||||
|
Gzip-compressed GDSII input is recognized by its header.
|
||||||
Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete.
|
Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete.
|
||||||
";
|
";
|
||||||
|
|
||||||
|
fn read_input(path: &OsStr) -> Result<Vec<u8>, Diagnostic> {
|
||||||
|
let io_error =
|
||||||
|
|error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None);
|
||||||
|
let mut file = fs::File::open(path).map_err(io_error)?;
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
(&mut file)
|
||||||
|
.take(2)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(io_error)?;
|
||||||
|
if bytes == b"\x1f\x8b" {
|
||||||
|
let mut decoded = Vec::new();
|
||||||
|
MultiGzDecoder::new(bytes.as_slice().chain(file))
|
||||||
|
.read_to_end(&mut decoded)
|
||||||
|
.map_err(|error| {
|
||||||
|
Diagnostic::new(
|
||||||
|
"invalid_input",
|
||||||
|
format!("{path:?}: gzip decompression failed: {error}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
return Ok(decoded);
|
||||||
|
}
|
||||||
|
file.read_to_end(&mut bytes).map_err(io_error)?;
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
fn value(args: &[OsString], index: &mut usize) -> Result<String, String> {
|
fn value(args: &[OsString], index: &mut usize) -> Result<String, String> {
|
||||||
*index += 1;
|
*index += 1;
|
||||||
args.get(*index)
|
args.get(*index)
|
||||||
|
|
@ -61,7 +89,7 @@ fn name_layers(args: &[OsString]) -> Result<u8, String> {
|
||||||
if names.is_empty() {
|
if names.is_empty() {
|
||||||
return Err("provide at least one --name".to_string());
|
return Err("provide at least one --name".to_string());
|
||||||
}
|
}
|
||||||
let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?;
|
let input = read_input(&args[0]).map_err(|error| error.message)?;
|
||||||
let output = write_layer_names(
|
let output = write_layer_names(
|
||||||
&input,
|
&input,
|
||||||
&cell.ok_or("provide --cell")?,
|
&cell.ok_or("provide --cell")?,
|
||||||
|
|
@ -101,8 +129,9 @@ fn prepare_comparison(
|
||||||
) -> Result<Comparison, Diagnostic> {
|
) -> Result<Comparison, Diagnostic> {
|
||||||
let read = |path: Option<&OsStr>, side| {
|
let read = |path: Option<&OsStr>, side| {
|
||||||
path.map(|path| {
|
path.map(|path| {
|
||||||
fs::read(path).map_err(|error| {
|
read_input(path).map_err(|mut diagnostic| {
|
||||||
Diagnostic::new("io_error", format!("{path:?}: {error}"), Some(side))
|
diagnostic.side = Some(side);
|
||||||
|
diagnostic
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.transpose()
|
.transpose()
|
||||||
|
|
@ -267,7 +296,7 @@ fn run(args: &[OsString]) -> Result<u8, String> {
|
||||||
{
|
{
|
||||||
return Err("format must be ascii or mermaid (diff also accepts none)".to_string());
|
return Err("format must be ascii or mermaid (diff also accepts none)".to_string());
|
||||||
}
|
}
|
||||||
let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}"));
|
let read = |path: &OsString| read_input(path).map_err(|error| error.message);
|
||||||
if raw {
|
if raw {
|
||||||
if scope != Scope::default() {
|
if scope != Scope::default() {
|
||||||
return Err("record dumps do not accept object scope selectors".to_string());
|
return Err("record dumps do not accept object scope selectors".to_string());
|
||||||
|
|
|
||||||
|
|
@ -500,6 +500,16 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec<Diagnostic>) -> Result<Ob
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
|
fn read_records(input: &[u8]) -> Result<Vec<Record<'_>>, String> {
|
||||||
|
if input.starts_with(b"\x1f\x8b") {
|
||||||
|
return Err(error(
|
||||||
|
0,
|
||||||
|
"gzip-compressed input. Decompress it before reading GDSII",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if input.starts_with(b"%SEMI-OASIS\r\n") {
|
||||||
|
return Err(error(0, "OASIS input. Export the layout as GDSII"));
|
||||||
|
}
|
||||||
|
|
||||||
let mut records = Vec::new();
|
let mut records = Vec::new();
|
||||||
let mut offset = 0;
|
let mut offset = 0;
|
||||||
while offset < input.len() {
|
while offset < input.len() {
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,14 @@ fn layout(cells: &[(&str, Vec<Vec<u8>>)]) -> Vec<u8> {
|
||||||
bytes
|
bytes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn gzip(data: &[u8]) -> Vec<u8> {
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||||
|
encoder.write_all(data).unwrap();
|
||||||
|
encoder.finish().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn polygon(points: &[[i32; 2]]) -> Vec<u8> {
|
fn polygon(points: &[[i32; 2]]) -> Vec<u8> {
|
||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
record(&mut bytes, RTAG_BOUNDARY, &[]);
|
record(&mut bytes, RTAG_BOUNDARY, &[]);
|
||||||
|
|
@ -283,6 +291,21 @@ fn ten_fixture_pairs_have_explicit_expected_changes() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn review_identifies_gzip_and_oasis_input() {
|
||||||
|
for (bytes, expected) in [
|
||||||
|
(&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"),
|
||||||
|
(&b"%SEMI-OASIS\r\n"[..], "OASIS input"),
|
||||||
|
] {
|
||||||
|
let error = Document::parse(bytes).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error.starts_with(&format!("at byte 0: {expected}")),
|
||||||
|
"{error}"
|
||||||
|
);
|
||||||
|
assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn canonical_polygon_dump_is_readable_and_offsets_are_separate() {
|
fn canonical_polygon_dump_is_readable_and_offsets_are_separate() {
|
||||||
let bytes = layout(&[("CHIP", vec![square()])]);
|
let bytes = layout(&[("CHIP", vec![square()])]);
|
||||||
|
|
@ -691,6 +714,73 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cli_decodes_gzip_by_header_and_checks_the_trailer() {
|
||||||
|
let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id()));
|
||||||
|
fs::create_dir_all(&directory).unwrap();
|
||||||
|
struct Cleanup(PathBuf);
|
||||||
|
impl Drop for Cleanup {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = fs::remove_dir_all(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _cleanup = Cleanup(directory.clone());
|
||||||
|
let plain = directory.join("plain.gds");
|
||||||
|
let compressed = directory.join("compressed.gds");
|
||||||
|
let data = shared(square());
|
||||||
|
// Splitting a record between gzip members must preserve the decompressed layout.
|
||||||
|
let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat();
|
||||||
|
fs::write(&plain, &data).unwrap();
|
||||||
|
fs::write(&compressed, &encoded).unwrap();
|
||||||
|
let command = || Command::new(env!("CARGO_BIN_EXE_pheon"));
|
||||||
|
let dump = |path: &PathBuf| {
|
||||||
|
let output = command()
|
||||||
|
.arg("dump")
|
||||||
|
.arg(path)
|
||||||
|
.arg("--records")
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"{}",
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
output.stdout
|
||||||
|
};
|
||||||
|
assert_eq!(dump(&plain), dump(&compressed));
|
||||||
|
let compare = || {
|
||||||
|
let output = command()
|
||||||
|
.arg("diff")
|
||||||
|
.arg(&plain)
|
||||||
|
.arg(&compressed)
|
||||||
|
.args(["--format", "json"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
|
||||||
|
(output.status.code(), report)
|
||||||
|
};
|
||||||
|
let (code, report) = compare();
|
||||||
|
assert_eq!(code, Some(0));
|
||||||
|
assert_eq!(report["result"], "equal");
|
||||||
|
|
||||||
|
// Corrupt the last CRC after all the GDSII bytes have already been decoded.
|
||||||
|
let crc = encoded.len() - 8;
|
||||||
|
encoded[crc] ^= 0xff;
|
||||||
|
fs::write(&compressed, encoded).unwrap();
|
||||||
|
let (code, report) = compare();
|
||||||
|
assert_eq!(code, Some(2));
|
||||||
|
assert_eq!(report["status"], "failed");
|
||||||
|
let diagnostic = &report["diagnostics"][0];
|
||||||
|
assert_eq!(diagnostic["code"], "invalid_input");
|
||||||
|
assert_eq!(diagnostic["side"], "new");
|
||||||
|
assert!(
|
||||||
|
diagnostic["message"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("gzip decompression failed")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() {
|
fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() {
|
||||||
use pheon::review::names::write_layer_names;
|
use pheon::review::names::write_layer_names;
|
||||||
|
|
@ -776,9 +866,9 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() {
|
||||||
fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() {
|
fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() {
|
||||||
let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id()));
|
let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id()));
|
||||||
fs::create_dir_all(&directory).unwrap();
|
fs::create_dir_all(&directory).unwrap();
|
||||||
let source = directory.join("source.gds");
|
let source = directory.join("source.gds.gz");
|
||||||
let output = directory.join("output.gds");
|
let output = directory.join("output.gds");
|
||||||
let bytes = shared(square());
|
let bytes = gzip(&shared(square()));
|
||||||
fs::write(&source, &bytes).unwrap();
|
fs::write(&source, &bytes).unwrap();
|
||||||
let _ = fs::remove_file(&output);
|
let _ = fs::remove_file(&output);
|
||||||
let mut command = Command::new(env!("CARGO_BIN_EXE_pheon"));
|
let mut command = Command::new(env!("CARGO_BIN_EXE_pheon"));
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue