Add a reproducible upstream GDSII corpus check

Pin 30 layouts from five upstream repositories and verify downloads and license
files by SHA-256. Exercise full reads, scans, selected reads, and CLI
self-comparisons with a per-layout JSON report. Cache the inputs locally for
offline reruns.

The baseline exposes a CLI rejection of KLayout's padded-name fixture. The
gzip fixture passes directly through the CLI.
This commit is contained in:
Jan Petykiewicz 2026-11-17 16:15:05 -08:00
commit d95c378690

167
tools/check_gds_corpus.py Normal file
View file

@ -0,0 +1,167 @@
"""Check a pinned open-source layout corpus against Pheon's installed readers."""
import argparse
import gzip
import hashlib
import json
import os
from pathlib import Path
import subprocess
import tomllib
from urllib.parse import quote
from pheon import arrow
ROOT = Path(__file__).resolve().parents[1]
def read_input(path, url, digest, fetch):
if path.is_file():
data = path.read_bytes()
elif fetch:
data = subprocess.check_output(
["curl", "-fLsS", "--retry", "2", "--max-time", "60", url]
)
else:
raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.")
if hashlib.sha256(data).hexdigest() != digest:
raise ValueError(f"SHA-256 mismatch for {path}")
if not path.is_file():
path.write_bytes(data)
return data
def inspect_layout(cli, path, data):
results = {}
reads = {
"full": lambda: arrow.read_path(path),
"scan": lambda: arrow.scan_bytes(data),
}
layouts = {}
for name, read in reads.items():
try:
layout = read()
layouts[name] = layout
results[name] = {
"status": "ok",
"cells": len(layout.field("cells")[0].values),
}
except (ValueError, OSError, TypeError) as error:
results[name] = {"status": "failed", "message": str(error)}
if "scan" in layouts:
cells = layouts["scan"].field("cells")[0].values
ranges = list(
zip(
cells.field("struct_start_offset").to_pylist(),
cells.field("struct_end_offset").to_pylist(),
strict=True,
)
)
try:
selected = arrow.read_cells_bytes(data, ranges)
if "full" in layouts and not selected.equals(layouts["full"]):
raise ValueError(
"Reading all scanned cell ranges differs from the full read"
)
results["selected"] = {
"status": "ok",
"cells": len(selected.field("cells")[0].values),
}
except (ValueError, OSError, TypeError) as error:
results["selected"] = {"status": "failed", "message": str(error)}
else:
results["selected"] = {"status": "skipped"}
try:
run = subprocess.run(
[cli, "diff", path, path, "--format", "json"],
capture_output=True,
timeout=60,
)
except (subprocess.TimeoutExpired, OSError) as error:
results["cli"] = {"status": "failed", "message": str(error)}
return results
try:
report = json.loads(run.stdout)
results["cli"] = {
"status": report["status"],
"result": report["result"],
"exit_code": run.returncode,
"diagnostics": report["diagnostics"],
}
if (report["status"], report["result"], run.returncode) not in {
("complete", "equal", 0),
("incomplete", "unknown", 2),
("failed", "unknown", 2),
}:
results["cli"]["status"] = "failed"
except (ValueError, KeyError, TypeError) as error:
results["cli"] = {
"status": "failed",
"message": str(error),
"exit_code": run.returncode,
"stderr": run.stderr[-4096:].decode("utf-8", errors="replace"),
}
return results
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--fetch", action="store_true", help="download missing inputs using curl"
)
binary = "pheon.exe" if os.name == "nt" else "pheon"
parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary)
parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus")
args = parser.parse_args()
cli = args.cli.resolve()
if not cli.is_file():
parser.error(f"{cli} is missing. Build the CLI or pass --cli.")
for name in ("files", "licenses"):
(args.cache / name).mkdir(parents=True, exist_ok=True)
sources = tomllib.loads(
(ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8")
)
all_results = {}
for key, source in sources.items():
base = (
f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}"
)
read_input(
args.cache / "licenses" / f"{key}-LICENSE",
base + "/LICENSE",
source["license_sha256"],
args.fetch,
)
for upstream, digest in source["layouts"].items():
name = f"{key}-{Path(upstream).name}"
path = args.cache / "files" / name
raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch)
data = gzip.decompress(raw) if path.suffix == ".gz" else raw
results = inspect_layout(cli, path, data)
all_results[name] = results
print(
name,
*(
f"{operation}={result['status']}"
for operation, result in results.items()
),
)
report = args.cache / "report.json"
report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8")
failures = sum(
any(result["status"] == "failed" for result in case.values())
for case in all_results.values()
)
print(
f"{len(all_results)} layouts checked, {failures} with failures. See {report}."
)
return int(failures != 0)
if __name__ == "__main__":
raise SystemExit(main())