Add a reproducible upstream GDSII corpus check
Pin 30 layouts from five upstream repositories and verify downloads and license files by SHA-256. Exercise full reads, scans, selected reads, and CLI self-comparisons with a per-layout JSON report. Cache the inputs locally for offline reruns. The baseline exposes a CLI rejection of KLayout's padded-name fixture. The gzip fixture passes directly through the CLI.
This commit is contained in:
parent
6e4b371046
commit
d95c378690
1 changed files with 252 additions and 0 deletions
167
tools/check_gds_corpus.py
Normal file
167
tools/check_gds_corpus.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Check a pinned open-source layout corpus against Pheon's installed readers."""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tomllib
|
||||
from urllib.parse import quote
|
||||
|
||||
from pheon import arrow
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def read_input(path, url, digest, fetch):
|
||||
if path.is_file():
|
||||
data = path.read_bytes()
|
||||
elif fetch:
|
||||
data = subprocess.check_output(
|
||||
["curl", "-fLsS", "--retry", "2", "--max-time", "60", url]
|
||||
)
|
||||
else:
|
||||
raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.")
|
||||
if hashlib.sha256(data).hexdigest() != digest:
|
||||
raise ValueError(f"SHA-256 mismatch for {path}")
|
||||
if not path.is_file():
|
||||
path.write_bytes(data)
|
||||
return data
|
||||
|
||||
|
||||
def inspect_layout(cli, path, data):
|
||||
results = {}
|
||||
reads = {
|
||||
"full": lambda: arrow.read_path(path),
|
||||
"scan": lambda: arrow.scan_bytes(data),
|
||||
}
|
||||
layouts = {}
|
||||
for name, read in reads.items():
|
||||
try:
|
||||
layout = read()
|
||||
layouts[name] = layout
|
||||
results[name] = {
|
||||
"status": "ok",
|
||||
"cells": len(layout.field("cells")[0].values),
|
||||
}
|
||||
except (ValueError, OSError, TypeError) as error:
|
||||
results[name] = {"status": "failed", "message": str(error)}
|
||||
|
||||
if "scan" in layouts:
|
||||
cells = layouts["scan"].field("cells")[0].values
|
||||
ranges = list(
|
||||
zip(
|
||||
cells.field("struct_start_offset").to_pylist(),
|
||||
cells.field("struct_end_offset").to_pylist(),
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
try:
|
||||
selected = arrow.read_cells_bytes(data, ranges)
|
||||
if "full" in layouts and not selected.equals(layouts["full"]):
|
||||
raise ValueError(
|
||||
"Reading all scanned cell ranges differs from the full read"
|
||||
)
|
||||
results["selected"] = {
|
||||
"status": "ok",
|
||||
"cells": len(selected.field("cells")[0].values),
|
||||
}
|
||||
except (ValueError, OSError, TypeError) as error:
|
||||
results["selected"] = {"status": "failed", "message": str(error)}
|
||||
else:
|
||||
results["selected"] = {"status": "skipped"}
|
||||
|
||||
try:
|
||||
run = subprocess.run(
|
||||
[cli, "diff", path, path, "--format", "json"],
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as error:
|
||||
results["cli"] = {"status": "failed", "message": str(error)}
|
||||
return results
|
||||
try:
|
||||
report = json.loads(run.stdout)
|
||||
results["cli"] = {
|
||||
"status": report["status"],
|
||||
"result": report["result"],
|
||||
"exit_code": run.returncode,
|
||||
"diagnostics": report["diagnostics"],
|
||||
}
|
||||
if (report["status"], report["result"], run.returncode) not in {
|
||||
("complete", "equal", 0),
|
||||
("incomplete", "unknown", 2),
|
||||
("failed", "unknown", 2),
|
||||
}:
|
||||
results["cli"]["status"] = "failed"
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
results["cli"] = {
|
||||
"status": "failed",
|
||||
"message": str(error),
|
||||
"exit_code": run.returncode,
|
||||
"stderr": run.stderr[-4096:].decode("utf-8", errors="replace"),
|
||||
}
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--fetch", action="store_true", help="download missing inputs using curl"
|
||||
)
|
||||
binary = "pheon.exe" if os.name == "nt" else "pheon"
|
||||
parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary)
|
||||
parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus")
|
||||
args = parser.parse_args()
|
||||
cli = args.cli.resolve()
|
||||
if not cli.is_file():
|
||||
parser.error(f"{cli} is missing. Build the CLI or pass --cli.")
|
||||
|
||||
for name in ("files", "licenses"):
|
||||
(args.cache / name).mkdir(parents=True, exist_ok=True)
|
||||
sources = tomllib.loads(
|
||||
(ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8")
|
||||
)
|
||||
all_results = {}
|
||||
for key, source in sources.items():
|
||||
base = (
|
||||
f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}"
|
||||
)
|
||||
read_input(
|
||||
args.cache / "licenses" / f"{key}-LICENSE",
|
||||
base + "/LICENSE",
|
||||
source["license_sha256"],
|
||||
args.fetch,
|
||||
)
|
||||
for upstream, digest in source["layouts"].items():
|
||||
name = f"{key}-{Path(upstream).name}"
|
||||
path = args.cache / "files" / name
|
||||
raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch)
|
||||
data = gzip.decompress(raw) if path.suffix == ".gz" else raw
|
||||
results = inspect_layout(cli, path, data)
|
||||
all_results[name] = results
|
||||
print(
|
||||
name,
|
||||
*(
|
||||
f"{operation}={result['status']}"
|
||||
for operation, result in results.items()
|
||||
),
|
||||
)
|
||||
|
||||
report = args.cache / "report.json"
|
||||
report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8")
|
||||
failures = sum(
|
||||
any(result["status"] == "failed" for result in case.values())
|
||||
for case in all_results.values()
|
||||
)
|
||||
print(
|
||||
f"{len(all_results)} layouts checked, {failures} with failures. See {report}."
|
||||
)
|
||||
return int(failures != 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in a new issue