"""Check a pinned open-source layout corpus against Pheon's installed readers.""" import argparse import gzip import hashlib import json import os from pathlib import Path import subprocess import tomllib from urllib.parse import quote from pheon import arrow ROOT = Path(__file__).resolve().parents[1] def read_input(path, url, digest, fetch): if path.is_file(): data = path.read_bytes() elif fetch: data = subprocess.check_output( ["curl", "-fLsS", "--retry", "2", "--max-time", "60", url] ) else: raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.") if hashlib.sha256(data).hexdigest() != digest: raise ValueError(f"SHA-256 mismatch for {path}") if not path.is_file(): path.write_bytes(data) return data def inspect_layout(cli, path, data): results = {} reads = { "full": lambda: arrow.read_path(path), "scan": lambda: arrow.scan_bytes(data), } layouts = {} for name, read in reads.items(): try: layout = read() layouts[name] = layout results[name] = { "status": "ok", "cells": len(layout.field("cells")[0].values), } except (ValueError, OSError, TypeError) as error: results[name] = {"status": "failed", "message": str(error)} if "scan" in layouts: cells = layouts["scan"].field("cells")[0].values ranges = list( zip( cells.field("struct_start_offset").to_pylist(), cells.field("struct_end_offset").to_pylist(), strict=True, ) ) try: selected = arrow.read_cells_bytes(data, ranges) if "full" in layouts and not selected.equals(layouts["full"]): raise ValueError( "Reading all scanned cell ranges differs from the full read" ) results["selected"] = { "status": "ok", "cells": len(selected.field("cells")[0].values), } except (ValueError, OSError, TypeError) as error: results["selected"] = {"status": "failed", "message": str(error)} else: results["selected"] = {"status": "skipped"} try: run = subprocess.run( [cli, "diff", path, path, "--format", "json"], capture_output=True, timeout=60, ) except (subprocess.TimeoutExpired, OSError) as error: results["cli"] = {"status": "failed", "message": str(error)} return results try: report = json.loads(run.stdout) results["cli"] = { "status": report["status"], "result": report["result"], "exit_code": run.returncode, "diagnostics": report["diagnostics"], } if (report["status"], report["result"], run.returncode) not in { ("complete", "equal", 0), ("incomplete", "unknown", 2), ("failed", "unknown", 2), }: results["cli"]["status"] = "failed" except (ValueError, KeyError, TypeError) as error: results["cli"] = { "status": "failed", "message": str(error), "exit_code": run.returncode, "stderr": run.stderr[-4096:].decode("utf-8", errors="replace"), } return results def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--fetch", action="store_true", help="download missing inputs using curl" ) binary = "pheon.exe" if os.name == "nt" else "pheon" parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary) parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus") args = parser.parse_args() cli = args.cli.resolve() if not cli.is_file(): parser.error(f"{cli} is missing. Build the CLI or pass --cli.") for name in ("files", "licenses"): (args.cache / name).mkdir(parents=True, exist_ok=True) sources = tomllib.loads( (ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8") ) all_results = {} for key, source in sources.items(): base = ( f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}" ) read_input( args.cache / "licenses" / f"{key}-LICENSE", base + "/LICENSE", source["license_sha256"], args.fetch, ) for upstream, digest in source["layouts"].items(): name = f"{key}-{Path(upstream).name}" path = args.cache / "files" / name raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch) data = gzip.decompress(raw) if path.suffix == ".gz" else raw results = inspect_layout(cli, path, data) all_results[name] = results print( name, *( f"{operation}={result['status']}" for operation, result in results.items() ), ) report = args.cache / "report.json" report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8") failures = sum( any(result["status"] == "failed" for result in case.values()) for case in all_results.values() ) print( f"{len(all_results)} layouts checked, {failures} with failures. See {report}." ) return int(failures != 0) if __name__ == "__main__": raise SystemExit(main())