diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2d14a76 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: Native checks + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + rust: + runs-on: ubuntu-latest + strategy: + matrix: + rust: ['1.88.0', stable] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: rustup toolchain install ${{ matrix.rust }} --profile minimal + - run: cargo +${{ matrix.rust }} test --locked + - run: cargo +${{ matrix.rust }} test --locked --release diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..321f3c6 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,122 @@ +name: Package artifacts + +on: + pull_request: + workflow_dispatch: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - run: python -m pip install build + - run: python -m build --sdist + - uses: actions/upload-artifact@v4 + with: + name: source + path: dist/*.tar.gz + + wheels: + needs: sdist + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: source + path: source-dist + - name: Unpack the source release + shell: bash + run: | + mkdir source-build + tar -xzf source-dist/*.tar.gz -C source-build --strip-components=1 + - name: Build and repair Python-independent wheels from the sdist + uses: pypa/cibuildwheel@v4.3.5 + with: + package-dir: source-build + env: + CIBW_BUILD: cp311-* + CIBW_SKIP: '*-musllinux_* *-win32' + CIBW_ARCHS: native + CIBW_ENVIRONMENT_LINUX: 'CARGO_HOME=/tmp/pheon-cargo RUSTUP_HOME=/tmp/pheon-rustup PATH=/tmp/pheon-cargo/bin:$PATH' + CIBW_BEFORE_ALL_LINUX: 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable' + CIBW_TEST_COMMAND: 'python -I {project}/tools/check_installed.py' + - name: Check wheel ABI tags + shell: python + run: | + from pathlib import Path + wheels = list(Path('wheelhouse').glob('*.whl')) + assert wheels + assert all('-py3-none-' in p.name for p in wheels), wheels + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse/*.whl + + same-wheel: + needs: wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + python: ['3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse + - name: Install the same artifact on another Python version + shell: bash + run: python -m pip install wheelhouse/*.whl + - run: python -I tools/check_installed.py + + cli: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: cargo test --release --locked + - run: cargo build --release --locked --bin pheon + - name: Assemble standalone CLI artifact + shell: bash + run: | + mkdir cli-dist + if [ "$RUNNER_OS" = Windows ]; then + cp target/release/pheon.exe cli-dist/ + else + cp target/release/pheon cli-dist/ + fi + cp LICENSE.md README.md cli-dist/ + cp -R contrib docs cli-dist/ + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.os }} + path: cli-dist/ diff --git a/.gitignore b/.gitignore index ed58073..c2a3759 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /target/ **/*.rs.bk -Cargo.lock *.swp *.swo @@ -8,4 +7,6 @@ Cargo.lock __pycache__/ /build/ /dist/ +/.cache/ +/.venv/ /*.egg-info/ diff --git a/ARROW_SCHEMA.md b/ARROW_SCHEMA.md deleted file mode 100644 index b916d67..0000000 --- a/ARROW_SCHEMA.md +++ /dev/null @@ -1,148 +0,0 @@ -# Arrow Schema Documentation - -This project uses Apache Arrow to represent GDSII (Graphic Data System II) library data. The schema is optimized for memory efficiency and fast access, employing several packed representations for common data types. - -## Top-Level Structure - -The top-level object is a `StructArray` representing the entire GDSII library. - -| Field | Type | Description | -|---|---|---| -| `meters_per_db_unit` | `Float64` | Database unit size in meters. | -| `user_units_per_db_unit` | `Float64` | Number of user units in one database unit. | -| `lib_name` | `Utf8` | The name of the GDSII library. | -| `mod_time` | `FixedSizeList(Int16, 6)` | Last modification time [year, month, day, hour, minute, second]. | -| `acc_time` | `FixedSizeList(Int16, 6)` | Last access time [year, month, day, hour, minute, second]. | -| `cells` | `List(Cell)` | A list of cells (structures) in the library. | -| `cell_names` | `List(Utf8)` | Names of all cells, indexed by their ID. | -| `layers` | `List(UInt32)` | List of packed layer/datatype pairs, indexed by their ID. | - -## Cell Structure - -Each cell in the `cells` list is a `StructArray`. - -| Field | Type | Description | -|---|---|---| -| `id` | `UInt32` | Unique identifier for the cell. | -| `file_offset` | `UInt64` | Byte offset immediately after the cell's `STRNAME` record (the first element, or `ENDSTR` for an empty cell). Identical for full and selected reads. | -| `srefs` | `List(SRef)` | Simple structure references (no properties). | -| `arefs` | `List(ARef)` | Array references (no properties). | -| `sref_props` | `List(SRefProp)` | Structure references with properties. | -| `aref_props` | `List(ARefProp)` | Array references with properties. | -| `rect_batches` | `List(RectBatch)` | Batches of rectangular boundaries (optimized). | -| `boundary_batches` | `List(BoundaryBatch)` | Batches of generic boundaries. | -| `boundary_props` | `List(BoundaryProp)` | Individual boundaries with properties. | -| `paths` | `List(Path)` | Path elements. | -| `nodes` | `List(BoxNode)` | Node elements. | -| `boxes` | `List(BoxNode)` | Box elements. | -| `texts` | `List(Text)` | Text labels. | - -## Hierarchy Scan Schema - -`scan_library` and `scan_bytes` return the same library metadata and `cell_names` -as a full read, with the following `cells` schema and no `layers` field: - -| Field | Type | Description | -|---|---|---| -| `id` | `UInt32` | Cell ID indexing this result's `cell_names`. | -| `struct_start_offset` | `UInt64` | Inclusive byte offset of the `BGNSTR` record. | -| `struct_end_offset` | `UInt64` | Exclusive byte offset immediately after `ENDSTR`. | -| `refs` | `List(ScanRef)` | SREFs and AREFs, including references with properties, sorted by target ID. | - -Each scan reference has `target`, `invert_y`, `scale`, `angle_rad`, `xy`, `xy0`, -`xy1`, and `counts`, using the ARef types and packed encodings described below. -SREFs use zero basis vectors and counts `(1, 1)`. Properties are omitted from the -scan output; geometry and text are skipped without decoding. - -Pass the half-open scan ranges to `read_selected_library` or `read_cells_bytes` -to obtain the full library schema for just those cells. Each range must contain -exactly one complete cell, from `BGNSTR` through `ENDSTR`. Cells appear in the -requested order. Cell and layer IDs are assigned within each result, so use -`cell_names` and `layers` from that result when resolving them; IDs can differ -from a full read or a scan. An empty selection returns metadata and empty lists. -The Rust random-access reader returns the original input slice as its remainder. - -## Element Types - -### Common Packed Encodings - -- **Layer ID (`UInt32`)**: A packed representation of `(layer, datatype)`. - - `layer = (layer_id >> 16) as i16` - - `datatype = (layer_id & 0xFFFF) as i16` -- **XY Coordinate (`UInt64`)**: A packed representation of a single point `(x, y)`. - - `x = (xy >> 32) as i32` - - `y = (xy & 0xFFFFFFFF) as i32` -- **Array Counts (`UInt32`)**: A packed representation of `(columns, rows)`. - - `columns = (counts >> 16) as i16` - - `rows = (counts & 0xFFFF) as i16` - -### Reference Elements (SRef / ARef) - -References point to other cells in the library. - -**SRef (Simple Reference)**: -- `target`: `UInt32` (Cell ID) -- `invert_y`: `Boolean` (Vertical flip) -- `scale`: `Float64` (Magnification) -- `angle_rad`: `Float64` (Rotation in radians) -- `xy`: `UInt64` (Packed origin point) - -**ARef (Array Reference)**: -- Same as SRef, plus: -- `xy0`: `UInt64` (Packed column basis vector increment) -- `xy1`: `UInt64` (Packed row basis vector increment) -- `counts`: `UInt32` (Packed columns and rows) - -Array increments must be exact signed 32-bit integer database-unit values. Full -reads, hierarchy scans, selected reads, and their FFI entry points reject arrays -whose endpoint/count division is fractional or outside that range. They never -round or truncate placements. The review CLI retains original endpoints and can -compare fractional arrays independently of this Arrow representation. - -**SRefProp / ARefProp**: -- Same as above, plus a `properties` field. - -### Boundary Elements - -Boundaries are grouped into batches for efficiency when they share properties or are simple rectangles. - -**RectBatch**: -- `layer`: `UInt32` (Packed Layer ID) -- `rects`: `List(Int32)` (Flattened list of `[min_x, min_y, max_x, max_y]` for each rectangle) - -**BoundaryBatch**: -- `layer`: `UInt32` (Packed Layer ID) -- `vertices`: `List(Int32)` (Flattened list of all vertices for all boundaries in the batch) -- `vertex_offsets`: `List(UInt32)` (Start index for each boundary in the `vertices` list) - -**BoundaryProp**: -- Individual boundary with `properties`. - -### Other Elements - -**Path**: -- `layer`: `UInt32` -- `path_type`: `Int16` (Optional) -- `extension_start`: `Int32` (Optional) -- `extension_end`: `Int32` (Optional) -- `width`: `Int32` (Optional; null when `WIDTH` is absent) -- `xy`: `List(Int32)` (centerline coordinates) -- `properties`: `List(Property)` - -**Text**: -- `layer`: `UInt32` -- `presentation_horiz`: `UInt8` -- `presentation_vert`: `UInt8` -- `presentation_font`: `UInt8` -- `path_type`: `Int16` -- `width`: `Int32` -- `invert_y`: `Boolean` -- `mag`: `Float64` -- `angle_deg`: `Float64` -- `xy`: `UInt64` (Packed position) -- `string`: `Utf8` -- `properties`: `List(Property)` - -**Property**: -- `key`: `Int16` -- `value`: `Utf8` diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..cfec364 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,664 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link 0.1.1", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pheon" +version = "0.3.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "flate2", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.1", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link 0.1.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link 0.1.1", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index b08e0a5..93004b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,19 +1,27 @@ [package] -name = "klamath_rs_ext" -version = "0.2.0" +name = "pheon" +version = "0.3.0" authors = ["jan "] edition = "2024" +rust-version = "1.88" +description = "GDSII parsing, Arrow interchange, and structural layout review" +license = "GPL-3.0-only" +readme = "README.md" +repository = "https://mpxd.net/code/jan/pheon" [lib] -name = "klamath_rs_ext" +name = "pheon" crate-type = ["cdylib", "rlib"] [dependencies] -byteorder = "^1" -arrow = {version = "^54", features = ["ffi"]} +arrow-array = {version = "^54", features = ["ffi"]} +arrow-schema = "^54" +flate2 = "1" +serde = "1" serde_json = "1" +termcolor = "1.4" [profile.release] diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..175443c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,595 @@ +GNU General Public License +========================== + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +* **a)** The work must carry prominent notices stating that you modified it, and giving a +relevant date. +* **b)** The work must carry prominent notices stating that it is released under this +License and any conditions added under section 7. This requirement modifies the +requirement in section 4 to “keep intact all notices”. +* **c)** You must license the entire work, as a whole, under this License to anyone who +comes into possession of a copy. This License will therefore apply, along with any +applicable section 7 additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no permission to license the +work in any other way, but it does not invalidate such permission if you have +separately received it. +* **d)** If the work has interactive user interfaces, each must display Appropriate Legal +Notices; however, if the Program has interactive interfaces that do not display +Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +* **a)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by the Corresponding Source fixed on a +durable physical medium customarily used for software interchange. +* **b)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by a written offer, valid for at least +three years and valid for as long as you offer spare parts or customer support for +that product model, to give anyone who possesses the object code either **(1)** a copy of +the Corresponding Source for all the software in the product that is covered by this +License, on a durable physical medium customarily used for software interchange, for +a price no more than your reasonable cost of physically performing this conveying of +source, or **(2)** access to copy the Corresponding Source from a network server at no +charge. +* **c)** Convey individual copies of the object code with a copy of the written offer to +provide the Corresponding Source. This alternative is allowed only occasionally and +noncommercially, and only if you received the object code with such an offer, in +accord with subsection 6b. +* **d)** Convey the object code by offering access from a designated place (gratis or for +a charge), and offer equivalent access to the Corresponding Source in the same way +through the same place at no further charge. You need not require recipients to copy +the Corresponding Source along with the object code. If the place to copy the object +code is a network server, the Corresponding Source may be on a different server +(operated by you or a third party) that supports equivalent copying facilities, +provided you maintain clear directions next to the object code saying where to find +the Corresponding Source. Regardless of what server hosts the Corresponding Source, +you remain obligated to ensure that it is available for as long as needed to satisfy +these requirements. +* **e)** Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are being +offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +* **a)** Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or +* **b)** Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed by works +containing it; or +* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that +modified versions of such material be marked in reasonable ways as different from the +original version; or +* **d)** Limiting the use for publicity purposes of names of licensors or authors of the +material; or +* **e)** Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or +* **f)** Requiring indemnification of licensors and authors of that material by anyone +who conveys the material (or modified versions of it) with contractual assumptions of +liability to the recipient, for any liability that these contractual assumptions +directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. diff --git a/MANIFEST.in b/MANIFEST.in index 9ba61f7..ff57a3d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ -include Cargo.toml Cargo.lock README.md ARROW_SCHEMA.md +include Cargo.toml Cargo.lock rust-toolchain.toml README.md LICENSE.md recursive-include src *.rs -recursive-include klamath_rs_ext *.py *.h py.typed +recursive-include pheon *.py *.h py.typed recursive-include tests *.rs *.py *.json *.txt *.mmd recursive-include docs *.md *.json recursive-include contrib * +recursive-include tools *.py diff --git a/README.md b/README.md index 2e51acf..eaf6d3c 100644 --- a/README.md +++ b/README.md @@ -1,281 +1,67 @@ -# klamath-rs +# Pheon -A fast GDSII parser and writer in Rust, with Apache Arrow integration for efficient data handling. +Pheon reads GDSII layouts into Apache Arrow and provides command-line tools for +inspecting and comparing layout revisions. It is written in Rust, with Python +bindings and a C API. -## Overview +- Inspect cells, records, and reference hierarchies. +- Compare layout objects, with text or JSON reports and Git integration. +- Read geometry into Arrow, or scan the hierarchy before loading selected cells. -`klamath-rs` is designed to handle large GDSII files with high performance. It uses Apache Arrow as its primary in-memory data representation, allowing for efficient memory usage and easy integration with other tools in the Arrow ecosystem (like pandas, polars, or DuckDB via Python bindings). +Comparisons ignore timestamps and element order while preserving duplicates and +other object details. They compare object fields and coordinates exactly; see the +[comparison rules][comparison] for supported normalization and limits. -## Documentation +## Try the command-line tools -- [Arrow Schema Documentation](ARROW_SCHEMA.md): Detailed information about the Arrow schema used for GDSII data. - -## Features - -- Fast parsing of GDSII files. -- Efficient memory representation using Apache Arrow. -- Support for common GDSII elements: SREF, AREF, Boundary, Path, Text, etc. -- Optimized "batch" representations for rectangles and boundaries sharing the same layer. -- C-compatible FFI for integration with other languages (e.g., Python). - -REAL8 writing uses normalized GDS encodings and represents every in-range `f64` -exactly. Finite magnitudes below `2^-260` underflow to canonical zero; nonfinite -values and magnitudes at or above `2^252` are rejected by the encoder. - -## Usage - -`read_path` exports a library through the Arrow C Data Interface. Allocate -initialized `ArrowArray` and `ArrowSchema` output structures, then check the -returned status: `0` means success and `-1` means failure. Only import the Arrow -outputs after success; the caller owns their release callbacks. Failed calls -leave the outputs unchanged. See `test.py` for a Python CFFI example. - -`last_error_message()` returns a UTF-8, null-terminated error string, or null -after a successful parser call. Errors belong to the calling thread. Copy the -message before making another parser call on that thread, and before the thread -exits; calls on other threads do not invalidate it. Parser panics that unwind -are converted into errors. - -For partial reads, call `scan_bytes` with a borrowed GDSII byte buffer to obtain -cell ranges and references without decoding geometry. Pass the desired ranges -to `read_cells_bytes` to load their full contents. Both calls use the same Arrow -outputs and status/error handling as `read_path`: - -```c -int scan_bytes(const uint8_t *data, size_t size, - struct ArrowArray *array, struct ArrowSchema *schema); -int read_cells_bytes(const uint8_t *data, size_t size, - const uint64_t *ranges, size_t range_count, - struct ArrowArray *array, struct ArrowSchema *schema); -``` - -`ranges` contains alternating inclusive start and exclusive end offsets, with -two native-endian `uint64_t` values per cell. Each range must cover one complete -cell. A null `ranges` pointer is accepted when `range_count` is zero. Likewise, -null `data` is accepted for zero bytes and produces an ordinary parse error for -the empty input. Nonempty buffers must be readable for the duration of the call; -range values must be aligned. Exported Arrow data owns its storage independently -of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan -schema, offset definitions, and ID assignment. - -## Python package and validation - -The Python packing helpers require NumPy and CFFI, both declared as runtime -dependencies. Setuptools invokes `cargo build --release --lib` and bundles the -C ABI shared library in a platform-specific wheel. Its version comes from -`Cargo.toml`. The C declarations are in `klamath_rs_ext/_ffi.h`; the loader uses -CFFI ABI mode, so no Python extension compilation or Maturin is required. +Install the CLI [from crates.io][crate] with Rust 1.88 or newer. +Use your own GDSII paths in the examples. ```sh -python3 -m pip install . +cargo install pheon --locked +pheon dump chip.gds +pheon diff before.gds after.gds +pheon hierarchy chip.gds --format mermaid ``` -To build a source archive and a wheel with the `build` frontend and installed -Setuptools/Wheel tooling: +`diff` returns 0 for equality, 1 for differences, and 2 for invalid or incomplete +comparisons. Start with the [CLI guide][cli], then set up +[Git integration][git] if you review layouts in a repository. + +## Read layouts in Python + +Install the Python package [from PyPI][pypi] with Python 3.11 or newer. ```sh -python3 -m build --no-isolation +python -m pip install pheon ``` -Source archives include a Cargo lockfile, and wheel builds use `--locked` so -building from an archive retains the resolved Rust dependency versions. Wheels -are tagged for the build platform and use the Python-independent `py3-none` ABI. -The CLI continues to build directly with Cargo and has no Python dependency. +Run this example from a directory containing `chip.gds`. -For source-tree development, point the CFFI loader at a Cargo-built library: +```python +from pheon import arrow -```sh -cargo build --release --lib -KLAMATH_RS_LIBRARY="$PWD/target/release/libklamath_rs_ext.so" \ - python3 -m unittest discover -s tests/python -v +layout = arrow.read_path("chip.gds") +print(layout.field("lib_name")[0].as_py()) ``` -Use `libklamath_rs_ext.dylib` on macOS or `klamath_rs_ext.dll` on Windows. Without -the explicit override, the loader uses the library bundled inside the package. -After installing a wheel, run its tests outside the source checkout to exercise -the installed package: +The Python package installs PyArrow. See the [Python guide][python] for source +installs and selected reads, and the [Arrow reference][arrow] for the returned data. -```sh -python3 -I -m unittest discover -s /absolute/path/to/klamath-rs/tests/python -v -``` +## Learn more -The Python tests cover multidimensional and noncontiguous packing, input -preservation, range errors, and the C declarations for Arrow exports and release -callbacks. Run `cargo test` and `cargo test --release` for native validation. +Browse the [documentation][docs] for layer names, native integration, +JSON reports, development, and releases. For writing, Pheon provides low-level +GDSII record helpers in Rust. -## Layout review CLI +Licensed under [GPL-3.0-only][license]. -The review CLI is a standalone Rust executable: - -```sh -cargo build --bin klamath -target/debug/klamath dump chip.gds -target/debug/klamath dump chip.gds --records -target/debug/klamath diff old.gds new.gds --hierarchy ascii -target/debug/klamath diff old.gds new.gds --hierarchy mermaid --max-changes 20 -target/debug/klamath hierarchy chip.gds --format mermaid --depth 4 -target/debug/klamath diff old.gds new.gds --top CHIP --layer 1/0 -target/debug/klamath diff old.gds new.gds --show-layer 1/0 -target/debug/klamath diff old.gds new.gds --format json -target/debug/klamath diff old.gds new.gds --format json --hierarchy graph -target/debug/klamath name-layers chip.gds named.gds --cell CHIP --carrier 900/7 --name 1:0:WG -``` - -The object dump is deterministic and omits source offsets. Record dumps preserve -record order and show offsets, tags, sizes, and values, including records the -object reviewer does not support. They can inspect an invalid metadata declaration -without invoking object-level validation. Zero padding after ENDLIB is accepted. - -The fixed `klamath-object-v2` comparison profile: - -- Matches cells by name and layers by numeric pair. The default scope is the - entire library; explicit selectors are recorded in each report. -- Ignores library/cell timestamps, cell order, element order, and property order. -- Normalizes boundary/box starting vertex and winding, retaining other vertices. -- Preserves duplicate objects and duplicate properties as multisets. -- Compares paths, labels, references, array endpoints/counts, properties, and all - retained flags exactly. Explicit defaults and their omission remain distinct; - SREFs and single-element AREFs remain distinct. REAL8 encodings are retained. -- Reports library fields, including UNITS, separately. It does not rescale files - into a shared grid, infer cell renames, or perform geometric Boolean operations. -- Uses bounding boxes as coordinate envelopes, not evidence of equality. Path - envelopes do not include stroke width; text/reference envelopes are coordinates, - not rendered or recursively transformed bounds. -- Returns an incomplete result for unsupported records, future metadata versions, - unresolved references, or cycles. These cannot produce an equality claim. - -`diff` exits with **0** for equality under this profile, **1** for differences, -and **2** for invalid input or incomplete comparison. `--max-changes` limits the -number of object details displayed, not the comparison. Source offsets are -included in changed-object reports for tracing changes back to record dumps. - -`diff --format json` emits a versioned report with typed before/after evidence, -structured diagnostics, and independent comparison status and completeness. -JSON includes all changes by default. Explicit display filters and `--max-changes` -record omitted evidence without changing totals or exit status. `--hierarchy graph` -adds the complete scoped reference graph, with symbolic array multiplicities; -`--depth` applies to text output only. Timestamp-only changes still return 0. -Input and argument errors also produce JSON when requested. See the -[report contract](docs/diff-reports.md) and -[version 1 JSON Schema](docs/diff-report-v1.schema.json). - -Unmatched objects are paired for explanation only when candidates are unique by -type/layer/bounding box, then by type/layer among the remaining objects. -References additionally match by target. Ambiguous groups stay additions and -removals. Reports explain reference movement, array-count changes, path widths, -and polygons changed within the same bounds. Full before/after records remain -visible, and duplicate counts remain significant. Unchanged layer names appear -once as a legend for changed objects on those layers. - -### Comparison scope and display filters - -`--top NAME` compares that cell and its reachable subtree independently in each -revision. `--cell NAME` compares the exact cell, including its reference records, -without comparing descendant contents. Combine the two to select a cell within -a subtree. A selector absent from both revisions is an error; absence on one -side is an addition/deletion. - -`--layer L/T` selects objects on a numeric layer/type pair, plus references whose -targets use that layer transitively. Cells with no relevant content are omitted. -Unresolved references, cycles, and unsupported records with unknown layer effect -remain incomplete results. Unsupported objects on another known layer can be -excluded. Library fields, including units, remain compared. Unsupported layer-name -version diagnostics are always retained, since those declarations can affect any -layer or cell in the library. Scoped dumps and -hierarchy views accept the same selectors. Scope selection parses the complete -input: malformed record grammar and conflicting metadata are still errors. - -`--show-cell NAME` and `--show-layer L/T` filter object details in a diff report -without changing comparison status, totals, metadata changes, or hierarchy. -References have no numeric layer and are hidden by `--show-layer`. The comparison -scope and any display-only filter are printed explicitly. The Rust API accepts -one combined `Scope`; applying another scope to an already scoped document is -rejected rather than losing reference coverage information. - -Hierarchy output distinguishes directly changed cells, affected ancestors, and -unchanged targets of modified references. It includes edges from both revisions, -so removed branches remain visible. Counts are direct edge multiplicities, -including arrays, rather than expanded placements. Shared definitions are marked -as already shown in text and appear once in Mermaid. `--depth` limits displayed -descendants only. The CLI emits Mermaid source, requiring no diagram renderer. - -Layer names are read from labels in an existing cell. A label with the exact -string `KLAMATH_LAYER_NAMES_V1` identifies an annotation carrier `(layer, texttype)` -pair within that cell. All other labels on that same pair in that same cell must -be `layer:datatype:name` declarations, for example `1:0:WG`. The carrier pair is -not fixed; choose a suitable annotation layer in the design's technology. -Declarations split at the first two colons, allowing colons in names. Numeric -fields are unsigned 16-bit decimal integers. Names are nonempty UTF-8 strings. -Repeated identical declarations are accepted; conflicting declarations fail. -Names are collected across the library. Recognized metadata labels are omitted -from object comparison; mappings are compared separately and printed once per -numeric pair, so renaming a layer does not rewrite every shape description. - -`name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ...` writes -this convention. It consolidates existing recognized declarations into one legend -in the chosen existing cell, retains existing names not explicitly updated, and -overrides the named pairs. The chosen carrier must contain no ordinary design -objects in that cell. Unrelated records, timestamps, unknown records and trailing -padding are copied byte-for-byte. No structures or references are added. Output -must be a new file: the CLI refuses to overwrite an existing destination, including -the input. Invalid declarations, conflicting arguments, oversized labels and -future metadata versions fail before output creation. - -The record-preserving review adapter retains information normalized or omitted -by the Arrow geometry reader; existing Arrow APIs are unchanged. Structured Rust -change records feed the text and hierarchy renderers. LayoutEditor's `LayerNames` -extension, layout extraction/remapping, and geometric Boolean comparison remain -outside the current CLI. - -### Git integration - -Install the executable on PATH with `cargo install --path . --bin klamath`. -The templates in [contrib/git/config](contrib/git/config) and -[contrib/git/attributes](contrib/git/attributes) provide two alternative drivers. -For structural summaries and affected hierarchy, add this attribute: - -```gitattributes -*.gds diff=klamath -``` - -Configure the native external driver in Git config: - -```gitconfig -[diff "klamath"] - command = klamath git-diff --hierarchy ascii -- - binary = true - trustExitCode = false -``` - -The adapter consumes Git's old/new temporary files directly, handles additions, -deletions and renames, and quotes display paths. It returns success after rendering -either equality or differences, as expected by the external-driver protocol. -Invalid or incomplete comparisons retain a nonzero exit and diagnostics. Scope -and display options can precede the final `--` in the configured command. Existing -repositories using `diff=layout` can put the same command under `[diff "layout"]`. - -For ordinary Git line diffs, use canonical textconv instead: - -```gitattributes -*.gds diff=klamath-text -``` - -Configure the driver in Git config: - -```gitconfig -[diff "klamath-text"] - textconv = klamath dump - binary = true - cachetextconv = true -``` - -This is a human review representation, not an applicable GDS patch. Invalidate -Git's textconv cache when changing the dump profile/tool version. For automated -equality checks, use `klamath diff` and its exit status directly. - -`tests/review.rs` contains ten initial fixture pairs, checked-in canonical/text/ -Mermaid goldens, metadata byte-preservation checks, scope and report regressions, -and actual Git integration tests. It also checks near-limit polygon records, -2048-cell hierarchies and arrays representing over a billion placements without -expanding them. Run `cargo test` and `cargo test --release` for debug and optimized -validation. No KLayout installation or Python package is required. +[arrow]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/arrow-schema.md +[cli]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/cli.md +[comparison]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/comparison.md +[crate]: https://crates.io/crates/pheon +[docs]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/index.md +[git]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/git.md +[license]: https://mpxd.net/code/jan/pheon/src/branch/master/LICENSE.md +[pypi]: https://pypi.org/project/pheon/ +[python]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/python.md diff --git a/contrib/gds-corpus.toml b/contrib/gds-corpus.toml new file mode 100644 index 0000000..25eeefb --- /dev/null +++ b/contrib/gds-corpus.toml @@ -0,0 +1,71 @@ +# Pinned GDSII test inputs. Each digest covers the original upstream bytes. + +[gdstk] +repo = "heitzmann/gdstk" +revision = "1099be134da34d4afd9e5fb1f4f0dbae9ef4c17a" +license = "BSL-1.0" +license_sha256 = "beb8e42e9d6b4284e03304d05a81a0755200a965fc8d0a5e0aea1e84cf805d6e" + +[gdstk.layouts] +"tests/proof_lib.gds" = "af7b95ed191ebe32438c0f0cb5ed824ba2a98673cfa9ce6f1a9ff5a827c3dbf7" + +[gdsfactory] +repo = "gdsfactory/gdsfactory" +revision = "7407836dd5d872c704fc06f38562a2ffc88049dc" +license = "MIT" +license_sha256 = "cdace89991b496bd052909790d00ece97feab566afd964d0883d2c0605462b5b" + +[gdsfactory.layouts] +"tests/gds/alphabet_3top_cells.gds" = "09b75d5bfeb0051e4972a4d9fc26df5d342604f7ed2a0994376cd7c77e929019" +"tests/gds/big_rect.gds" = "de71342365dc4bd84dbb03307537b7ca2b4dffab98efdf88713cc451e44e6ac2" +"tests/gds/mmi1x2.gds" = "e12e81a9d5b9dc8463cce05915f567bc7174f848a52c7e092dc8a602819dafa0" +"tests/gds/mzi2x2.gds" = "49499f13c888de137eb26342c45ea379d9ea2ecd82fc990780d38fa014002d38" +"tests/gds/small_rect.gds" = "345480096d17bc724ae32cc0f42bd915a39c6e64806b43054d687755afd5faba" +"tests/gds/straight.gds" = "dc5e971b5be91864895870ef70a5ae959bd52dab5971caa0dcfc9aa6e13fa40d" + +[klayout] +repo = "KLayout/klayout" +revision = "53911acd6dfaa113c188d11d20b2ae290aa60a6b" +license = "GPL-3.0 (upstream LICENSE)" +license_sha256 = "589ed823e9a84c56feb95ac58e7cf384626b9cbf4fda2a907bc36e103de1bad2" + +[klayout.layouts] +"samples/lvs/ringo.gds" = "f24d4d11ac7941021d0b0c15d8b32ced1c6735cbd4da68de1482830ca45be3bd" +"testdata/drc/texts.gds" = "ee8ddf7de65ebccf9772fd98cb5d45313e353f8a3455d162611320885061f907" +"testdata/algo/deep_texts_l1.gds" = "253d24491a097102a729ef7f101c9b7acf7684c4a90c16d212ac91d9087e3b4d" +"testdata/gds/arefs.gds" = "fadde330ade22d0e8b92e4040377dcccb6e7a572d9d69df6b3fa722a2bfbcdcd" +"testdata/gds/arefs_skew.gds" = "a681a47cd1d2d74b881d0ab10c3d6fd6c9674c819cf39c077936a4328511b1ae" +"testdata/gds/arefs_skew2.gds" = "363dd2650b4a3011cbe33a6d8febccc77b22dd9b7563f087029a1c5afbe2ae94" +"testdata/gds/basic_instances.gds" = "6ccfffd590c159a5ab450c8eed826afb8e1d68c9ae2f85ec089ab21c0b40ec1b" +"testdata/gds/issue-1353.gds" = "f0e9cbe2b0582d766b3e6e1737617865ca5e74dc525f7bc2872e09ffe6a82482" +"testdata/gds/issue_1835.gds" = "b2d78e8ed4f43e63738857c7aa81606c8ccc5ca0910e3d4e4569c058415d9171" +"testdata/gds/issue_893.gds" = "30a2bf58ea8ae5b36f6df67c5d5221bc257e916b985bc1e7a74cf19ba244eff8" +"testdata/gds/issue_893_au.gds" = "e33cbc5ae2a3f3e9779f6c1cbaf89762c33644c76836c96445e887820dd4eab6" +"testdata/gds/t166_au.gds.gz" = "bd5f0eda73cf13daae77d61eb2a2d9376f2752e66cd458307cfee0e4d5b5ea57" + +[sky130] +repo = "google/skywater-pdk-libs-sky130_fd_sc_hd" +revision = "ac7fb61f06e6470b94e8afdf7c25268f62fbd7b1" +license = "Apache-2.0" +license_sha256 = "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + +[sky130.layouts] +"cells/inv/sky130_fd_sc_hd__inv_1.gds" = "18a9cfa399957c7c7b5fc75175e03fc921d56ecc6b8de8fae34ba301bb7914c4" +"cells/inv/sky130_fd_sc_hd__inv_4.gds" = "1e0fb091a3e4583297e7124d68542e4f29ee6d1f711e80dcb41fc1ed8465c0ce" +"cells/nand2/sky130_fd_sc_hd__nand2_1.gds" = "4e7fa6255df901f59151791d6e8ff4c421b63c1ea3822d6d5d25b48221782a3a" +"cells/dfxtp/sky130_fd_sc_hd__dfxtp_1.gds" = "2b0eaaeedd5b7b2c359942098aee0cef69387108e02e10bc09940680e4cc7ec3" +"cells/tapvpwrvgnd/sky130_fd_sc_hd__tapvpwrvgnd_1.gds" = "642f8841345c7c09ec910fe7893ae803ab760606adfab65229e2aedb055005a5" +"cells/fill/sky130_fd_sc_hd__fill_1.gds" = "9f718b6252a027fc769c2e4298c182e1c4ee3b160fefcd6acf7d947d5c6c3453" + +[ihp] +repo = "IHP-GmbH/IHP-Open-PDK" +revision = "e73adb4045731ddec21634146d536380057f6a26" +license = "Apache-2.0" +license_sha256 = "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + +[ihp.layouts] +"ihp-sg13g2/libs.ref/sg13g2_pr/gds/sg13g2_pr.gds" = "3c640413b455354da11ec4db01c25f165489bcfd4cdec0f79559a9f78331ab42" +"ihp-sg13g2/libs.ref/sg13g2_stdcell/gds/sg13g2_stdcell.gds" = "ab9f2802c78935d4463093c24860f9d3f244057a839179fb19fc0243a18cf3f4" +"ihp-sg13g2/libs.ref/sg13g2_sram/gds/RM_IHPSG13_1P_1024x16_c2_bm_bist.gds" = "1ed0af3f9cc3a3455f4c0beb08394502a875dda3c54bc03123357e397b26e929" +"ihp-sg13g2/libs.qa/drc/devices/sg13g2_qacells.gds" = "21d8b2663ba15fa4c99752ad7a030f486043ba17950236bee8dc483e64c948b2" +"ihp-sg13g2/libs.qa/drc/special/sg13g2_stdcell_abutment.gds" = "e125bd8e6bf8976f41bf29fb761c28559064b50571289da98c148684676e1728" diff --git a/contrib/git/attributes b/contrib/git/attributes index 051952d..ed817cc 100644 --- a/contrib/git/attributes +++ b/contrib/git/attributes @@ -1,5 +1,5 @@ # Add these lines to the layout repository's .gitattributes. -*.gds diff=klamath +*.gds diff=pheon # For canonical textconv instead, replace the line above with: -# *.gds diff=klamath-text +# *.gds diff=pheon-text diff --git a/contrib/git/config b/contrib/git/config index 2852e9a..ca84adf 100644 --- a/contrib/git/config +++ b/contrib/git/config @@ -1,10 +1,10 @@ -# Include this file from Git config after installing klamath on PATH. -[diff "klamath"] - command = klamath git-diff --hierarchy ascii -- +# Include this file from Git config after installing pheon on PATH. +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- binary = true trustExitCode = false -[diff "klamath-text"] - textconv = klamath dump +[diff "pheon-text"] + textconv = pheon dump binary = true cachetextconv = true diff --git a/contrib/wheels/manylinux-image.txt b/contrib/wheels/manylinux-image.txt new file mode 100644 index 0000000..a8468dd --- /dev/null +++ b/contrib/wheels/manylinux-image.txt @@ -0,0 +1 @@ +quay.io/pypa/manylinux_2_28_x86_64@sha256:e0adf18521ad35192df733a6c62d35df7801957d24e0470238aacecfab2188a3 diff --git a/docs/arrow-schema.md b/docs/arrow-schema.md new file mode 100644 index 0000000..a0abd94 --- /dev/null +++ b/docs/arrow-schema.md @@ -0,0 +1,133 @@ +# Arrow layout reference + +This reference is for consumers of Pheon's Arrow readers. Each result is a +one-row `StructArray` representing a GDSII library. The +[Python guide](python.md) shows how to obtain and access it. + +Geometry fields named `layer` are **indices into the result's `layers` table**. +Reference `target` and cell `id` values index its `cell_names` table. Resolve IDs +using tables from the same result; a selected read can assign different IDs. + +`pheon_arrow_schema_version()` currently returns 1 for all layouts below. +Consumers of the [C API](native-api.md) must check both this application schema +version and the separate C API version. + +## Full library + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `meters_per_db_unit` | Float64 | Database unit size in meters. | +| `user_units_per_db_unit` | Float64 | User units per database unit. | +| `lib_name` | Utf8 | Library name. | +| `mod_time`, `acc_time` | FixedSizeList(Int16, 6) | Modification/access time: year, month, day, hour, minute, second. | +| `cells` | List(Cell) | Cell records. | +| `cell_names` | List(Utf8) | Names indexed by cell ID, including targets without cell definitions. | +| `layers` | List(UInt32) | Packed layer/type pairs indexed by layer ID. | + +### Cell + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into `cell_names`. | +| `file_offset` | UInt64 | Byte offset immediately after STRNAME, before the first element or ENDSTR. | +| `srefs`, `arefs` | List(SRef), List(ARef) | References without properties. | +| `sref_props`, `aref_props` | List(SRefProp), List(ARefProp) | References with properties. | +| `rect_batches` | List(RectBatch) | Optimized rectangular boundaries. | +| `boundary_batches` | List(BoundaryBatch) | Other boundaries without properties. | +| `boundary_props` | List(BoundaryProp) | Boundaries with properties. | +| `paths` | List(Path) | Paths. | +| `nodes`, `boxes` | List(BoxNode) | Node and box elements. | +| `texts` | List(Text) | Text labels. | + +## Hierarchy scan and selected reads + +Scans return the same metadata and `cell_names`, omit `layers`, and replace +`cells` with these records: + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into this result's `cell_names`. | +| `struct_start_offset` | UInt64 | Inclusive offset of BGNSTR. | +| `struct_end_offset` | UInt64 | Exclusive offset after ENDSTR. | +| `refs` | List(ScanRef) | References sorted by target ID, with properties omitted. | + +ScanRef has ARef's fields below. An SREF uses zero basis vectors and counts +`(1, 1)`. Geometry and text are skipped without decoding. + +Pass complete half-open cell ranges to `read_selected_library` in Rust or +`read_cells_bytes` in Python/C. The result uses the full library layout, with +cells in requested order. An empty selection returns metadata and empty lists. +Offsets always refer to the original uncompressed input. The Rust selected +reader returns the original input slice as its remainder. + +## Packed values + +| Value | Storage | Decoding | +| --- | --- | --- | +| Layer/type pair in `layers` | UInt32 | Layer in high 16 bits; datatype/texttype/boxtype/nodetype in low 16 bits. | +| Point or vector | UInt64 | Signed Int32 x in high 32 bits, signed Int32 y in low 32 bits. | +| Array counts | UInt32 | Columns in high 16 bits, rows in low 16 bits. | + +Layer halves preserve the original 16-bit patterns; reinterpret them as signed +Int16 where an integration uses signed GDS layer values. Array counts originate +from positive Int16 values. Flattened coordinate lists below use `[x, y, ...]` +in integer database units. + +## References + +| SRef field | Arrow type | Meaning | +| --- | --- | --- | +| `target` | UInt32 | Target cell ID. | +| `invert_y` | Boolean | Reflection flag. | +| `scale` | Float64 | Magnification. | +| `angle_rad` | Float64 | Rotation in radians. | +| `xy` | UInt64 | Packed origin. | + +ARef adds `xy0` and `xy1` (UInt64 packed column/row increments) and `counts` +(UInt32). SRefProp and ARefProp add `properties: List(Property)`. + +Array increments must be exact signed 32-bit integers. All Arrow readers reject +fractional or out-of-range endpoint/count division rather than rounding it. +One-placement AREFs can be normalized to SREFs. The review CLI retains original +endpoints and distinguishes SREF from AREF independently of this representation. + +## Geometry and text + +Every `layer` below is a UInt32 **table index**, not a packed pair. + +| Type | Fields beyond `layer` | +| --- | --- | +| RectBatch | `rects: List(Int32)`, flattened `[min_x, min_y, max_x, max_y]` rectangles. | +| BoundaryBatch | `vertices: List(Int32)`; `vertex_offsets: List(UInt32)` gives each boundary's start in points, not scalar coordinates. | +| BoundaryProp | `vertices: List(Int32)`; `properties: List(Property)`. | +| BoxNode | `xy: List(Int32)`; `properties: List(Property)`. | +| Path | Fields below. | +| Text | Fields below. | + +| Path field | Arrow type | Meaning | +| --- | --- | --- | +| `path_type` | Int16, nullable | Stored path type. | +| `extension_start`, `extension_end` | Int32, nullable | End extensions. | +| `width` | Int32, nullable | Null when WIDTH is absent. | +| `xy` | List(Int32) | Centerline coordinates. | +| `properties` | List(Property) | Element properties. | + +| Text field | Arrow type | Meaning | +| --- | --- | --- | +| `presentation_horiz`, `presentation_vert`, `presentation_font` | UInt8, nullable | Presentation attributes. | +| `path_type` | Int16, nullable | Stored path type. | +| `width` | Int32, nullable | Stored width. | +| `invert_y` | Boolean, nullable | Reflection flag. | +| `mag` | Float64, nullable | Magnification. | +| `angle_deg` | Float64, nullable | Rotation in degrees. | +| `xy` | UInt64 | Packed position. | +| `string` | Utf8 | Label text. | +| `properties` | List(Property) | Element properties. | + +Property contains `key: Int16` and `value: Utf8`. The reader preserves property +order and duplicates. Property-list fields are nullable in the schema; the +reader normally emits valid lists, including empty ones. + +Geometry batching and reference normalization mean this layout is not a verbatim +record stream; use the [CLI's record dump](cli.md#inspect-a-layout) when original +records matter. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..2d66ea6 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,151 @@ +# Command-line tools + +Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. +The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets +refer to the decompressed layout. The CLI runs independently of Python. + +## Install + +Install the CLI [from crates.io](https://crates.io/crates/pheon) with Rust 1.88 +or newer. + +```sh +cargo install pheon --locked +pheon --help +``` + +From a source checkout, install with `cargo install --path . --locked --bin pheon` +or build with `cargo build --release --locked --bin pheon` and run +`target/release/pheon` (`pheon.exe` on Windows). + +## Inspect a layout + +```sh +pheon dump chip.gds +pheon dump chip.gds --records +pheon hierarchy chip.gds --format ascii --depth 4 +pheon hierarchy chip.gds --format mermaid --top CHIP +``` + +`dump` lists objects by cell. A prefix such as `3x` counts copies of the same +normalized object within a cell. Coordinates are in database units. Rectangles +show two corners, and longer vertex lists wrap across lines. References show +the target and any array dimensions before the coordinates. Add `--records` +to inspect the original record sequence, including timestamps. In color +output, the object dump highlights cell headings and layer numbers, and the +record dump highlights record names. + +`hierarchy` shows direct placement counts on references. `[subtree elided]` +marks a repeated cell whose children were skipped. Repeated leaves show +each parent's placement count without the marker. At the depth limit, +`[children omitted]` marks a cell with further descendants. Mermaid +output is diagram source for a Mermaid renderer. + +## Compare revisions + +Run `pheon diff before.gds after.gds` with your own files, or download these +small layouts from the pinned test corpus to try a comparison. +[small_rect.gds][small] · [big_rect.gds][big] + +```sh +pheon diff small_rect.gds big_rect.gds +``` + +The example exits with code 1 and shows one changed rectangle in `TOP`. + +```text +Different +Cells: +* TOP + 2/0: 1 rectangle changed +``` + +| Exit code | Meaning | +| --- | --- | +| 0 | Equal under the comparison rules and selected scope. | +| 1 | Differences found in a complete comparison. | +| 2 | Invalid input, a command error, or an incomplete comparison. | + +A code of 1 is an expected comparison result. A code of 2 can accompany definite +differences, but equality cannot be established. Read the +[comparison rules](comparison.md) before relying on results in automation, and +the [JSON reference](diff-reports.md) to consume reports programmatically. + +### Read the text report + +The cell list marks additions with `+`, removals with `-`, and modifications +with `*`. Modified cells summarize shape changes by layer and repeated +references by target. A single move can show its displacement in database units. +Layer names appear inline or in a legend. Object rows use `~` for likely +replacements and `+` or `-` for unmatched additions and removals. + +Use `--details` for individual objects, `--hierarchy ascii` for affected +parents, or `--format json` for machine-readable evidence. + +`diff` and `dump` color supported terminals by default. Set `NO_COLOR`, use +`TERM=dumb`, or pass `--color=never` for plain text. Pass `--color=always` when +piping into `less -R`. JSON and Mermaid diagram source stay plain. + +## Choose the comparison scope + +```sh +pheon diff before.gds after.gds --top CHIP --layer 1/0 +pheon diff before.gds after.gds --cell RESONATOR +``` + +| Option | Compared content | +| --- | --- | +| `--top NAME` | The named cell and its reachable descendants in each revision. | +| `--cell NAME` | One cell, including its references, without its descendants. | +| `--layer L/T\|NAME` | Objects on the selected layer/type pair, plus references leading to that layer. | + +Selectors combine: `--top CHIP --cell RESONATOR` selects that cell within CHIP's +subtree. A name absent from both revisions is an error; presence on only one +side is an addition or deletion. Layer selection omits cells irrelevant to that +layer, but library fields and metadata-version diagnostics remain relevant. +The complete input is parsed before scope selection, so malformed records +outside the scope still fail the command. + +Both `--layer` and `--show-layer` accept exact, case-sensitive names from the +embedded metadata in either revision. If a name refers to several numeric pairs, +Pheon lists the choices. A valid numeric pair always selects that pair directly. + +## Control report detail + +```sh +pheon diff before.gds after.gds --show-cell RESONATOR --show-layer WG +pheon diff before.gds after.gds --show-cell RESONATOR --details --max-changes 20 +pheon diff before.gds after.gds --hierarchy mermaid --depth 4 +``` + +`--show-cell` and `--show-layer` filter displayed changes without changing the +comparison, totals, or exit status. In text reports, `--show-layer` filters +object changes within modified cells and hides references without a numeric +layer. Added and removed cells appear as name-only entries outside the object +budget. Use `--show-cell` to filter those entries. + +`--max-changes N` limits displayed object occurrences after filtering. Text +defaults to 100 occurrences and shows a replacement only when both objects fit +within the limit. Grouped rows count each object toward the limit. When a +layer filter selects one side of a layer-only replacement, that side appears as +an addition or removal. The omission line lists changes hidden by filters or the +limit. JSON has no default object limit and records omissions in +`report_selection`. See the [JSON reference](diff-reports.md) for the selection +counters. + +`--details` shows individual object evidence inside modified cells in text +reports. Replacement rows include a nested `-` row for the old object and a +`+` row for the new object, with cell-local coordinates in database units, +retained fields, and properties. + +The diff hierarchy shows changed cells, affected ancestors, and modified +references from both revisions. Counts are direct placements, including +symbolic array counts. ASCII folds straight chains of unchanged edges and can +omit terminal edges already explained by the cell list. `--depth` counts the +original edges. For the full graph within the comparison scope, use +`--format json --hierarchy graph` without `--depth`. + +Next: configure [Git integration](git.md) or add [layer names](layer-names.md). + +[small]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/small_rect.gds +[big]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/big_rect.gds diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 0000000..739fab3 --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,59 @@ +# Comparison rules + +Pheon compares GDSII objects using the `pheon-structural-v1` profile. Use it to +review exact layout changes while ignoring incidental ordering and timestamps. +The [CLI guide](cli.md) explains how to choose cells and layers. + +## What counts as equal + +Cells match by name and layers by numeric layer/type pair. By default, the +comparison includes the entire library. + +Pheon ignores library and cell timestamps, cell order, element order, and +property order. Boundary and box rings are normalized for their starting vertex +and winding direction. Other vertices remain significant. Repeated objects and +properties are compared as multisets, so duplicates still count. + +Other retained details compare exactly: paths, labels, reference transforms, +array endpoints and counts, properties, and flags. An explicitly stored default +is distinct from an omitted field. An SREF is distinct from a one-element AREF. +REAL8 encodings are retained, so differently encoded values can compare as +changed even when their decoded values agree. + +Library fields, including UNITS, compare separately. Coordinates are not +rescaled to a shared grid. Cell renames are additions and deletions; Pheon does +not infer identity from similar content. It does not perform geometric Boolean +operations, XOR, or tolerance-based matching. + +## Read change explanations + +Pheon first pairs unmatched polygons, boxes, and paths that differ only in +layer or type, with exactly one candidate in each revision. It then checks the +remaining objects for unique matches by kind, layer, and bounds, followed by +kind and layer. Reference matching uses the target cell. Ambiguous candidates +are reported as separate removals and additions. These explanations apply to +the current report and preserve the underlying addition and removal counts. + +Reported bounds are envelopes of stored coordinates in cell-local database +units. They exclude path width, rendered text size, and recursively transformed +reference geometry. Account for UNITS before interpreting a physical dimension +or movement. + +Recognized [layer-name annotations](layer-names.md) are removed from ordinary +object comparison. Their name mapping compares separately, so a layer rename +appears once. + +## Know when a result is incomplete + +Unsupported records, future metadata versions, unresolved references, and +hierarchy cycles prevent a complete comparison. Pheon reports known differences +but returns 2 rather than claiming equality. + +Scope can exclude a record known to belong to another layer. Unsupported +records whose layer is unknown can still make a layer-scoped comparison +incomplete. Library fields and future metadata diagnostics remain visible. +Malformed input fails before scope is applied. + +For automation, check completeness as well as differences. The +[JSON report reference](diff-reports.md) describes the status fields and retained +evidence. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..1d960d7 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,68 @@ +# Development + +The checkout selects stable Rust. The minimum supported compiler is 1.88. +Use the checked-in `Cargo.lock` and update dependencies explicitly with Cargo. + +## Check a change + +| Area | Start here | First check | +| --- | --- | --- | +| GDSII parsing and Arrow reads | [`src/record.rs`](../src/record.rs), [`src/library.rs`](../src/library.rs) | `cargo test --locked --test parser_errors --test selective_reads` | +| Comparisons and CLI | [`src/review.rs`](../src/review.rs), [`src/review/`](../src/review/), [`src/bin/pheon.rs`](../src/bin/pheon.rs) | `cargo test --locked --test review` | +| Python and CFFI | [`pheon/`](../pheon/), [`src/lib.rs`](../src/lib.rs) | `python -I tools/check_installed.py` after installation | + +Run `cargo test --locked` for the full Rust suite or `cargo test --release --locked` +for release builds. Set `RUSTUP_TOOLCHAIN=1.88.0` to check the minimum compiler. + +## Test the Python package + +Use Python 3.11 or newer and install into a virtual environment. + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install . +python -I tools/check_installed.py +``` + +To test a newly built shared library, set `PHEON_LIBRARY` to its absolute path +and run `python -m unittest discover -s tests/python` from the checkout. Clear +the override when checking the bundled library with `check_installed.py`. + +## Check upstream layouts + +The [GDSII corpus](../contrib/gds-corpus.toml) pins open-source fixtures and cell +libraries. Install Pheon in the virtual environment before running the corpus +check. + +```sh +cargo build --release --locked --bin pheon +python -I tools/check_gds_corpus.py --fetch +``` + +The checker uses `curl` to cache layouts and license files in `.cache/gds-corpus/` +and verifies their SHA-256 digests. Omit `--fetch` to rerun offline. It checks +full reads, scans, selected reads, and CLI self-comparisons. It writes diagnostics +to `.cache/gds-corpus/report.json` and exits with status 1 if a check fails. + +## Interfaces and reports + +The CFFI declarations live in [`pheon/_ffi.h`](../pheon/_ffi.h). Keep them aligned +with the exported native functions. The C API version and +[Arrow layout version](arrow-schema.md) are independent of package releases and +the [JSON report schema](diff-reports.md). + +Review tests cover comparison semantics, CLI behavior, symbolic array counts, +and versioned report goldens. To regenerate JSON goldens after deliberately +changing their contract: + +```sh +PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens +``` + +Inspect the resulting diff. Goldens normalize the producer version to `test`. +Use the [JSON Schema](diff-report-v1.schema.json) to check structure. Rust tests +verify links, totals, and agreement between the result and exit status. + +See [building releases](releases.md) for source archives, wheels, and standalone +CLI artifacts. diff --git a/docs/diff-report-v1.schema.json b/docs/diff-report-v1.schema.json index c47d031..4427d90 100644 --- a/docs/diff-report-v1.schema.json +++ b/docs/diff-report-v1.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Klamath diff report version 1", - "description": "Object-profile comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", + "title": "Pheon diff report version 1", + "description": "Structural comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", "type": "object", "required": ["schema_version", "kind", "producer", "comparison_profile", "inputs", "scope", "coordinates", "status", "result", "exit_code", "summary", "report_selection", "layer_legend", "library_changes", "layer_name_changes", "cell_changes", "hierarchy", "diagnostics"], "properties": { @@ -9,7 +9,7 @@ "kind": {"const": "diff"}, "producer": { "type": "object", "required": ["name", "version"], - "properties": {"name": {"const": "klamath"}, "version": {"type": "string"}} + "properties": {"name": {"const": "pheon"}, "version": {"type": "string"}} }, "comparison_profile": {"type": "string"}, "inputs": { diff --git a/docs/diff-reports.md b/docs/diff-reports.md index 1557109..b71b0ab 100644 --- a/docs/diff-reports.md +++ b/docs/diff-reports.md @@ -1,185 +1,131 @@ -# Machine-readable diff reports +# JSON diff reports + +Use JSON reports to automate layout review while retaining the evidence behind +a result. The [comparison rules](comparison.md) define equality; the +[version 1 JSON Schema](diff-report-v1.schema.json) defines every field. +A [complete example](https://mpxd.net/code/jan/pheon/src/branch/master/tests/golden/report-geometry.json) shows geometry changes. ```sh -klamath diff before.gds after.gds --format json > report.json -klamath diff before.gds after.gds --format json --hierarchy graph -klamath diff before.gds after.gds --format json --top CHIP --layer 1/0 +pheon diff before.gds after.gds --format json > report.json +pheon diff before.gds after.gds --format json --hierarchy graph +pheon diff before.gds after.gds --format json --top CHIP --layer 1/0 ``` -The command writes one UTF-8 JSON document followed by a newline to stdout. -Comparison diagnostics belong to that document. Output write failures are -reported on stderr and return 2; interrupted output is not a valid report. -JSON output currently belongs to `diff`; `dump`, `hierarchy`, and the Git -external driver retain their existing text interfaces. No external process is -used to produce a report. +`diff` writes one UTF-8 JSON document and a newline to stdout, including comparison +diagnostics. Output write failures go to stderr and return 2; interrupted output +is invalid. Other CLI commands retain text output. -The [version 1 JSON Schema](diff-report-v1.schema.json) defines the structure. -[Golden reports](../tests/golden/report-geometry.json) illustrate complete -evidence; other `report-*.json` goldens cover equality, metadata, incomplete -comparison, and failure. Their producer version is replaced with `test` so -package version changes do not alter the fixtures. +## Check the result first -## Versions and determinism - -`schema_version` describes the report structure. `comparison_profile` names the -equality rules, currently `klamath-object-v2`. `producer.version` identifies the -implementation. These versions are independent. Consumers interpreting equality -must understand the comparison profile as well as the schema. - -Compatible additions may introduce optional fields. Consumers should ignore -unfamiliar additional fields and handle unfamiliar diagnostic codes and matching -rules without treating them as equality. Removed fields or changed meanings -require a new schema version. The schema intentionally permits additional fields. - -For the same inputs, paths, options, and producer version, serialization is -deterministic. Cells, layer pairs, fields, properties, and graph edges are sorted. -Duplicate objects and properties remain separate occurrences. Reports contain -no generation timestamp or required content fingerprint. - -## Result and exit status - -| `status` | `result` | Exit code | Interpretation | +| `status` | `result` | Exit code | Meaning | | --- | --- | --- | --- | | `complete` | `equal` | 0 | Equal under the recorded profile and scope. | | `complete` | `different` | 1 | Differences found. | -| `incomplete` | `different` | 2 | Definite differences found, with comparison limitations. | +| `incomplete` | `different` | 2 | Definite differences, with comparison limitations. | | `incomplete` | `unknown` | 2 | Equality cannot be established. | -| `failed` | `unknown` | 2 | No comparison result is available. | +| `failed` | `unknown` | 2 | No comparison result. | -Unsupported records, future metadata versions, unresolved references, and -hierarchy cycles make the comparison incomplete. Malformed input, read errors, -invalid scopes, and invalid command arguments produce failed reports. CI must -check completeness as well as differences. An exit code of 1 is a normal -comparison outcome, including when stdout is redirected to a report file. +A code of 1 is an expected result, including when saving a report. Unsupported +records, future metadata, unresolved references, and cycles make a comparison +incomplete. Malformed input, I/O errors, invalid scopes, and command errors fail. -Library and cell timestamps are excluded from the comparison profile. A valid, -complete timestamp-only comparison returns 0. Raw record dumps remain available -for examining those timestamps. +`summary` covers the whole scoped comparison before display filters. Counts are +`exact` for complete results and `observed` for incomplete ones. Failed reports +have a null summary and empty change arrays. Replacement suggestions and affected +ancestors do not add to change counts. -`summary` counts the entire scoped comparison, before display selection. Its -`counts` is `exact` for a complete comparison and `observed` for an incomplete -one. Failed reports have a null summary and empty change arrays. Replacement -suggestions and affected ancestors never add to change counts. +## Scope and evidence selection -## Scope, selection, and numbers +`scope` records `top`, `cell`, and numeric `layer` selectors; null means +unrestricted. The full input is parsed before scope selection. -`scope` records the actual `top`, `cell`, and numeric `layer` selectors. Null -means unrestricted. The complete file is parsed before applying scope, so an -invalid record outside the selected subtree still fails the command. +`report_selection` records display-only cell/layer filters and `max_objects`. +JSON is unbounded by default. Explicit `--max-changes N` limits total emitted +removals plus additions, with removals first within each cell. Selection counters +`filtered_cells`, `filtered_objects`, `limited_objects`, and `emitted_objects` +explain omissions. `truncated` is true when filtering or limits omit evidence. +Library changes, layer-name changes, diagnostics, summaries, and the optional +graph remain. Replacement links appear only if both objects are emitted. -`report_selection` records display-only `cell`, `layer`, and `max_objects`. -JSON has no default object limit. Explicit `--max-changes N` limits the total -number of emitted object occurrences across both sides. Removals precede -additions within each cell. `filtered_cells`, `filtered_objects`, -`limited_objects`, and `emitted_objects` explain evidence selection. `truncated` -is true if filtering or a limit omits any changed cell or object. Library and -layer-name changes, diagnostics, summaries, and the optional graph are retained. -A replacement is emitted only when both linked objects are included. +## Read values safely -All aggregate counts, limits, and byte offsets are unsigned decimal **strings**. -This avoids loss of precision in JSON consumers using binary64 numbers. -Coordinates, numeric tags, layer/type pairs, widths, and array dimensions are -bounded integers represented as JSON numbers. Null means absent or unavailable; -zero and empty arrays do not imply absence. +Aggregate counts, limits, and byte offsets are unsigned decimal **strings** to +avoid JSON number precision loss. Coordinates, numeric tags, layer/type pairs, +widths, and array dimensions are bounded JSON numbers. Null means absent or +unavailable; zero and empty arrays remain meaningful values. -`inputs.old` and `inputs.new` contain input paths, presence, and UNITS fields. -Presence is distinct from an empty library. The Rust API also supports absent -Git revisions using `Document::empty_like`; no units are invented for absent -inputs. Failed reports may have null presence and units. Argument failures can -lack input/selection context because the command could not be parsed. +`inputs.old` and `inputs.new` record paths, presence, and UNITS. An absent revision +is distinct from an empty library. The Rust API supports absent Git revisions +with `Document::empty_like`. Failed reports may lack presence, units, or argument +context. Non-UTF-8 paths use null `path` and `path_encoded_hex`: `0x`-prefixed +platform-native Rust path bytes, not a portable filename encoding. -Paths are UTF-8 when representable. Otherwise `path` is null and -`path_encoded_hex` preserves Rust's platform-native encoded path bytes, prefixed -with `0x`. This is platform-dependent evidence rather than a portable path name. +## Interpret changes -## Change evidence +| Section | Evidence | +| --- | --- | +| `library_changes` | Typed before/after header fields, including unscaled UNITS. Presence changes use a null tag and boolean values. | +| `layer_name_changes` | Before/after names for numeric pairs `{ "number": 1, "purpose": 0 }`. | +| `layer_legend` | Unchanged names available in scope. Annotation labels do not also appear as object changes. | +| `cell_changes` | Name, nullable before/after cell records, attributes, removed/added objects, and replacement suggestions. A null side means absent; an existing empty cell still has a byte offset. | -`library_changes` records changed header fields as typed before/after arrays. -Library presence changes use the special field `presence`, a null tag, and -boolean before/after values. Units are not rescaled to a common grid. +Objects retain kind, numeric tag, layer, target, bounds, source side and half-open +byte range, typed fields, and a sorted property multiset including duplicates. +Convenience fields include origin, width, and reference dimensions. SREF counts +are `[1, 1]`; kind and retained fields still distinguish a one-element AREF. -`layer_name_changes` records numeric pairs with before/after names. A layer -pair is `{ "number": 1, "purpose": 0 }`, where purpose is DATATYPE, TEXTTYPE, -BOXTYPE, or NODETYPE according to the object kind. `layer_legend` contains -unchanged names available in the scope. Embedded annotation labels are excluded -from ordinary object changes. A metadata rename appears once. +Coordinates are cell-local database units. Bounds are stored-coordinate +envelopes, excluding stroke width, rendered text, and recursive target geometry. +Consider UNITS before deriving physical dimensions. -Each `cell_changes` entry has its name, nullable before/after cell records, -attribute evidence, removed and added objects, and replacement suggestions. -A null cell side means the cell is absent in that scope. An existing empty -cell has a byte offset and an empty attribute array. +Fields contain `tag`, display `name`, `encoding`, decoded `value`, and +`payload_hex`. Numeric tags are authoritative. XY values are lists of points; +other numeric values are arrays. Unsupported encodings have null values and +preserved bytes. Invalid UTF-8 strings/properties have null decoded values; +hex preserves their bytes. -Each object retains: +Evidence follows the comparison profile: boundary/box rings are canonicalized, +string padding is removed, and fields/properties are sorted. REAL8 encodings and +explicit defaults remain significant. For gzip inputs, source offsets locate +records in the decompressed layout. -- Its kind and numeric tag, numeric layer pair, target cell, and coordinate envelope. -- Its original source side and byte range `[byte_offset, end_offset)`. -- Typed field records and the sorted property multiset, including duplicates. -- Convenience fields for text/reference origin, path/text width, and reference - array dimensions. SREF dimensions are `[1, 1]`; the kind and retained fields - distinguish an SREF from a one-element AREF and explicit fields from omission. - -Coordinates are cell-local integer database units. `bounds` contains -`[xmin, ymin, xmax, ymax]` of stored coordinates. It excludes path stroke width, -rendered text dimensions, and recursively transformed target geometry. -Equal envelopes do not establish geometric equality. Unit changes must be -considered before deriving physical movements or dimensions. - -A retained field has a numeric `tag`, convenience `name`, `encoding`, decoded -`value`, and `payload_hex`. The numeric tag is authoritative; names are display -aids. XY values are arrays of `[x, y]` pairs. Other numeric fields are arrays -of their declared type. Unsupported encodings retain bytes with a null value. -String and property values are null if their bytes are not valid UTF-8; -`payload_hex` always preserves them without replacement characters. - -Payloads follow the comparison profile: boundary/box rings use canonical start -and winding, string padding is removed, and fields/properties are sorted. They -are comparison evidence, not a verbatim serialization of the source records. -REAL8 payload encodings are retained exactly alongside their decoded values. -Explicit defaults and omitted fields remain distinct. Source ranges allow raw -record inspection when original encoding or order is needed. - -Replacement suggestions link report-local removed/added IDs, with a `match_rule`, -typed `field_changes`, and `properties_changed`. Their explanation is optional -for consumers to display and must not be parsed. Current matching rules are -`unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. -These are explanatory matches, not persistent identity or shape equivalence. -Ambiguous candidates stay independent removals and additions. IDs are unique -within a report and have no meaning across reports. +Replacements link report-local removed/added IDs with `match_rule`, +`field_changes`, and `properties_changed`. The `unique_layer_change` rule +matches a unique polygon, box, or path on each side with identical retained +fields and properties outside the layer and datatype fields. The other rules +are `unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. +Ambiguous candidates stay separate. Replacement IDs refer only to objects in +the current report. Explanation text is for display only. ## Diagnostics and hierarchy -Diagnostics have `code`, `severity`, `message`, input `side`, original byte -offset, cell, layer, and target context. Unknown context is null. Messages are -for display; codes and context support automation. Current limitation codes are -`unsupported_record`, `unsupported_metadata_version`, `unresolved_reference`, -`hierarchy_cycle`, and `scope_mismatch`. Failure codes are `invalid_input`, -`io_error`, `invalid_scope`, and `command_error`. Fatal parser errors currently -retain their location in the human message; their structured byte offset is -null. Consumers must not depend on message wording. +Diagnostics provide `code`, `severity`, `message`, input side, byte offset, cell, +layer, and target; unknown context is null. Use codes and context for automation. +Fatal parser locations currently appear only in the human message, with null +structured offsets. Do not parse message wording. -`--hierarchy graph` includes the full scoped union graph of both revisions. -Otherwise `hierarchy` is null. Nodes are keyed by cell name, with per-side -states `included`, `outside_scope`, `unresolved`, or `absent`. Their `change` -is `direct`, `affected` through a descendant, or `unchanged`. +Limitation codes are `unsupported_record`, `unsupported_metadata_version`, +`unresolved_reference`, `hierarchy_cycle`, and `scope_mismatch`. Failure codes are +`invalid_input`, `io_error`, `invalid_scope`, and `command_error`. -Edges record parent, target, before/after direct placement counts, and whether -the underlying reference records changed. AREF columns times rows contribute -symbolically; arrays are never expanded. Counts are direct edge multiplicities, -not recursively accumulated placements. Shared cells occur once. Removed edges -remain visible, and ancestors are computed over the union of both revisions. -Graph content is independent of display filters and object limits. Text depth -limits and ASCII/Mermaid choices do not apply to JSON graphs. +`--hierarchy graph` includes the full scoped union graph. By default, `hierarchy` +is null. JSON output rejects `--depth` with exit code 2. Nodes are keyed by name, +with per-side states `included`, `outside_scope`, `unresolved`, or `absent`, and +change states `direct`, `affected`, or `unchanged`. +Edges record parent, target, before/after direct placement counts, and changed +reference records. Array multiplicities are symbolic, not expanded or recursively +accumulated. Shared cells occur once and removed edges remain. The graph covers +the full comparison scope regardless of display filters and object limits. -## Development +## Versions -`cargo test --test review` checks report semantics, CLI behavior, and versioned -goldens. To intentionally regenerate report goldens after reviewing a contract -change, run: +`schema_version` is 1, `comparison_profile` is `pheon-structural-v1`, and +`producer.name` is `pheon`. Structure, equality rules, and producer version evolve +independently. Understand the profile before interpreting equality. Consumers +should tolerate additional optional fields and unknown codes/rules without +assuming equality. Removed fields or changed meanings require a schema change. -```sh -KLAMATH_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens -``` - -Schema validators can validate the goldens or actual CLI output against -`diff-report-v1.schema.json`. Schema validation does not replace semantic checks -such as replacement links, summary totals, or status/exit-code agreement. +Serialization is deterministic for identical inputs, paths, options, and producer +version. Sorted evidence retains duplicates. Reports have no generation timestamp +or required content fingerprint. See [development](development.md) for contract +tests and golden updates. diff --git a/docs/git.md b/docs/git.md new file mode 100644 index 0000000..2a571d0 --- /dev/null +++ b/docs/git.md @@ -0,0 +1,63 @@ +# Git integration + +Configure Git to show meaningful GDSII changes during review. Install the +[Pheon CLI](cli.md#install) first, then choose one of the following drivers. +Templates are also available in [contrib/git](../contrib/git/). + +## Object comparison + +Use the external driver for object-aware comparison and affected hierarchy. +Add this to your repository's `.gitattributes`: + +```gitattributes +*.gds diff=pheon +``` + +Add this to `.git/config`, or use the same section in your user Git config: + +```gitconfig +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false +``` + +Then run `git diff` as usual. The driver reads Git's temporary old and new files +and handles additions, deletions, and renames. The heading names the file and +indicates whether it was added or deleted. Keep the final `--` because Git appends +its path and revision arguments after it. Put Pheon options such as `--top CHIP` +or `--show-layer 1/0` before that marker. + +The external driver also accepts `--color=always` before `--` to keep colors +when its output goes through a pager that supports ANSI colors. + +For an existing `diff=layout` attribute, put the configuration under +`[diff "layout"]` instead. Attribute names only select a Git driver. + +The external driver returns 0 after rendering either an equal or different +comparison, and nonzero for invalid or incomplete comparisons. For automation +that needs equality status, run `pheon diff OLD NEW` directly; it returns +0, 1, or 2 as described in the [CLI guide](cli.md). + +## Normalized text comparison + +Use text conversion when you want Git to compare normalized text dumps. Set +`.gitattributes` to: + +```gitattributes +*.gds diff=pheon-text +``` + +Configure the driver: + +```gitconfig +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true +``` + +Git can cache converted text. Invalidate that cache when upgrading to a tool or +comparison profile that changes the dump format, or disable `cachetextconv`. +These diffs are for human review; they are not patches that can be applied to +GDSII files. See the [comparison rules](comparison.md) for normalization details. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f4e876d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,19 @@ +# Pheon documentation + +Start with a guide for the task you want to perform. + +| Task | Guide | +| --- | --- | +| Inspect layouts, compare revisions, or view hierarchy | [Command-line tools](cli.md) | +| Load geometry or selected cells in Python | [Python](python.md) | +| Review GDSII files with Git | [Git integration](git.md) | +| Add readable names to numeric layers | [Layer names](layer-names.md) | +| Understand what a comparison considers equal | [Comparison rules](comparison.md) | + +For integrations, use the [Arrow layout reference](arrow-schema.md), +[Rust and C API guide](native-api.md), or [JSON report reference](diff-reports.md). + +For contributors and maintainers, see [development](development.md) and +[building releases](releases.md). + +Return to the [project introduction](../README.md). diff --git a/docs/layer-names.md b/docs/layer-names.md new file mode 100644 index 0000000..505b08d --- /dev/null +++ b/docs/layer-names.md @@ -0,0 +1,46 @@ +# Layer names + +Attach readable names to numeric GDSII layer/type pairs so Pheon can show a +legend in comparison reports. Names are annotations; geometry keeps its numeric +layer assignments. + +## Add or update names + +```sh +pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 \ + --name 1:0:WG --name 2:0:METAL +``` + +Choose an existing cell and an otherwise unused `(layer, texttype)` pair in that +cell as the carrier. The carrier must contain no ordinary design objects. +The output path must be new; Pheon refuses to overwrite any existing file, +including the input. + +The output is uncompressed GDSII, including when the input is gzip-compressed. + +The command consolidates the legend into that cell. Existing mappings remain +unless you override them with `--name`. It creates no cells or references, and +copies other records, timestamps, unknown records, and trailing padding byte for +byte. Input and annotation validation finish before the output is created. + +## Annotation format + +A text label whose exact string is `PHEON_LAYER_NAMES_V1` identifies a carrier +pair in its cell. Every other text label on that pair in the same cell is a +mapping of the form: + +```text +layer:datatype:name +``` + +For example, `1:0:WG` names pair `(1, 0)` as WG. Numbers are unsigned 16-bit +decimals. The name must be nonempty UTF-8 and may contain colons; only the first +two colons separate fields. + +Mappings are collected across the library. Identical duplicates are accepted; +conflicting names for the same pair are errors. Recognized annotation labels +are excluded from ordinary object comparison. The name mapping compares +separately, while unchanged names remain available in the report legend. + +Pheon does not currently read LayoutEditor LayerNames metadata, extract layers, +or remap geometry. See the [CLI guide](cli.md) for layer selection during review. diff --git a/docs/native-api.md b/docs/native-api.md new file mode 100644 index 0000000..01ddcf4 --- /dev/null +++ b/docs/native-api.md @@ -0,0 +1,102 @@ +# Rust and C integration + +Use the Rust library or C API to read GDSII into Arrow without Python. For the +returned data layout, see the [Arrow reference](arrow-schema.md). + +## Rust + +Add [pheon](https://crates.io/crates/pheon) from your Rust project. + +```sh +cargo add pheon +``` + +The library readers accept GDSII byte slices. + +```rust +use pheon::library::read_library; + +let bytes = std::fs::read("chip.gds").expect("read layout"); +let (_remainder, layout) = read_library(&bytes).expect("parse layout"); +println!("{layout:?}"); +``` + +`read_library` reads geometry; `scan_library` returns metadata, references, and +cell ranges. Pass `(start, end)` ranges from a scan to `read_selected_library` +to load only those cells. Each range must span one complete cell. Readers return +`(remainder, StructArray)` on success. The selected reader returns the original +input as its remainder, since selection is random access. + +Layout-review APIs live in `pheon::review`, with layer-name helpers in +`pheon::review::names` and scope/display selectors in `pheon::review::selection`. + +`Comparison::between(&old, &new)` captures an owned result with revision metadata, +changes, and structured diagnostics. Use `outcome()` for completeness, equality, +and exit status. Its renderers work after the source documents and byte buffers +are dropped. `json_report` returns a JSON value, and `write_json_report` writes +object evidence incrementally to a writer. `Document::hierarchy` inspects a +single revision. A comparison captures reference counts and analyzes the union +graph and ancestors when a hierarchy is requested. + +`TextComparison::between` captures the data needed for text and hierarchy +reports. Both comparison types provide `render_text_with_ascii_hierarchy` +to render a combined report with redundant terminal edges omitted. Pass +`TextReportOptions { details: true, ..Default::default() }` to +`render_text_with_options` to show individual object evidence. +Use `TextReport::new(true)` with `write_text_report`, `write_canonical_dump`, +or `write_record_dump` to get plain text and style ranges for presentation. + +Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison +that can reuse matching cells. Its `compare` and `compare_text` methods return +the corresponding owned comparison. + +The `record`, `records`, and `basic` modules also provide low-level GDSII record +writing. There is no high-level writer for the Arrow layout. REAL8 writing is +exact for representable in-range `f64` values: magnitudes below `2^-260` +underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected. + +Record writes check payload sizes before emitting bytes. Header packing returns +an `io::Result`, including an error when the total size exceeds the format limit. +Arrow readers return parse errors for invalid UTF-8 strings and timestamp years +that exceed the Int16 range after conversion from the 1900 epoch. + +## C API + +Build the shared library with `cargo build --release --locked`. The +[header](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h) defines the functions and Arrow C Data structures. +The library is `libpheon.so`, `libpheon.dylib`, or `pheon.dll` on Linux, macOS, +or Windows respectively. + +| Function | Input | Result | +| --- | --- | --- | +| `read_path` | NUL-terminated path | Full layout. | +| `read_bytes` | Byte pointer and size | Full layout. | +| `scan_bytes` | Byte pointer and size | Metadata, references, and cell ranges. | +| `read_cells_bytes` | Bytes and cell ranges | Full layout for selected cells. | + +`read_path` accepts UTF-8 paths on Windows and filesystem bytes on Unix. It reads +uncompressed GDSII; Python's `read_path` adds gzip handling. + +For `read_cells_bytes`, `ranges` points to alternating uint64 start/end offsets +and `range_count` is the number of cells, not the number of offsets. Ranges are +half-open, from BGNSTR through the end of ENDSTR. Range values must be aligned, +native-endian uint64 values. Input buffers must remain readable for the call. +Null buffer pointers are allowed only when their corresponding length is zero. + +### Status and ownership + +Each reader returns 0 on success and -1 on failure. Call `last_error_message()` +on the same thread after a failure. It returns a borrowed NUL-terminated message; +copy it before the next parser call or thread exit. It returns null after a +successful parser call. + +Supply valid, writable, initialized `ArrowArray` and `ArrowSchema` structures +without live allocations to overwrite. A failed call leaves them unchanged. +Successful outputs own their storage independently of the input buffer. Transfer +ownership to an Arrow importer, or call each non-null release callback exactly +once. Do not release structures again after an importer takes ownership. + +`pheon_abi_version()` and `pheon_arrow_schema_version()` both currently return 1. +Check both before using the API. The C calling/ownership contract and Arrow +layout have separate versions; neither is the package version or the +[JSON report schema](diff-reports.md). diff --git a/docs/python.md b/docs/python.md new file mode 100644 index 0000000..f02fffb --- /dev/null +++ b/docs/python.md @@ -0,0 +1,91 @@ +# Python + +Use `pheon.arrow` to load GDSII geometry into PyArrow, scan a hierarchy, or read +only selected cells. The package also provides integer packing. +Python 3.11 or newer is required. + +## Install + +Install Pheon from [PyPI](https://pypi.org/project/pheon/). + +```sh +python -m pip install pheon +``` + +The package installs PyArrow for layout reads. Existing `pheon[arrow]` +requirements still work. [Prebuilt wheels](https://pypi.org/project/pheon/#files) +bundle the native library. On platforms without a compatible wheel, pip builds +the library from source and needs Rust 1.88 or newer and a native linker. + +From a source checkout, install with `python -m pip install .`, then run Python +outside the checkout to import the installed package. + +## Read a layout + +```python +from pheon import arrow + +layout = arrow.read_path("chip.gds") +names = layout.field("cell_names")[0].as_py() +cells = layout.field("cells")[0].values +print([names[cell_id] for cell_id in cells.field("id").to_pylist()]) +``` + +`read_path` accepts strings and path-like objects, expands `~`, and recognizes +gzip by its magic bytes. `read_bytes` accepts an uncompressed, contiguous byte +buffer, such as `bytes`, `bytearray`, or a compatible memoryview. + +Every reader returns a one-row `pyarrow.StructArray`. The example uses each +cell's ID to look up its name. The name table can also contain reference targets +without cell definitions. Geometry is grouped by cell and element kind. The +returned arrays own their storage independently of the input buffer. +See the [Arrow reference](arrow-schema.md) for fields and encodings. + +## Scan, then read selected cells + +```python +from pathlib import Path +from pheon import arrow + +content = Path("chip.gds").read_bytes() # Uncompressed GDSII bytes. +scan = arrow.scan_bytes(content) +cells = scan.field("cells")[0].values +ranges = list(zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), +)) +selected = arrow.read_cells_bytes(content, ranges[:2]) +``` + +Scanning returns library metadata, cell names, references, and complete cell byte +ranges without decoding geometry or text. Selected reads return the full layout +schema for the requested cells, in the requested order. Use the selected result's +own name and layer tables: IDs may differ between reads. + +Ranges must have shape `(N, 2)` with nonnegative integer offsets. Each half-open +range must span exactly one complete cell, from BGNSTR through ENDSTR. An empty +list returns metadata without cells. For compressed files, decompress the bytes +before scanning or selecting ranges. + +## Pack integers + +```python +import pheon + +int2_bytes = pheon.pack_int2([1, 2, 3]) +int4_bytes = pheon.pack_int4([100, 200]) +``` + +These helpers return big-endian GDSII integer bytes. They accept NumPy arrays, +Python sequences, or scalars and preserve the caller's data, including for +noncontiguous or differently ordered arrays. Pheon currently has no high-level +Python layout writer. + +## Handle read errors + +Native parse failures raise `pheon.PheonError`, a subclass of `ValueError`. +Invalid buffer or range arguments raise `ValueError` or `TypeError`; file access +errors use Python's normal I/O exceptions. + +See [development](development.md) to test a locally built native library from +Python, or the [native API guide](native-api.md) to call the library directly. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..c36c7ba --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,95 @@ +# Building releases + +Build packages locally for distribution. The checkout holds Python build tools +in `.venv/`, release artifacts in `dist/release/`, and Linux wheel caches in +`.cache/wheels/`. Use your installed Rust toolchain and Cargo registry cache. + +## Start a release + +Use Python 3.11+ and Rust 1.88+. Run the commands from the repository root and +start with an empty `dist/release/` directory. + +```sh +python3 -m venv .venv +. .venv/bin/activate +export PIP_CACHE_DIR="$PWD/.cache/wheels/pip" +python -m pip install build twine +python -m build --sdist --outdir dist/release +``` + +Use `python -m build --wheel` to build a wheel for your own machine. For a +portable Linux wheel, use the container below. See [development](development.md) +for compiler checks. + +## Build Linux x86-64 wheels + +On x86-64 Linux, use rootless Podman and a Rust toolchain installed through +`rustup`. Build the portable wheel from the source archive. + +```sh +python -m pip install 'cibuildwheel==4.3.5' +python tools/build_manylinux.py dist/release/pheon-*.tar.gz +``` + +The helper pulls the [pinned image](../contrib/wheels/manylinux-image.txt) when +needed and reuses it from `.cache/wheels/podman/storage/`. It keeps Podman’s +storage runtime directory, download scratch space, and wheel caches under +`.cache/wheels/`. Podman still uses the normal system user runtime for host +services. To inspect this checkout’s images, use the same helper. + +```sh +python tools/build_manylinux.py podman image ls +``` + +cibuildwheel repairs the wheel and tests it on Python 3.11–3.14. The container +uses the host Rust toolchain and Cargo registry through read-only mounts. + +## Build Windows x86-64 packages from Linux + +Install MinGW-w64 with `x86_64-w64-mingw32-gcc` on `PATH`. From the project +environment, build the Windows wheel and the standalone CLI: + +```sh +rustup target add x86_64-pc-windows-gnu +PHEON_BUILD_TARGET=x86_64-pc-windows-gnu python -m build --wheel --outdir dist/release +cargo build --locked --release --target x86_64-pc-windows-gnu --bin pheon +``` + +The `py3-none-win_amd64` wheel lands in `dist/release/`. The executable is +`target/x86_64-pc-windows-gnu/release/pheon.exe`. Test the wheel and CLI on Windows +before publishing. + +## Validate artifacts + +Check the Python package metadata and verify that the packaged crate builds. + +```sh +python -m twine check --strict dist/release/*.tar.gz dist/release/*.whl +cargo publish --dry-run --locked +``` + +Wheel filenames use `py3-none` plus a platform tag, such as +`manylinux_2_28_x86_64` or `win_amd64`. Use repaired `manylinux_*` wheels for +Linux releases. + +To test a wheel manually, install it in a test environment on the target OS. +Clear `PHEON_LIBRARY` so the checks load the bundled library. Replace the +example path with the wheel you built. + +```sh +python -m pip install "path/to/pheon-wheel.whl" +python -I tools/check_installed.py +``` + +Build standalone CLI artifacts for each OS and architecture. Include the +executable, license, README, documentation, and Git templates. + +## Publish + +Upload the validated Python files you selected, then publish the crate from a +clean release checkout. + +```sh +python -m twine upload --repository pypi FILE ... +cargo publish --locked +``` diff --git a/klamath_rs_ext/__init__.py b/klamath_rs_ext/__init__.py deleted file mode 100644 index 2916461..0000000 --- a/klamath_rs_ext/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .basic import pack_int2 as pack_int2 -from .basic import pack_int4 as pack_int4 - - -__version__ = "0.2.0" - diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py deleted file mode 100644 index c7bcd8e..0000000 --- a/klamath_rs_ext/basic.py +++ /dev/null @@ -1,92 +0,0 @@ -from collections.abc import Sequence -import ctypes -from pathlib import Path -from itertools import chain - -import numpy -from numpy.typing import NDArray - -from .klamath_rs_ext import lib, ffi - - -CONV_TABLE_i16 = { - numpy.float64: lib.f64_to_i16, - numpy.float32: lib.f32_to_i16, - numpy.int64: lib.i64_to_i16, - numpy.int32: lib.i32_to_i16, - numpy.int16: lib.i16_to_i16, - numpy.uint64: lib.u64_to_i16, - numpy.uint32: lib.u32_to_i16, - numpy.uint16: lib.u16_to_i16, - } - -CONV_TABLE_i32 = { - numpy.float64: lib.f64_to_i32, - numpy.float32: lib.f32_to_i32, - numpy.int64: lib.i64_to_i32, - numpy.int32: lib.i32_to_i32, - numpy.uint64: lib.u64_to_i32, - numpy.uint32: lib.u32_to_i32, - } - - - -def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.atleast_1d(data) - - for dtype in CONV_TABLE_i16.keys(): - if arr.dtype != dtype: - continue - - arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) - if arr is data: - arr = numpy.array(arr, copy=True) - - fn = CONV_TABLE_i16[dtype] - buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) - result = fn(buf, arr.size) - - if result != 0: - raise ValueError(f'Invalid value for conversion to Int2: {result}') - - i2arr = arr.reshape(-1).view('>i2')[::arr.itemsize // 2] - return i2arr.tobytes() - - if arr.dtype == numpy.dtype('>i2'): - return arr.tobytes() - - if (arr > 32767).any() or (arr < -32768).any(): - raise Exception(f'int2 data out of range: {arr}') - - return arr.astype('>i2').tobytes() - - -def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.atleast_1d(data) - - for dtype in CONV_TABLE_i32.keys(): - if arr.dtype != dtype: - continue - - arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) - if arr is data: - arr = numpy.array(arr, copy=True) - - fn = CONV_TABLE_i32[dtype] - buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) - result = fn(buf, arr.size) - - if result != 0: - raise ValueError(f'Invalid value for conversion to Int4: {result}') - - i4arr = arr.reshape(-1).view('>i4')[::arr.itemsize // 4] - return i4arr.tobytes() - - if arr.dtype == numpy.dtype('>i4'): - return arr.tobytes() - - if (arr > 2147483647).any() or (arr < -2147483648).any(): - raise Exception(f'int4 data out of range: {arr}') - - return arr.astype('>i4').tobytes() - diff --git a/klamath_rs_ext/klamath_rs_ext.py b/klamath_rs_ext/klamath_rs_ext.py deleted file mode 100644 index 0fc0ccd..0000000 --- a/klamath_rs_ext/klamath_rs_ext.py +++ /dev/null @@ -1,23 +0,0 @@ -"""CFFI ABI access to the Cargo-built shared library.""" -import os -from pathlib import Path -import sys - -from cffi import FFI - - -ffi = FFI() -header = Path(__file__).with_name("_ffi.h").read_text() -ffi.cdef("\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#"))) - -if sys.platform == "win32": - filename = "klamath_rs_ext.dll" -elif sys.platform == "darwin": - filename = "libklamath_rs_ext.dylib" -else: - filename = "libklamath_rs_ext.so" - -# An explicit override supports source-tree development without copying binaries -# into the package. Installed wheels load only their bundled native library. -library = Path(os.environ["KLAMATH_RS_LIBRARY"]) if "KLAMATH_RS_LIBRARY" in os.environ else Path(__file__).with_name(filename) -lib = ffi.dlopen(str(library.resolve())) diff --git a/pheon/__init__.py b/pheon/__init__.py new file mode 100644 index 0000000..535f808 --- /dev/null +++ b/pheon/__init__.py @@ -0,0 +1,10 @@ +"""Native GDSII helpers. Use pheon.arrow to read layouts into PyArrow.""" + +from importlib.metadata import version as _version + +from ._native import PheonError as PheonError +from .basic import pack_int2 as pack_int2 +from .basic import pack_int4 as pack_int4 + + +__version__ = _version("pheon") diff --git a/klamath_rs_ext/_ffi.h b/pheon/_ffi.h similarity index 88% rename from klamath_rs_ext/_ffi.h rename to pheon/_ffi.h index 273067e..6bca31f 100644 --- a/klamath_rs_ext/_ffi.h +++ b/pheon/_ffi.h @@ -1,5 +1,5 @@ -#ifndef KLAMATH_RS_EXT_H -#define KLAMATH_RS_EXT_H +#ifndef PHEON_H +#define PHEON_H #include #include @@ -28,7 +28,10 @@ struct ArrowSchema { void *private_data; }; +uint32_t pheon_abi_version(void); +uint32_t pheon_arrow_schema_version(void); int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema); const char *last_error_message(void); diff --git a/pheon/_native.py b/pheon/_native.py new file mode 100644 index 0000000..fc9dbd7 --- /dev/null +++ b/pheon/_native.py @@ -0,0 +1,55 @@ +"""Load a bundled or explicitly selected library through its stable C ABI.""" + +import os +from pathlib import Path +import sys + +from cffi import FFI + + +ffi = FFI() +header = Path(__file__).with_name("_ffi.h").read_text() +ffi.cdef( + "\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")) +) + +if sys.platform == "win32": + filename = "pheon.dll" +elif sys.platform == "darwin": + filename = "libpheon.dylib" +else: + filename = "libpheon.so" + +# An explicit override supports source-tree development without copying binaries +# into the package. Installed wheels load only their bundled native library. +library = ( + Path(os.environ["PHEON_LIBRARY"]).expanduser() + if "PHEON_LIBRARY" in os.environ + else Path(__file__).with_name(filename) +) +try: + lib = ffi.dlopen(str(library.resolve())) +except OSError as error: + raise ImportError( + f"Could not load Pheon library {library}: {error}. " + "Install a Pheon wheel or set PHEON_LIBRARY to a Cargo-built shared library." + ) from error + +ABI_VERSION = 1 +ARROW_SCHEMA_VERSION = 1 +try: + native_abi = lib.pheon_abi_version() + native_schema = lib.pheon_arrow_schema_version() +except AttributeError as error: + raise ImportError( + "The selected library does not expose the Pheon version API" + ) from error +if (native_abi, native_schema) != (ABI_VERSION, ARROW_SCHEMA_VERSION): + raise ImportError( + f"Incompatible Pheon library: C ABI {native_abi}, Arrow schema {native_schema}; " + f"expected C ABI {ABI_VERSION}, Arrow schema {ARROW_SCHEMA_VERSION}" + ) + + +class PheonError(ValueError): + """A native GDSII parse or selection error.""" diff --git a/pheon/arrow.py b/pheon/arrow.py new file mode 100644 index 0000000..af3a39a --- /dev/null +++ b/pheon/arrow.py @@ -0,0 +1,101 @@ +"""GDSII readers returning a one-row Arrow StructArray for the library. + +The returned arrays own their buffers independently of the input. +""" + +from __future__ import annotations + +from collections.abc import Sequence +import gzip +import os +from typing import Any, Callable + +import numpy +import pyarrow + +from ._native import PheonError, ffi, lib + + +def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray: + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + if call(array, schema) != 0: + error = lib.last_error_message() + message = ( + ffi.string(error).decode("utf-8", errors="replace") + if error != ffi.NULL + else "GDSII read failed" + ) + raise PheonError(message) + # PyArrow consumes the structures, setting their release callbacks to + # null. The finally block also covers failure during import. + return pyarrow.Array._import_from_c( + int(ffi.cast("uintptr_t", array)), int(ffi.cast("uintptr_t", schema)) + ) + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + +def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: + """Read a GDSII file, recognizing gzip by its magic bytes.""" + with open(os.path.expanduser(path), "rb") as stream: + compressed = stream.read(2) == b"\x1f\x8b" + stream.seek(0) + if compressed: + with gzip.GzipFile(fileobj=stream) as source: + return read_bytes(source.read()) + return read_bytes(stream.read()) + + +def _read_buffer( + buffer: Any, call: Callable[[Any, int, Any, Any], int] +) -> pyarrow.StructArray: + with memoryview(buffer) as view: + if not view.c_contiguous: + raise ValueError("GDSII input must be a contiguous buffer") + with ffi.from_buffer("const uint8_t[]", view) as data: + return _export(lambda array, schema: call(data, view.nbytes, array, schema)) + + +def read_bytes(buffer: Any) -> pyarrow.StructArray: + """Read a complete GDSII library from a contiguous borrowed byte buffer.""" + return _read_buffer(buffer, lib.read_bytes) + + +def scan_bytes(buffer: Any) -> pyarrow.StructArray: + """Scan metadata, cell byte ranges and references without decoding geometry.""" + return _read_buffer(buffer, lib.scan_bytes) + + +def read_cells_bytes( + buffer: Any, ranges: Sequence[Sequence[int]] | numpy.ndarray +) -> pyarrow.StructArray: + """Read complete cells in the requested order from half-open byte ranges. + + ``ranges`` has shape (N, 2) and unsigned 64-bit integer offsets. Signed + integer inputs are accepted after checking that every offset is nonnegative. + An empty sequence requests metadata with no cells. + """ + values = numpy.asarray(ranges) + if values.shape == (0,): + values = numpy.empty((0, 2), dtype=numpy.uint64) + if values.ndim != 2 or values.shape[1] != 2: + raise ValueError("Cell ranges must have shape (N, 2)") + if values.dtype.kind not in "iu": + raise TypeError("Cell offsets must be integers") + if values.dtype.kind == "i" and numpy.any(values < 0): + raise ValueError("Cell offsets must be nonnegative") + values = numpy.require( + values, dtype=numpy.uint64, requirements=("C_CONTIGUOUS", "ALIGNED") + ) + with ffi.from_buffer("const uint64_t[]", values) as offsets: + return _read_buffer( + buffer, + lambda data, size, array, schema: lib.read_cells_bytes( + data, size, offsets, len(values), array, schema + ), + ) diff --git a/pheon/basic.py b/pheon/basic.py new file mode 100644 index 0000000..51100aa --- /dev/null +++ b/pheon/basic.py @@ -0,0 +1,70 @@ +from collections.abc import Sequence +from typing import Any + +import numpy +from numpy.typing import NDArray + +from ._native import lib, ffi + + +CONV_TABLE_i16 = { + numpy.float64: lib.f64_to_i16, + numpy.float32: lib.f32_to_i16, + numpy.int64: lib.i64_to_i16, + numpy.int32: lib.i32_to_i16, + numpy.int16: lib.i16_to_i16, + numpy.uint64: lib.u64_to_i16, + numpy.uint32: lib.u32_to_i16, + numpy.uint16: lib.u16_to_i16, +} + +CONV_TABLE_i32 = { + numpy.float64: lib.f64_to_i32, + numpy.float32: lib.f32_to_i32, + numpy.int64: lib.i64_to_i32, + numpy.int32: lib.i32_to_i32, + numpy.uint64: lib.u64_to_i32, + numpy.uint32: lib.u32_to_i32, +} + + +def _pack_int( + data: NDArray[numpy.integer] | Sequence[int] | int, + width: int, + converters: dict[type[numpy.generic], Any], +) -> bytes: + arr = numpy.atleast_1d(data) + target_dtype = numpy.dtype(f">i{width}") + + convert = converters.get(arr.dtype.type) if arr.dtype.isnative else None + if convert is not None: + arr = numpy.require( + arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") + ) + if arr is data: + arr = numpy.array(arr, copy=True) + + with ffi.from_buffer( + ffi.typeof(convert).args[0], arr, require_writable=True + ) as buf: + result = convert(buf, arr.size) + if result != 0: + raise ValueError(f"Invalid value for conversion to Int{width}: {result}") + + return arr.reshape(-1).view(target_dtype)[:: arr.itemsize // width].tobytes() + + if arr.dtype == target_dtype: + return arr.tobytes() + + limit = 1 << (width * 8 - 1) + if (arr > limit - 1).any() or (arr < -limit).any(): + raise ValueError(f"int{width} data out of range: {arr}") + return arr.astype(target_dtype).tobytes() + + +def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 2, CONV_TABLE_i16) + + +def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 4, CONV_TABLE_i32) diff --git a/klamath_rs_ext/py.typed b/pheon/py.typed similarity index 100% rename from klamath_rs_ext/py.typed rename to pheon/py.typed diff --git a/pyproject.toml b/pyproject.toml index 7a14183..aaa9512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ [build-system] -requires = ["setuptools>=64", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "klamath_rs_ext" -description = "Compiled extensions for klamath GDS library" -#readme = "README.md" -#license = { file = "LICENSE.md" } +name = "pheon" +description = "Native GDSII parsing, Arrow interchange, and structural layout review" +readme = "README.md" +license = "GPL-3.0-only" +license-files = ["LICENSE.md"] authors = [ { name="Jan Petykiewicz", email="jan@mpxd.net" }, ] @@ -18,26 +19,26 @@ classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: Manufacturing", "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", ] requires-python = ">=3.11" -#include = [ -# "LICENSE.md" -# ] dynamic = ["version"] dependencies = [ "cffi", "numpy", + "pyarrow>=14", ] +[project.optional-dependencies] +arrow = [] # Compatibility with existing requirements for pheon[arrow]. + [project.urls] -Homepage = "https://mpxd.net/code/jan/klamath-rs" -Repository = "https://mpxd.net/code/jan/klamath-rs" +Homepage = "https://mpxd.net/code/jan/pheon" +Repository = "https://mpxd.net/code/jan/pheon" [tool.setuptools] -packages = ["klamath_rs_ext"] +packages = ["pheon"] [tool.setuptools.package-data] -klamath_rs_ext = ["py.typed", "_ffi.h"] +pheon = ["py.typed", "_ffi.h"] diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..292fe49 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/setup.py b/setup.py index 698a4dd..7a22f81 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ """Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" import json +import os from pathlib import Path import shutil import subprocess @@ -8,25 +9,30 @@ import tomllib from setuptools import Distribution, setup from setuptools.command.build_py import build_py -from setuptools.command.sdist import sdist from wheel.bdist_wheel import bdist_wheel ROOT = Path(__file__).resolve().parent VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] +BUILD_TARGET = os.environ.get("PHEON_BUILD_TARGET") +WINDOWS_TARGET = "x86_64-pc-windows-gnu" + +if BUILD_TARGET is not None and BUILD_TARGET != WINDOWS_TARGET: + raise ValueError(f"PHEON_BUILD_TARGET must be {WINDOWS_TARGET}, got {BUILD_TARGET!r}") -def ensure_lockfile(): - if not (ROOT / "Cargo.lock").is_file(): - subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True) +if not (ROOT / "Cargo.lock").is_file(): + raise FileNotFoundError("Source builds require the supplied Cargo.lock") class CargoBuild(build_py): def run(self): - ensure_lockfile() super().run() + command = ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"] + if BUILD_TARGET: + command.extend(["--target", BUILD_TARGET]) result = subprocess.run( - ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], + command, cwd=ROOT, capture_output=True, text=True, ) sys.stderr.write(result.stderr) @@ -40,21 +46,15 @@ class CargoBuild(build_py): if message.get("reason") == "compiler-message": sys.stderr.write(message["message"].get("rendered") or "") if (message.get("reason") == "compiler-artifact" - and message["target"]["name"] == "klamath_rs_ext" + and message["target"]["name"] == "pheon" and "cdylib" in message["target"]["crate_types"]): for filename in message["filenames"]: if Path(filename).suffix in (".so", ".dylib", ".dll"): library = Path(filename) result.check_returncode() - if library is None: - raise RuntimeError("Cargo did not produce the klamath_rs_ext shared library") - shutil.copy2(library, Path(self.build_lib) / "klamath_rs_ext" / library.name) - - -class CargoSdist(sdist): - def run(self): - ensure_lockfile() - super().run() + if library is None or (BUILD_TARGET and library.name != "pheon.dll"): + raise RuntimeError("Cargo did not produce the expected Pheon shared library") + shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) class NativeDistribution(Distribution): @@ -64,7 +64,9 @@ class NativeDistribution(Distribution): class NativeWheel(bdist_wheel): def get_tag(self): - # CFFI ABI mode needs the host platform, but no CPython extension ABI. + # CFFI ABI mode needs a platform tag and no CPython extension ABI. + if BUILD_TARGET == WINDOWS_TARGET: + return "py3", "none", "win_amd64" _, _, platform = super().get_tag() return "py3", "none", platform @@ -72,5 +74,6 @@ class NativeWheel(bdist_wheel): setup( version=VERSION, distclass=NativeDistribution, - cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, + cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel}, + options={"build": {"build_base": f"build/{BUILD_TARGET}"}} if BUILD_TARGET else {}, ) diff --git a/src/basic.rs b/src/basic.rs index b0b5057..e4de7a0 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -1,7 +1,6 @@ /// /// Functionality for parsing and writing basic data types /// -use byteorder::{ByteOrder, BigEndian}; use std::io; use std::fmt; @@ -37,22 +36,26 @@ pub fn take_bytes>(input: &[u8], count: CC) -> IResult<'_, &[u8] /* * Parse functions */ +fn take_array(input: &[u8]) -> IResult<'_, [u8; N]> { + match input.split_first_chunk::() { + Some((bytes, remaining)) => Ok((remaining, *bytes)), + None => incomplete(input, Some(N)), + } +} + pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> { - let (input, buf) = take_bytes(input, 2_usize)?; - let val = BigEndian::read_u16(buf); - Ok((input, val)) + let (input, bytes) = take_array(input)?; + Ok((input, u16::from_be_bytes(bytes))) } pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> { - let (input, buf) = take_bytes(input, 2_usize)?; - let val = BigEndian::read_i16(buf); - Ok((input, val)) + let (input, bytes) = take_array(input)?; + Ok((input, i16::from_be_bytes(bytes))) } pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> { - let (input, buf) = take_bytes(input, 4_usize)?; - let val = BigEndian::read_i32(buf); - Ok((input, val)) + let (input, bytes) = take_array(input)?; + Ok((input, i32::from_be_bytes(bytes))) } /// Convert GDS REAL8 to IEEE float64 @@ -68,20 +71,41 @@ pub fn decode_real8(int: u64) -> f64 { } pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { - let (input, buf) = take_bytes(input, 8_usize)?; - let data = BigEndian::read_u64(buf); + let (input, bytes) = take_array(input)?; + let data = u64::from_be_bytes(bytes); Ok((input, decode_real8(data))) } pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { let mut buf = [0_i16; 6]; - let mut input = input; - for bb in &mut buf { - (input, *bb) = parse_int2(input)?; + let (mut remaining, year) = parse_int2(input)?; + let Some(year) = year.checked_add(1900) else { + return fail( + input, + "Timestamp year exceeds the Int16 range after adding 1900".to_string(), + ); + }; + buf[0] = year; + for bb in &mut buf[1..] { + (remaining, *bb) = parse_int2(remaining)?; } - buf[0] += 1900; // Year is from 1900 - Ok((input, buf)) + Ok((remaining, buf)) +} + +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + String::from_utf8(bytes) + .map_err(|error| ErrType::Failed(format!("Invalid UTF-8 string: {}", error.utf8_error()))) +} + +pub(crate) fn decode_name(mut bytes: &[u8]) -> Result<&str, String> { + while let Some(unpadded) = bytes.strip_suffix(&[0]) { + bytes = unpadded; + } + if bytes.contains(&0) { + return Err("name contains an embedded NUL".to_string()); + } + std::str::from_utf8(bytes).map_err(|error| format!("Invalid UTF-8 string: {error}")) } @@ -96,11 +120,12 @@ pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec> { - let length = length as usize; let (input, data) = take_bytes(input, length)?; - let true_length = length - usize::from(data.last() == Some(&0)); - let vec = data[..true_length].to_vec(); - Ok((input, vec)) + Ok((input, unpad_ascii(data).to_vec())) +} + +pub(crate) fn unpad_ascii(data: &[u8]) -> &[u8] { + data.strip_suffix(&[0]).unwrap_or(data) } @@ -117,20 +142,21 @@ pub fn bitarray2int(bits: &[bool; 16]) -> u16 { } pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { - BigEndian::write_u16(buf, bitarray2int(bits)); + buf[..2].copy_from_slice(&bitarray2int(bits).to_be_bytes()); } pub fn pack_int2(buf: &mut [u8], int: i16) { - BigEndian::write_i16(buf, int); + buf[..2].copy_from_slice(&int.to_be_bytes()); } pub fn pack_int4(buf: &mut [u8], int: i32) { - BigEndian::write_i32(buf, int); + buf[..4].copy_from_slice(&int.to_be_bytes()); } pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { - BigEndian::write_u64(buf, encode_real8(fnum)?); + let bytes = encode_real8(fnum)?.to_be_bytes(); + buf[..8].copy_from_slice(&bytes); Ok(()) } @@ -202,7 +228,6 @@ mod tests { fn test_parse_bitarray() { use crate::basic::parse_bitarray; - //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); diff --git a/src/bin/klamath.rs b/src/bin/klamath.rs deleted file mode 100644 index 37e0a02..0000000 --- a/src/bin/klamath.rs +++ /dev/null @@ -1,375 +0,0 @@ -use klamath_rs_ext::review::report::{ReportInputs, ReportOptions, failure_report}; -use klamath_rs_ext::review::{ - Comparison, DisplayFilter, Document, Scope, dump_records, hierarchy, parse_layer, - parse_layer_name, write_layer_names, -}; -use std::{ - collections::BTreeMap, - ffi::OsString, - fs, - io::{self, Write}, - path::PathBuf, - process::ExitCode, -}; - -const HELP: &str = "Usage: - klamath dump FILE [--records] - klamath diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T]\n [--show-cell NAME] [--show-layer L/T] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] - klamath hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n klamath git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE - klamath name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... - -Object profile: ignore timestamps and element order, normalize polygon start/winding, -preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. -Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. -"; - -fn value(args: &[OsString], index: &mut usize) -> Result { - *index += 1; - args.get(*index) - .and_then(|s| s.to_str()) - .map(str::to_owned) - .ok_or_else(|| "missing UTF-8 option value".to_string()) -} - -fn name_layers(args: &[OsString]) -> Result { - if args.len() < 2 { - return Err(HELP.to_string()); - } - let (mut cell, mut carrier) = (None, None); - let mut names = BTreeMap::new(); - let mut i = 2; - while i < args.len() { - match args[i].to_str() { - Some("--cell") => cell = Some(value(args, &mut i)?), - Some("--carrier") => carrier = Some(parse_layer(&value(args, &mut i)?)?), - Some("--name") => { - let (pair, name) = parse_layer_name(&value(args, &mut i)?)?; - if names.get(&pair).is_some_and(|old| old != &name) { - return Err("conflicting --name arguments".to_string()); - } - names.insert(pair, name); - } - _ => { - return Err( - "name-layers requires --cell, --carrier and --name arguments".to_string(), - ); - } - } - i += 1; - } - if names.is_empty() { - return Err("provide at least one --name".to_string()); - } - let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; - let output = write_layer_names( - &input, - &cell.ok_or("provide --cell")?, - carrier.ok_or("provide --carrier")?, - &names, - )?; - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&args[1]) - .map_err(|e| format!("{:?}: {e}", args[1]))?; - if let Err(e) = file.write_all(&output).and_then(|_| file.sync_all()) { - drop(file); - let _ = fs::remove_file(&args[1]); - return Err(e.to_string()); - } - Ok(0) -} - -fn stdout(output: &str) -> Result<(), String> { - io::stdout() - .lock() - .write_all(output.as_bytes()) - .map_err(|e| e.to_string()) -} - -fn json_stdout(report: &serde_json::Value) -> Result<(), String> { - let mut output = io::stdout().lock(); - serde_json::to_writer_pretty(&mut output, report).map_err(|e| e.to_string())?; - output.write_all(b"\n").map_err(|e| e.to_string()) -} - -fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { - let inputs = ReportInputs { - old: Some(PathBuf::from(&paths[0])), - new: Some(PathBuf::from(&paths[1])), - }; - let comparison = || { - let read = |index, side| { - fs::read(&paths[index]) - .map_err(|e| ("io_error", format!("{:?}: {e}", paths[index]), Some(side))) - }; - let old_bytes = read(0, "old")?; - let new_bytes = read(1, "new")?; - let old = Document::parse(&old_bytes).map_err(|e| ("invalid_input", e, Some("old")))?; - let new = Document::parse(&new_bytes).map_err(|e| ("invalid_input", e, Some("new")))?; - scope - .validate_pair(&old, &new) - .map_err(|e| ("invalid_scope", e, None))?; - let old = old - .selected(scope, true) - .map_err(|e| ("invalid_scope", e, Some("old")))?; - let new = new - .selected(scope, true) - .map_err(|e| ("invalid_scope", e, Some("new")))?; - let diff = Comparison::between(&old, &new); - Ok::<_, (&str, String, Option<&str>)>(( - diff.json_report(&old, &new, &inputs, options), - diff.exit_code(), - )) - }; - let (report, status) = match comparison() { - Ok(result) => result, - Err((code, message, side)) => ( - failure_report(&inputs, scope, options, code, &message, side), - 2, - ), - }; - if let Err(error) = json_stdout(&report) { - // Once output starts, never append a second report on a write failure. - eprintln!("klamath: {error}"); - return Ok(2); - } - Ok(status) -} - -fn run(args: &[OsString]) -> Result { - if args.is_empty() || args[0] == "--help" || args[0] == "-h" { - stdout(HELP)?; - return Ok(0); - } - let command = args[0].to_str().ok_or("invalid command")?; - if command == "name-layers" { - return name_layers(&args[1..]); - } - let expected = match command { - "dump" | "hierarchy" => 1, - "diff" => 2, - "git-diff" => 7, - _ => return Err(HELP.to_string()), - }; - let compare = command == "diff" || command == "git-diff"; - let mut scope = Scope::default(); - let mut show = DisplayFilter::default(); - let mut raw = false; - let mut format = if compare { - "none".to_string() - } else { - "ascii".to_string() - }; - let mut max_changes = 100; - let mut explicit_max = None; - let mut output_format = "text".to_string(); - let mut depth = 8; - let mut explicit_depth = false; - let mut paths = Vec::new(); - let mut literal = false; - let mut i = 1; - while i < args.len() { - if literal { - paths.push(args[i].clone()); - i += 1; - continue; - } - match args[i].to_str() { - Some("--") => literal = true, - Some("--help" | "-h") => { - stdout(HELP)?; - return Ok(0); - } - Some("--records") if command == "dump" => raw = true, - Some("--top") => scope.top = Some(value(args, &mut i)?), - Some("--cell") => scope.cell = Some(value(args, &mut i)?), - Some("--layer") => scope.layer = Some(parse_layer(&value(args, &mut i)?)?), - Some("--show-cell") if compare => show.cell = Some(value(args, &mut i)?), - Some("--show-layer") if compare => { - show.layer = Some(parse_layer(&value(args, &mut i)?)?) - } - Some("--hierarchy") if compare => format = value(args, &mut i)?, - Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, - Some("--format") if command == "diff" => output_format = value(args, &mut i)?, - Some("--max-changes") if compare => { - max_changes = value(args, &mut i)? - .parse() - .map_err(|_| "invalid change limit")?; - explicit_max = Some(max_changes); - } - Some("--depth") if command != "dump" => { - depth = value(args, &mut i)?.parse().map_err(|_| "invalid depth")?; - explicit_depth = true; - } - Some(s) if s.starts_with('-') => return Err(format!("unknown option {s:?}")), - _ => paths.push(args[i].clone()), - } - i += 1; - } - if paths.len() != expected && !(command == "git-diff" && paths.len() == 9) { - return Err(HELP.to_string()); - } - if !matches!(output_format.as_str(), "text" | "json") { - return Err("diff format must be text or json".to_string()); - } - if output_format == "json" { - if !matches!(format.as_str(), "none" | "graph") || explicit_depth { - return Err( - "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), - ); - } - return json_diff( - &paths, - &scope, - &ReportOptions { - filter: show, - max_objects: explicit_max, - hierarchy: format == "graph", - }, - ); - } - if !matches!(format.as_str(), "ascii" | "mermaid" | "none") - || (command == "hierarchy" && format == "none") - { - return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); - } - let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); - if raw { - if scope != Scope::default() { - return Err("record dumps do not accept object scope selectors".to_string()); - } - stdout(&dump_records(&read(&paths[0])?)?)?; - return Ok(0); - } - if !compare { - let bytes = read(&paths[0])?; - let doc = Document::parse(&bytes)?.selected(&scope, false)?; - let output = if command == "dump" { - doc.canonical_dump() - } else { - hierarchy(&doc, &doc, None, format == "mermaid", depth) - }; - stdout(&output)?; - if command == "hierarchy" { - for issue in &doc.issues { - eprintln!("INCOMPLETE: {issue}"); - } - } - return Ok(if doc.issues.is_empty() { 0 } else { 2 }); - } - let git = command == "git-diff"; - let old_missing = git && (paths[3] == "000000" || paths[3] == "."); - let new_missing = git && (paths[6] == "000000" || paths[6] == "."); - if old_missing && new_missing { - return Err("both Git revisions are absent".to_string()); - } - let old_bytes = if old_missing { - None - } else { - Some(read(&paths[if git { 1 } else { 0 }])?) - }; - let new_bytes = if new_missing { - None - } else { - Some(read(&paths[if git { 4 } else { 1 }])?) - }; - let old = old_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; - let new = new_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; - let old = old.unwrap_or_else(|| new.as_ref().unwrap().empty_like()); - let new = new.unwrap_or_else(|| old.empty_like()); - scope.validate_pair(&old, &new)?; - let old = old.selected(&scope, true)?; - let new = new.selected(&scope, true)?; - let diff = Comparison::between(&old, &new); - let mut output = String::new(); - if git { - if paths.len() == 9 { - output.push_str(&format!("layout {:?} -> {:?}\n", paths[0], paths[7])); - } else { - output.push_str(&format!("layout {:?}\n", paths[0])); - } - if old_missing { - output.push_str("layout added\n"); - } else if new_missing { - output.push_str("layout deleted\n"); - } - if paths[3] != paths[6] { - output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); - } - } - output.push_str(&diff.text_filtered(max_changes, &show)); - if format != "none" { - if format == "mermaid" { - output.push_str("```mermaid\n"); - } - output.push_str(&hierarchy( - &old, - &new, - Some(&diff), - format == "mermaid", - depth, - )); - if format == "mermaid" { - output.push_str("```\n"); - } - } - stdout(&output)?; - // Git's external driver expects successful rendering for either equality - // or differences. Actual comparison/input failures remain nonzero. - Ok(if git && diff.exit_code() == 1 { - 0 - } else { - diff.exit_code() - }) -} - -fn wants_json(args: &[OsString]) -> bool { - if args.first().is_none_or(|arg| arg != "diff") { - return false; - } - let mut format = None; - let mut i = 1; - while i < args.len() { - match args[i].to_str() { - Some("--") => break, - Some("--format") => { - format = args.get(i + 1).and_then(|v| v.to_str()); - i += 1; - } - Some( - "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" - | "--max-changes" | "--depth", - ) => i += 1, - _ => (), - } - i += 1; - } - format == Some("json") -} - -fn main() -> ExitCode { - let args: Vec<_> = std::env::args_os().skip(1).collect(); - let json = wants_json(&args); - match run(&args) { - Ok(status) => ExitCode::from(status), - Err(message) => { - if json { - let report = failure_report( - &ReportInputs::default(), - &Scope::default(), - &ReportOptions::default(), - "command_error", - &message, - None, - ); - if let Err(error) = json_stdout(&report) { - eprintln!("klamath: {error}"); - } - } else { - eprintln!("klamath: {message}"); - } - ExitCode::from(2) - } - } -} diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs new file mode 100644 index 0000000..72db205 --- /dev/null +++ b/src/bin/pheon.rs @@ -0,0 +1,568 @@ +#[path = "pheon/color.rs"] +mod color; + +use flate2::read::MultiGzDecoder; +use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; +use pheon::review::{ + Comparison, Diagnostic, Document, Layer, Outcome, RevisionPair, Side, TextComparison, + TextReportOptions, TextStyle, + names::{parse_layer, parse_layer_name, resolve_layer_name, write_layer_names}, + selection::{DisplayFilter, Scope}, + write_record_dump, +}; +use std::{ + collections::BTreeMap, + ffi::{OsStr, OsString}, + fmt::Write as _, + fs, + io::{self, Read, Write}, + path::PathBuf, + process::ExitCode, + thread, +}; +use termcolor::ColorChoice; + +const HELP: &str = "Usage: + pheon dump FILE [--records] [--color auto|always|never] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T|NAME]\n [--show-cell NAME] [--show-layer L/T|NAME] [--hierarchy ascii|mermaid|none|graph]\n [--details] [--max-changes N] [--depth N] [--format text|json] + [--color auto|always|never] + pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE + pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... + +Structural comparison: ignore timestamps and element order, normalize polygon start/winding, +preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Gzip-compressed GDSII input is recognized by its header. +Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. +"; + +fn read_input(path: &OsStr) -> Result, Diagnostic> { + let io_error = + |error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None); + let mut file = fs::File::open(path).map_err(io_error)?; + let mut bytes = Vec::new(); + (&mut file) + .take(2) + .read_to_end(&mut bytes) + .map_err(io_error)?; + if bytes == b"\x1f\x8b" { + let mut decoded = Vec::new(); + MultiGzDecoder::new(bytes.as_slice().chain(file)) + .read_to_end(&mut decoded) + .map_err(|error| { + Diagnostic::new( + "invalid_input", + format!("{path:?}: gzip decompression failed: {error}"), + None, + ) + })?; + return Ok(decoded); + } + file.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + +fn value(args: &[OsString], index: &mut usize) -> Result { + *index += 1; + args.get(*index) + .and_then(|s| s.to_str()) + .map(str::to_owned) + .ok_or_else(|| "missing UTF-8 option value".to_string()) +} + +fn name_layers(args: &[OsString]) -> Result { + if args.len() < 2 { + return Err(HELP.to_string()); + } + let (mut cell, mut carrier) = (None, None); + let mut names = BTreeMap::new(); + let mut i = 2; + while i < args.len() { + match args[i].to_str() { + Some("--cell") => cell = Some(value(args, &mut i)?), + Some("--carrier") => carrier = Some(parse_layer(&value(args, &mut i)?)?), + Some("--name") => { + let (pair, name) = parse_layer_name(&value(args, &mut i)?)?; + if names.get(&pair).is_some_and(|old| old != &name) { + return Err("conflicting --name arguments".to_string()); + } + names.insert(pair, name); + } + _ => { + return Err( + "name-layers requires --cell, --carrier and --name arguments".to_string(), + ); + } + } + i += 1; + } + if names.is_empty() { + return Err("provide at least one --name".to_string()); + } + let input = read_input(&args[0]).map_err(|error| error.message)?; + let output = write_layer_names( + &input, + &cell.ok_or("provide --cell")?, + carrier.ok_or("provide --carrier")?, + &names, + )?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[1]) + .map_err(|e| format!("{:?}: {e}", args[1]))?; + if let Err(e) = file.write_all(&output).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(&args[1]); + return Err(e.to_string()); + } + Ok(0) +} + +fn stdout(output: &str) -> Result<(), String> { + io::stdout() + .lock() + .write_all(output.as_bytes()) + .map_err(|e| e.to_string()) +} + +fn json_stdout( + write: impl FnOnce(&mut io::BufWriter>) -> serde_json::Result<()>, +) -> Result<(), String> { + let mut output = io::BufWriter::new(io::stdout().lock()); + write(&mut output).map_err(|e| e.to_string())?; + output + .write_all(b"\n") + .and_then(|()| output.flush()) + .map_err(|e| e.to_string()) +} + +fn parse_revision(bytes: Option<&[u8]>, side: Side) -> Result>, Diagnostic> { + bytes + .map(|bytes| { + let mut document = Document::parse(bytes) + .map_err(|message| Diagnostic::new("invalid_input", message, Some(side)))?; + // Comparison uses the parsed cells and their source offsets. + document.records = Vec::new(); + Ok(document) + }) + .transpose() +} + +#[derive(Default)] +struct Selectors { + scope: Scope, + filter: DisplayFilter, + layer_name: Option, + show_layer_name: Option, +} + +fn layer_argument(value: String) -> (Option, Option) { + match parse_layer(&value) { + Ok(pair) => (Some(pair), None), + Err(_) => (None, Some(value)), + } +} + +impl Selectors { + fn resolve_layers( + &mut self, + resolve: impl Fn(&str) -> Result, + ) -> Result<(), Diagnostic> { + for (flag, code, name, pair) in [ + ( + "--layer", + "invalid_scope", + &self.layer_name, + &mut self.scope.layer, + ), + ( + "--show-layer", + "command_error", + &self.show_layer_name, + &mut self.filter.layer, + ), + ] { + if let Some(name) = name { + *pair = Some( + resolve(name) + .map_err(|error| Diagnostic::new(code, format!("{flag} {error}"), None))?, + ); + } + } + Ok(()) + } +} + +fn prepare_comparison( + old_path: Option<&OsStr>, + new_path: Option<&OsStr>, + selectors: &mut Selectors, + compare: impl FnOnce(&Document<'_>, &Document<'_>) -> T, + compare_pair: impl FnOnce(RevisionPair<'_>) -> T, +) -> Result { + let read = |path: Option<&OsStr>, side| { + path.map(|path| { + read_input(path).map_err(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic + }) + }) + .transpose() + }; + let old_bytes = read(old_path, Side::Old)?; + let new_bytes = read(new_path, Side::New)?; + const PARALLEL_PARSE_MIN_BYTES: usize = 2 * 1024 * 1024; + let large_pair = old_bytes + .as_deref() + .zip(new_bytes.as_deref()) + .filter(|(old, new)| old.len().min(new.len()) >= PARALLEL_PARSE_MIN_BYTES); + if selectors.scope == Scope::default() + && selectors.layer_name.is_none() + && let Some((old, new)) = large_pair + { + let pair = RevisionPair::parse(old, new)?; + selectors.resolve_layers(|name| pair.resolve_layer_name(name))?; + return Ok(compare_pair(pair)); + } + let parse_old = || parse_revision(old_bytes.as_deref(), Side::Old); + let parse_new = || parse_revision(new_bytes.as_deref(), Side::New); + let parallel = if large_pair.is_some() + && thread::available_parallelism().is_ok_and(|count| count.get() > 1) + { + thread::scope(|scope| { + let worker = thread::Builder::new().spawn_scoped(scope, parse_old).ok()?; + let new = parse_new(); + Some((worker.join().unwrap(), new)) + }) + } else { + None + }; + // Report an old-side parse error first if both inputs are malformed. + let (old, new) = match parallel { + Some((old, new)) => (old?, new?), + None => (parse_old()?, parse_new()?), + }; + let (old, new) = match (old, new) { + (Some(old), Some(new)) => (old, new), + (None, Some(new)) => (new.empty_like(), new), + (Some(old), None) => { + let new = old.empty_like(); + (old, new) + } + (None, None) => { + return Err(Diagnostic::new( + "invalid_input", + "both Git revisions are absent", + None, + )); + } + }; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&old.layer_names, &new.layer_names]))?; + let scope = &selectors.scope; + if scope == &Scope::default() { + return Ok(compare(&old, &new)); + } + scope + .validate_pair(&old, &new) + .map_err(|message| Diagnostic::new("invalid_scope", message, None))?; + let old = old + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::Old)))?; + let new = new + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::New)))?; + Ok(compare(&old, &new)) +} + +fn json_diff( + paths: &[OsString], + selectors: &mut Selectors, + max_objects: Option, + hierarchy: bool, +) -> Result { + let inputs = ReportInputs { + old: Some(PathBuf::from(&paths[0])), + new: Some(PathBuf::from(&paths[1])), + }; + let comparison = prepare_comparison( + Some(&paths[0]), + Some(&paths[1]), + selectors, + Comparison::between, + |pair| pair.compare(), + ); + let options = ReportOptions { + filter: selectors.filter.clone(), + max_objects, + hierarchy, + }; + let (result, outcome) = match comparison { + Ok(diff) => ( + json_stdout(|output| diff.write_json_report(output, &inputs, &options)), + diff.outcome(), + ), + Err(error) => { + let report = failure_report(&inputs, &selectors.scope, &options, &error); + ( + json_stdout(|output| serde_json::to_writer_pretty(output, &report)), + Outcome::Failed, + ) + } + }; + if let Err(error) = result { + // Once output starts, never append a second report on a write failure. + eprintln!("pheon: {error}"); + return Ok(2); + } + Ok(outcome.exit_code()) +} + +fn run(args: &[OsString]) -> Result { + if args.len() == 1 && matches!(args[0].to_str(), Some("--version" | "-V")) { + stdout(&format!("pheon {}\n", env!("CARGO_PKG_VERSION")))?; + return Ok(0); + } + if args.is_empty() || args[0] == "--help" || args[0] == "-h" { + stdout(HELP)?; + return Ok(0); + } + let command = args[0].to_str().ok_or("invalid command")?; + if command == "name-layers" { + return name_layers(&args[1..]); + } + let expected = match command { + "dump" | "hierarchy" => 1, + "diff" => 2, + "git-diff" => 7, + _ => return Err(HELP.to_string()), + }; + let compare = command == "diff" || command == "git-diff"; + let color_supported = compare || command == "dump"; + let mut color_choice = ColorChoice::Auto; + let mut selectors = Selectors::default(); + let mut raw = false; + let mut format = if compare { + "none".to_string() + } else { + "ascii".to_string() + }; + let mut max_changes = 100; + let mut details = false; + let mut explicit_max = None; + let mut output_format = "text".to_string(); + let mut depth = 8; + let mut explicit_depth = false; + let mut paths = Vec::new(); + let mut literal = false; + let mut i = 1; + while i < args.len() { + if literal { + paths.push(args[i].clone()); + i += 1; + continue; + } + match args[i].to_str() { + Some("--") => literal = true, + Some("--help" | "-h") => { + stdout(HELP)?; + return Ok(0); + } + Some("--records") if command == "dump" => raw = true, + Some("--color") if color_supported => { + color_choice = color::parse_choice(&value(args, &mut i)?)?; + } + Some(s) if color_supported && s.starts_with("--color=") => { + color_choice = color::parse_choice(&s["--color=".len()..])?; + } + Some("--top") => selectors.scope.top = Some(value(args, &mut i)?), + Some("--cell") => selectors.scope.cell = Some(value(args, &mut i)?), + Some("--layer") => { + (selectors.scope.layer, selectors.layer_name) = + layer_argument(value(args, &mut i)?); + } + Some("--show-cell") if compare => selectors.filter.cell = Some(value(args, &mut i)?), + Some("--show-layer") if compare => { + (selectors.filter.layer, selectors.show_layer_name) = + layer_argument(value(args, &mut i)?); + } + Some("--details") if compare => details = true, + Some("--hierarchy") if compare => format = value(args, &mut i)?, + Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, + Some("--format") if command == "diff" => output_format = value(args, &mut i)?, + Some("--max-changes") if compare => { + max_changes = value(args, &mut i)? + .parse() + .map_err(|_| "invalid change limit")?; + explicit_max = Some(max_changes); + } + Some("--depth") if command != "dump" => { + depth = value(args, &mut i)?.parse().map_err(|_| "invalid depth")?; + explicit_depth = true; + } + Some(s) if s.starts_with('-') => return Err(format!("unknown option {s:?}")), + _ => paths.push(args[i].clone()), + } + i += 1; + } + if paths.len() != expected && !(command == "git-diff" && paths.len() == 9) { + return Err(HELP.to_string()); + } + if !matches!(output_format.as_str(), "text" | "json") { + return Err("diff format must be text or json".to_string()); + } + if output_format == "json" { + if !matches!(format.as_str(), "none" | "graph") || explicit_depth { + return Err( + "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), + ); + } + return json_diff(&paths, &mut selectors, explicit_max, format == "graph"); + } + if !matches!(format.as_str(), "ascii" | "mermaid" | "none") + || (command == "hierarchy" && format == "none") + { + return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); + } + let read = |path: &OsString| read_input(path).map_err(|error| error.message); + let printer = color::Printer::new(color_choice); + if raw { + if selectors.scope != Scope::default() || selectors.layer_name.is_some() { + return Err("record dumps do not accept object scope selectors".to_string()); + } + let mut report = printer.report(); + write_record_dump(&mut report, &read(&paths[0])?)?; + printer.print(&report).map_err(|error| error.to_string())?; + return Ok(0); + } + if !compare { + let bytes = read(&paths[0])?; + let doc = Document::parse(&bytes)?; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&doc.layer_names])) + .map_err(|error| error.message)?; + let doc = doc.selected(&selectors.scope, false)?; + if command == "dump" { + let mut report = printer.report(); + doc.write_canonical_dump(&mut report); + printer.print(&report).map_err(|error| error.to_string())?; + } else { + stdout(&doc.hierarchy(format == "mermaid", depth))?; + for diagnostic in &doc.diagnostics { + eprintln!("INCOMPLETE: {}", diagnostic.message); + } + } + return Ok(if doc.diagnostics.is_empty() { 0 } else { 2 }); + } + let git = command == "git-diff"; + let old_missing = git && (paths[3] == "000000" || paths[3] == "."); + let new_missing = git && (paths[6] == "000000" || paths[6] == "."); + let diff = prepare_comparison( + (!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()), + (!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()), + &mut selectors, + TextComparison::between, + |pair| pair.compare_text(), + ) + .map_err(|error| error.message)?; + let mut output = printer.report(); + if git { + let action = match (old_missing, new_missing) { + (true, false) => " added", + (false, true) => " deleted", + _ => "", + }; + if paths.len() == 9 { + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?} -> {:?}{action}", paths[0], paths[7]), + ); + } else { + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?}{action}", paths[0]), + ); + } + writeln!(output).unwrap(); + if !old_missing && !new_missing && paths[3] != paths[6] { + output.write_styled( + TextStyle::Heading, + format_args!("mode {:?} -> {:?}", paths[3], paths[6]), + ); + writeln!(output).unwrap(); + } + } + let show_library_presence = !old_missing && !new_missing; + diff.write_text_report( + &mut output, + max_changes, + &selectors.filter, + show_library_presence, + TextReportOptions { + details, + ascii_depth: (format == "ascii").then_some(depth), + }, + ); + if format == "mermaid" { + output.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + writeln!(output, "\n```mermaid\n{}```", diff.hierarchy(true, depth)).unwrap(); + } + printer.print(&output).map_err(|error| error.to_string())?; + // Git's external driver expects successful rendering for either equality + // or differences. Actual comparison/input failures remain nonzero. + Ok(if git && diff.outcome().exit_code() == 1 { + 0 + } else { + diff.outcome().exit_code() + }) +} + +fn wants_json(args: &[OsString]) -> bool { + if args.first().is_none_or(|arg| arg != "diff") { + return false; + } + let mut format = None; + let mut i = 1; + while i < args.len() { + match args[i].to_str() { + Some("--") => break, + Some("--format") => { + format = args.get(i + 1).and_then(|v| v.to_str()); + i += 1; + } + Some( + "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" + | "--max-changes" | "--depth" | "--color", + ) => i += 1, + _ => (), + } + i += 1; + } + format == Some("json") +} + +fn main() -> ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let json = wants_json(&args); + match run(&args) { + Ok(status) => ExitCode::from(status), + Err(message) => { + if json { + let report = failure_report( + &ReportInputs::default(), + &Scope::default(), + &ReportOptions::default(), + &Diagnostic::new("command_error", message, None), + ); + if let Err(error) = + json_stdout(|output| serde_json::to_writer_pretty(output, &report)) + { + eprintln!("pheon: {error}"); + } + } else { + eprintln!("pheon: {message}"); + } + ExitCode::from(2) + } + } +} diff --git a/src/bin/pheon/color.rs b/src/bin/pheon/color.rs new file mode 100644 index 0000000..82fb933 --- /dev/null +++ b/src/bin/pheon/color.rs @@ -0,0 +1,57 @@ +use pheon::review::{TextReport, TextStyle}; +use std::io::{self, IsTerminal, Write}; +use termcolor::{BufferedStandardStream, Color, ColorChoice, ColorSpec, WriteColor}; + +pub(super) fn parse_choice(value: &str) -> Result { + match value { + "auto" => Ok(ColorChoice::Auto), + "always" => Ok(ColorChoice::Always), + "never" => Ok(ColorChoice::Never), + _ => Err("color must be auto, always, or never".to_string()), + } +} + +pub(super) struct Printer(Option); + +impl Printer { + pub fn new(choice: ColorChoice) -> Self { + if choice == ColorChoice::Never + || (choice == ColorChoice::Auto && !io::stdout().is_terminal()) + { + return Self(None); + } + let stream = BufferedStandardStream::stdout(choice); + Self(stream.supports_color().then_some(stream)) + } + + pub fn report(&self) -> TextReport { + TextReport::new(self.0.is_some()) + } + + pub fn print(self, report: &TextReport) -> io::Result<()> { + let text = report.as_str().as_bytes(); + let Some(mut out) = self.0 else { + return io::stdout().lock().write_all(text); + }; + let mut end = 0; + for (range, style) in report.styles() { + out.write_all(&text[end..range.start])?; + let (color, bold) = match style { + TextStyle::Heading => (Color::Cyan, true), + TextStyle::Layer => (Color::Cyan, false), + TextStyle::Added => (Color::Green, false), + TextStyle::Removed => (Color::Red, false), + TextStyle::Changed => (Color::Yellow, false), + TextStyle::Success => (Color::Green, true), + TextStyle::Warning => (Color::Yellow, true), + TextStyle::Error => (Color::Red, true), + }; + out.set_color(ColorSpec::new().set_fg(Some(color)).set_bold(bold))?; + let written = out.write_all(&text[range.clone()]); + written.and(out.reset())?; + end = range.end; + } + out.write_all(&text[end..])?; + out.flush() + } +} diff --git a/src/elements.rs b/src/elements.rs index dbf58e7..cfee79c 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -4,29 +4,27 @@ /// use crate::records::{ - //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, - LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, ArrayPlacement, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, ENDEL, BGNEXTN, ENDEXTN, SNAME, }; use crate::records; -use crate::record::{RecordHeader, Record}; -use crate::basic::{IResult, fail, take_bytes}; //OResult +use crate::record::{Record, RecordFrame}; +use crate::basic::{ErrType, IResult, decode_name, decode_utf8, fail}; use std::string::String; use std::collections::HashMap; -//use std::io::Write; -use arrow::array::{ +use arrow_array::builder::{ StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, Int32Builder, Int16Builder, UInt64Builder, UInt32Builder, UInt8Builder, Float64Builder, }; -pub type DListBuilder = ListBuilder>; -pub type FListBuilder = FixedSizeListBuilder>; +pub(crate) type DListBuilder = ListBuilder>; +pub(crate) type FListBuilder = FixedSizeListBuilder>; struct PropertyRecord { key: i16, @@ -72,15 +70,15 @@ struct ARefRecord { } #[derive(Clone)] -pub struct ScanRefRecord { - pub target_id: u32, - pub invert_y: bool, - pub scale: f64, - pub angle_rad: f64, - pub xy: u64, - pub xy0: u64, - pub xy1: u64, - pub counts: u32, +pub(crate) struct ScanRefRecord { + pub(crate) target_id: u32, + pub(crate) invert_y: bool, + pub(crate) scale: f64, + pub(crate) angle_rad: f64, + pub(crate) xy: u64, + pub(crate) xy0: u64, + pub(crate) xy1: u64, + pub(crate) counts: u32, } enum RefRecord { @@ -117,7 +115,7 @@ fn insert_layer( } -pub fn read_elements<'a>( +pub(crate) fn read_elements<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, names: &mut HashMap, @@ -127,11 +125,11 @@ pub fn read_elements<'a>( let mut boundaries = Vec::::new(); let mut srefs = Vec::::new(); let mut arefs = Vec::::new(); - let (_, mut header) = RecordHeader::read(input)?; // don't consume tag - while header.tag != records::RTAG_ENDSTR { - (input, _) = match header.tag { + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { records::RTAG_SREF | records::RTAG_AREF => { - let (next_input, ref_record) = read_ref(input, header.tag, names)?; + let (next_input, ref_record) = read_ref(input, record.tag, names)?; match ref_record { RefRecord::SRef(sref) => srefs.push(sref), RefRecord::ARef(aref) => arefs.push(aref), @@ -144,18 +142,17 @@ pub fn read_elements<'a>( (next_input, ()) }, records::RTAG_PATH => read_path(input, cell_builder, layers)?, - records::RTAG_NODE => read_boxnode(input, cell_builder, header.tag, layers)?, - records::RTAG_BOX => read_boxnode(input, cell_builder, header.tag, layers)?, - records::RTAG_TEXT => read_text(input, cell_builder, layers)?, - _ => { - // don't care, skip - let result = take_bytes(input, header.data_size + 4)?; - (result.0, ()) + records::RTAG_NODE | records::RTAG_BOX => { + read_boxnode(input, cell_builder, record.tag, layers)? } + records::RTAG_TEXT => read_text(input, cell_builder, layers)?, + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; + } + _ => (remaining, ()), }; - (_, header) = RecordHeader::read(input)?; // don't consume tag - } - (input, _) = take_bytes(input, 4_usize)?; // consume endstr tag + }; let mut sref_plain = Vec::::with_capacity(srefs.len() + arefs.len() / 4); let mut sref_props = Vec::::with_capacity(srefs.len() / 16 + arefs.len() / 64); @@ -222,7 +219,7 @@ pub fn read_elements<'a>( let mut next_offset: u32 = 0; for boundary in group { - if let Some(rect) = boundary_to_rect(&boundary.vertices) { + if let Some(rect) = XY::rectangle(boundary.vertices.as_chunks::<2>().0) { rect_batch.rects.extend_from_slice(&rect); } else { boundary_batch.vertex_offsets.push(next_offset); @@ -293,17 +290,17 @@ pub fn read_elements<'a>( } -pub fn scan_elements<'a>( +pub(crate) fn scan_elements<'a>( input: &'a [u8], names: &mut HashMap, ) -> IResult<'a, Vec> { let mut input = input; let mut refs = Vec::::new(); - let (_, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - (input, _) = match header.tag { + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { records::RTAG_SREF | records::RTAG_AREF => { - let (next_input, ref_record) = read_ref(input, header.tag, names)?; + let (next_input, ref_record) = read_ref(input, record.tag, names)?; refs.push(scan_ref_from_record(ref_record)); (next_input, ()) }, @@ -312,15 +309,14 @@ pub fn scan_elements<'a>( | records::RTAG_NODE | records::RTAG_BOX | records::RTAG_TEXT => skip_element(input)?, - _ => { - let result = take_bytes(input, header.data_size + 4)?; - (result.0, ()) + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; } + _ => (remaining, ()), }; - (_, header) = RecordHeader::read(input)?; - } + }; refs.sort_by_key(|scan_ref| scan_ref.target_id); - (input, _) = take_bytes(input, 4_usize)?; Ok((input, refs)) } @@ -375,12 +371,12 @@ fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { let mut input = input; loop { - let (next_input, header) = RecordHeader::read(input)?; - let (next_input, _) = take_bytes(next_input, header.data_size)?; - input = next_input; - if header.tag == ENDEL::tag() { - return Ok((input, ())); + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == ENDEL::tag() { + record.decode::()?; + return Ok((remaining, ())); } + input = remaining; } } @@ -412,19 +408,24 @@ fn append_aref( } +fn append_property(props_builder: &mut DListBuilder, property: PropertyRecord) { + let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); + let key_builder = prop_builder.field_builder::(0).unwrap(); + key_builder.append_value(property.key); + + let val_builder = prop_builder.field_builder::(1).unwrap(); + val_builder.append_value(property.value); + + prop_builder.append(true); +} + + fn append_properties( props_builder: &mut DListBuilder, properties: Vec, ) { for prop in properties { - let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); - let key_builder = prop_builder.field_builder::(0).unwrap(); - key_builder.append_value(prop.key); - - let val_builder = prop_builder.field_builder::(1).unwrap(); - val_builder.append_value(prop.value); - - prop_builder.append(true); + append_property(props_builder, prop); } props_builder.append(true); } @@ -572,23 +573,13 @@ fn append_boundary_prop( vertices_builder.append(true); let props_builder = boundary_prop_builder.field_builder::(2).unwrap(); - for prop in boundary.properties { - let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); - let key_builder = prop_builder.field_builder::(0).unwrap(); - key_builder.append_value(prop.key); - - let val_builder = prop_builder.field_builder::(1).unwrap(); - val_builder.append_value(prop.value); - - prop_builder.append(true); - } - props_builder.append(true); + append_properties(props_builder, boundary.properties); boundary_prop_builder.append(true); } -pub fn read_path<'a>( +fn read_path<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -607,33 +598,16 @@ pub fn read_path<'a>( let mut bgn_ext = None; let mut end_ext = None; - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PATHTYPE => { - let _path_type; - (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; - path_type = Some(_path_type); - }, - records::RTAG_WIDTH => { - let _width; - (input, _width) = WIDTH::read_data(input, header.data_size)?; - width = Some(_width); - }, - records::RTAG_BGNEXTN => { - let _bgn_ext; - (input, _bgn_ext) = BGNEXTN::read_data(input, header.data_size)?; - bgn_ext = Some(_bgn_ext); - }, - records::RTAG_ENDEXTN => { - let _end_ext; - (input, _end_ext) = ENDEXTN::read_data(input, header.data_size)?; - end_ext = Some(_end_ext); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), + records::RTAG_BGNEXTN => bgn_ext = Some(record.decode::()?), + records::RTAG_ENDEXTN => end_ext = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; } let path_type_builder = path_builder.field_builder::(1).unwrap(); path_type_builder.append_option(path_type); @@ -645,7 +619,7 @@ pub fn read_path<'a>( width_builder.append_option(width); let xys_builder = path_builder.field_builder::(5).unwrap(); - let (input, xy_iter) = XY::read_data(input, header.data_size)?; + let xy_iter = record.decode::()?; for xy in xy_iter { let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); xy_builder.append_value(xy); @@ -659,7 +633,7 @@ pub fn read_path<'a>( Ok((input, ())) } -pub fn read_boxnode<'a>( +fn read_boxnode<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16, @@ -703,7 +677,7 @@ pub fn read_boxnode<'a>( Ok((input, ())) } -pub fn read_text<'a>( +fn read_text<'a>( input: &'a [u8], cell_builder: &mut StructBuilder, layers: &mut HashMap, @@ -724,46 +698,27 @@ pub fn read_text<'a>( let (input, dtype) = TEXTTYPE::read(input)?; insert_layer(layer, dtype, layers, text_builder, 0); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { // TODO warn if repeat tags? records::RTAG_PRESENTATION => { - let _presentation; - (input, _presentation) = PRESENTATION::read_data(input, header.data_size)?; - pres_hori = Some(_presentation[14] as u8 * 2 + _presentation[15] as u8); - pres_vert = Some(_presentation[12] as u8 * 2 + _presentation[13] as u8); - pres_font = Some(_presentation[10] as u8 * 2 + _presentation[11] as u8); - }, - records::RTAG_PATHTYPE => { - let _path_type; - (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; - path_type = Some(_path_type); - }, - records::RTAG_WIDTH => { - let _width; - (input, _width) = WIDTH::read_data(input, header.data_size)?; - width = Some(_width); - }, + let presentation = record.decode::()?; + pres_hori = Some(presentation[14] as u8 * 2 + presentation[15] as u8); + pres_vert = Some(presentation[12] as u8 * 2 + presentation[13] as u8); + pres_font = Some(presentation[10] as u8 * 2 + presentation[11] as u8); + } + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), records::RTAG_STRANS => { - let strans; - (input, strans) = STRANS::read_data(input, header.data_size)?; + let strans = record.decode::()?; invert_y = Some(strans[0]); - }, - records::RTAG_MAG => { - let _mag; - (input, _mag) = MAG::read_data(input, header.data_size)?; - mag = Some(_mag); - }, - records::RTAG_ANGLE => { - let _angle_deg; - (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; - angle_deg = Some(_angle_deg); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), + } + records::RTAG_MAG => mag = Some(record.decode::()?), + records::RTAG_ANGLE => angle_deg = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), } - (input, header) = RecordHeader::read(input)?; + (input, record) = RecordFrame::read(input)?; } let pres_hori_builder = text_builder.field_builder::(1).unwrap(); @@ -783,15 +738,16 @@ pub fn read_text<'a>( let angle_builder = text_builder.field_builder::(8).unwrap(); angle_builder.append_option(angle_deg); - let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; - let xx = xy_iter.next().unwrap(); - let yy = xy_iter.next().unwrap(); - let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let mut xy_iter = record.decode::()?; + let (Some(xx), Some(yy)) = (xy_iter.next(), xy_iter.next()) else { + return fail(record.data, "TEXT needs an XY coordinate pair".to_string()); + }; + let xy = pack_point([xx, yy]); let xy_builder = text_builder.field_builder::(9).unwrap(); xy_builder.append_value(xy); let (input, string_bytes) = STRING::read(input)?; - let string = String::from_utf8(string_bytes).unwrap(); + let string = decode_utf8(string_bytes).map_err(|error| (input, error))?; let string_builder = text_builder.field_builder::(10).unwrap(); string_builder.append_value(string); @@ -803,48 +759,11 @@ pub fn read_text<'a>( } -fn boundary_to_rect(vertices: &[i32]) -> Option<[i32; 4]> { - if vertices.len() != 8 { - return None; - } - - let xs = [vertices[0], vertices[2], vertices[4], vertices[6]]; - let ys = [vertices[1], vertices[3], vertices[5], vertices[7]]; - - let min_x = *xs.iter().min().unwrap(); - let max_x = *xs.iter().max().unwrap(); - let min_y = *ys.iter().min().unwrap(); - let max_y = *ys.iter().max().unwrap(); - if min_x >= max_x || min_y >= max_y { - return None; - } - - let mut seen_corners: u8 = 0; - for ii in 0..4 { - let jj = (ii + 1) % 4; - let same_x = xs[jj] == xs[ii]; - let same_y = ys[jj] == ys[ii]; - if same_x == same_y { - return None; - } - - let x = xs[ii]; - let y = ys[ii]; - if (x != min_x && x != max_x) || (y != min_y && y != max_y) { - return None; - } - let code = ((x == max_x) as u8) << 1 | ((y == max_y) as u8); - seen_corners |= 1 << code; - } - - if seen_corners != 0b1111 { - return None; - } - Some([min_x, min_y, max_x, max_y]) +fn pack_point([x, y]: [i32; 2]) -> u64 { + ((x as u32 as u64) << 32) | (y as u32 as u64) } - fn read_ref<'a>( input: &'a [u8], tag: u16, @@ -863,65 +782,46 @@ fn read_ref<'a>( let mut colrow = None; let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; - let struct_name = String::from_utf8(struct_name_bytes).unwrap(); + let struct_name = decode_name(&struct_name_bytes) + .map_err(|error| (input, ErrType::Failed(error)))?; let next_id = names.len(); - let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); + let id = names.entry(struct_name.to_owned()).or_insert(next_id.try_into().unwrap()); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { records::RTAG_STRANS => { - let strans; - (input, strans) = STRANS::read_data(input, header.data_size)?; + let strans = record.decode::()?; invert_y = strans[0]; - }, - records::RTAG_MAG => { - let _mag; - (input, _mag) = MAG::read_data(input, header.data_size)?; - scale = _mag; - }, + } + records::RTAG_MAG => scale = record.decode::()?, records::RTAG_ANGLE => { - let _angle_deg; - (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; - angle_rad = _angle_deg.to_radians(); - }, + angle_rad = record.decode::()?.to_radians(); + } records::RTAG_COLROW => { - if header.data_size != 4 { - return fail(input, "COLROW must contain two counts".to_string()); - } - let mut _colrow; - (input, _colrow) = COLROW::read_data(input, header.data_size)?; - colrow = Some((_colrow.next().unwrap(), _colrow.next().unwrap())); if !is_aref { - return fail(input, "Got a COLROW record inside an SREF".to_string()); + return fail(record.data, "Got a COLROW record inside an SREF".to_string()); } - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; + COLROW::check_size(record.data.len() as u16) + .map_err(|error| (record.data, ErrType::Failed(error)))?; + colrow = Some(record.data); + } + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; } - let expected_xy_size = if is_aref { 24 } else { 8 }; - if header.data_size != expected_xy_size { - return fail(input, format!("Expected {expected_xy_size} bytes of reference coordinates, got {}", header.data_size)); - } - let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; - let xx = xy_iter.next().unwrap(); - let yy = xy_iter.next().unwrap(); - let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); let (input, properties) = read_properties_vec(input)?; if is_aref { - if colrow.is_none() { - return fail(input, "AREF without COLROW before XY".to_string()) - } - let (count0, count1) = colrow.unwrap(); - if count0 <= 0 || count1 <= 0 { - return fail(input, "AREF column and row counts must be positive".to_string()); - } + let Some(colrow) = colrow else { + return fail(input, "AREF without COLROW before XY".to_string()); + }; + let placement = ArrayPlacement::decode(colrow, record.data)?; + let [columns, rows] = placement.counts; + let [origin, column_end, row_end] = placement.points; - let pitch = |end: i32, origin: i32, count: i16| { + let pitch = |end: i32, origin: i32, count: u16| { let delta = i64::from(end) - i64::from(origin); let divisor = i64::from(count); if delta % divisor != 0 { @@ -934,32 +834,35 @@ fn read_ref<'a>( "AREF pitch exceeds signed 32-bit coordinate range".to_string() ))) }; - let x0a = pitch(xy_iter.next().unwrap(), xx, count0)?; - let y0a = pitch(xy_iter.next().unwrap(), yy, count0)?; - let x1a = pitch(xy_iter.next().unwrap(), xx, count1)?; - let y1a = pitch(xy_iter.next().unwrap(), yy, count1)?; - - let xy0a = ((x0a as u32 as u64) << 32) | (y0a as u32 as u64); - let xy1a = ((x1a as u32 as u64) << 32) | (y1a as u32 as u64); - let counts = ((count0 as u16 as u32) << 16) | (count1 as u16 as u32); + let axis_pitch = |end: [i32; 2], count| -> Result<_, _> { + Ok(pack_point([ + pitch(end[0], origin[0], count)?, + pitch(end[1], origin[1], count)?, + ])) + }; + let counts = (u32::from(columns) << 16) | u32::from(rows); Ok((input, RefRecord::ARef(ARefRecord { target_id: *id, invert_y, scale, angle_rad, - xy, - xy0: xy0a, - xy1: xy1a, + xy: pack_point(origin), + xy0: axis_pitch(column_end, columns)?, + xy1: axis_pitch(row_end, rows)?, counts, properties, }))) } else { + if record.data.len() != 8 { + return fail(record.data, "SREF needs exactly one XY coordinate pair".to_string()); + } + let origin = XY::points(record.data).next().unwrap(); Ok((input, RefRecord::SRef(SRefRecord { target_id: *id, invert_y, scale, angle_rad, - xy, + xy: pack_point(origin), properties, }))) } @@ -969,545 +872,44 @@ fn read_ref<'a>( /// /// Read element properties. /// -/// Assumes PROPATTR records have unique values. +/// Preserves record order and duplicate property attributes. /// Stops reading after consuming ENDEL record. /// -/// Args: -/// stream: Stream to read from. -/// -/// Returns: -/// propattr: -> propvalue mapping -/// -pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != ENDEL::tag() { - if header.tag == PROPATTR::tag() { - let key; - let value_bytes; - (input, key) = PROPATTR::read_data(input, header.data_size)?; - (input, value_bytes) = PROPVALUE::read(input)?; - - let value = String::from_utf8(value_bytes).unwrap(); - //assert!(!properties.contains_key(&key), "Duplicate property key: {}", key); - - let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); - let key_builder = prop_builder.field_builder::(0).unwrap(); - key_builder.append_value(key); - - let val_builder = prop_builder.field_builder::(1).unwrap(); - val_builder.append_value(value); - - prop_builder.append(true); - } - (input, header) = RecordHeader::read(input)?; - } +fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { + let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; props_builder.append(true); Ok((input, ())) } -fn read_properties_vec<'a>(input: &'a [u8]) -> IResult<'a, Vec> { - let mut properties = Vec::::new(); - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != ENDEL::tag() { - if header.tag == PROPATTR::tag() { - let key; - let value_bytes; - (input, key) = PROPATTR::read_data(input, header.data_size)?; - (input, value_bytes) = PROPVALUE::read(input)?; - - properties.push(PropertyRecord { - key, - value: String::from_utf8(value_bytes).unwrap(), - }); +fn walk_properties<'a>( + input: &'a [u8], + mut on_property: impl FnMut(PropertyRecord), + ) -> IResult<'a, ()> { + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { + records::RTAG_ENDEL => { + record.decode::()?; + return Ok((remaining, ())); + } + records::RTAG_PROPATTR => { + let key = record.decode::()?; + let value_bytes; + (input, value_bytes) = PROPVALUE::read(remaining)?; + let value = decode_utf8(value_bytes).map_err(|error| (input, error))?; + on_property(PropertyRecord { key, value }); + } + _ if record.data.is_empty() => input = remaining, + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), } - (input, header) = RecordHeader::read(input)?; } +} + +fn read_properties_vec<'a>(input: &'a [u8]) -> IResult<'a, Vec> { + let mut properties = Vec::new(); + let (input, ()) = walk_properties(input, |property| properties.push(property))?; Ok((input, properties)) } - -/* - -/// -/// Write element properties. -/// -/// This is does _not_ write the ENDEL record. -/// -/// Args: -/// stream: Stream to write to. -/// -pub fn write_properties(ww: &mut W, properties: &HashMap::>) -> OResult { - let mut size = 0; - for (key, value) in properties { - size += PROPATTR::write(ww, key)?; - size += PROPVALUE::write(ww, value)?; - } - Ok(size) -} - -pub trait Element { - /// - /// Read from a stream to construct this object. - /// Consumes up to (and including) the ENDEL record. - /// - fn read(input: &[u8]) -> IResult<'_, Self> where Self: Sized; - - /// - /// Write this element to a stream. - /// Finishes with an ENDEL record. - /// - fn write(&self, ww: &mut W) -> OResult; -} - -/// -/// Datastructure representing -/// an instance of a structure (SREF / structure reference) or -/// an array of instances (AREF / array reference). -/// Type is determined by the presence of the `colrow` tuple. -/// -/// Transforms are applied to each individual instance (_not_ -/// to the instance's origin location || array vectors). -/// -#[derive(Debug, Clone)] -pub struct Reference { - /// Name of the structure being referenced. - struct_name: Vec, - /// Whether to mirror the pattern (negate y-values / flip across x-axis). Default false. - invert_y: bool, - /// Scaling factor (default 1) - mag: f64, - /// Rotation (degrees counterclockwise) - angle_deg: f64, - - /// (For SREF) Location in the parent structure corresponding to the instance's origin (0, 0). - /// (For AREF) 3 locations: - /// [`offset`, - /// `offset + col_basis_vector * colrow[0]`, - /// `offset + row_basis_vector * colrow[1]`] - /// which define the first instance's offset and the array's basis vectors. - /// Note that many GDS implementations only support manhattan basis vectors, and some - /// assume a certain axis mapping (e.g. x->columns, y->rows) and "reinterpret" the - /// basis vectors to match it. - xy: Vec, - - /// Number of columns and rows (AREF) || None (SREF) - colrow: Option<(i16, i16)>, - /// Properties associated with this reference. - properties: HashMap::>, -} - -impl Element for Reference { - fn read(input: &[u8]) -> IResult<'_, Self> { - let mut invert_y = false; - let mut mag = 1.0; - let mut angle_deg = 0.0; - let mut colrow = None; - let (input, struct_name) = SNAME::skip_and_read(input)?; - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_STRANS => { - let result = STRANS::read_data(input, header.data_size)?; - input = result.0; - invert_y = result.1[0]; - }, - records::RTAG_MAG => - {(input, mag) = MAG::read_data(input, header.data_size)?;}, - records::RTAG_ANGLE => - {(input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, - records::RTAG_COLROW => { - let result = COLROW::read_data(input, header.data_size)?; - input = result.0; - colrow = Some((result.1[0], result.1[1])); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - let (input, properties) = read_properties(input)?; - Ok((input, Reference{ - struct_name: struct_name, - xy: xy, - properties: properties, - colrow: colrow, - invert_y: invert_y, - mag: mag, - angle_deg: angle_deg - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += match self.colrow { - None => SREF::write(ww, &())?, - Some(_) => AREF::write(ww, &())?, - }; - - size += SNAME::write(ww, &self.struct_name)?; - if self.angle_deg != 0.0 || self.mag != 1.0 || self.invert_y { - let strans = { - let mut arr = [false; 16]; - arr[0] = self.invert_y; - arr - }; - size += STRANS::write(ww, &strans)?; - if self.mag != 1.0 { - size += MAG::write(ww, &self.mag)?; - } - if self.angle_deg != 0.0 { - size += ANGLE::write(ww, &self.angle_deg)?; - } - } - - if let Some(cr) = self.colrow { - size += COLROW::write(ww, &vec!{cr.0, cr.1})?; - } - - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - -impl Reference { - pub fn check(&self) { - if self.colrow.is_some() { - assert!(self.xy.len() != 6, "colrow is Some, so expected size-6 xy. Got {:?}", self.xy); - } else { - assert!(self.xy.len() != 2, "Expected size-2 xy. Got {:?}", self.xy); - } - } -} - - -/// -/// Datastructure representing a Boundary element. -/// -#[derive(Debug, Clone)] -pub struct Boundary { - /// (layer, data_type) tuple - layer: (i16, i16), - /// Ordered vertices of the shape. First and last points should be identical. Order x0, y0, x1,... - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Boundary { - fn read(input: &[u8]) -> IResult<'_, Self> { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = DATATYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Boundary{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += BOUNDARY::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += DATATYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Path element. -/// -/// If `path_type < 4`, `extension` values are not written. -/// During read, `exension` defaults to (0, 0) even if unused. -/// -#[derive(Debug, Clone)] -pub struct Path { - /// (layer, data_type) tuple - layer: (i16, i16), - /// End-cap type (0: flush, 1: circle, 2: square, 4: custom) - path_type: i16, - /// Path width - width: i32, - /// Extension when using path_type=4. Ignored otherwise. - extension: (i32, i32), - /// Path centerline coordinates. [x0, y0, x1, y1,...] - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Path { - fn read(input: &[u8]) -> IResult<'_, Self> { - let mut path_type = 0; - let mut width = 0; - let mut bgn_ext = 0; - let mut end_ext = 0; - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = DATATYPE::read(input)?; - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PATHTYPE => - {(input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, - records::RTAG_WIDTH => - {(input, width) = WIDTH::read_data(input, header.data_size)?;}, - records::RTAG_BGNEXTN => - {(input, bgn_ext) = BGNEXTN::read_data(input, header.data_size)?;}, - records::RTAG_ENDEXTN => - {(input, end_ext) = ENDEXTN::read_data(input, header.data_size)?;}, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - let (input, properties) = read_properties(input)?; - Ok((input, Path{ - layer: (layer, dtype), - xy: xy, - properties: properties, - extension: (bgn_ext, end_ext), - path_type: path_type, - width: width, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += PATH::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += DATATYPE::write(ww, &self.layer.1)?; - if self.path_type != 0 { - size += PATHTYPE::write(ww, &self.path_type)?; - } - if self.width != 0 { - size += WIDTH::write(ww, &self.width)?; - } - - if self.path_type < 4 { - let (bgn_ext, end_ext) = self.extension; - if bgn_ext != 0 { - size += BGNEXTN::write(ww, &bgn_ext)?; - } - if end_ext != 0 { - size += ENDEXTN::write(ww, &end_ext)?; - } - } - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Box element. Rarely used. -/// -#[derive(Debug, Clone)] -pub struct GDSBox { - /// (layer, box_type) tuple - layer: (i16, i16), - /// Box coordinates (5 pairs) - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for GDSBox { - fn read(input: &[u8]) -> IResult<'_, Self> { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = BOXTYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, GDSBox{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += BOX::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += BOXTYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Node element. Rarely used. -/// -#[derive(Debug, Clone)] -pub struct Node { - /// (layer, box_type) tuple - layer: (i16, i16), - /// 1-50 pairs of coordinates. - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Node { - fn read(input: &[u8]) -> IResult<'_, Self> { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = NODETYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Node{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += NODE::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += NODETYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - -/// -/// Datastructure representing a text label. -/// -#[derive(Debug, Clone)] -pub struct Text { - /// (layer, node_type) tuple - layer: (i16, i16), - - /// Bit array. Default all zeros. - /// bits 0-1: 00 left/01 center/10 right - /// bits 2-3: 00 top/01 middle/10 bottom - /// bits 4-5: font number - presentation: [bool; 16], - - /// Default 0 - path_type: i16, - /// Default 0 - width: i32, - /// Vertical inversion. Default false. - invert_y: bool, - /// Scaling factor. Default 1. - mag: f64, - /// Rotation (ccw). Default 0. - angle_deg: f64, - /// Position (1 pair only) - xy: Vec, - /// Text content - string: Vec, - /// Properties for the element. - properties: HashMap::> -} - -impl Element for Text { - fn read(input: &[u8]) -> IResult<'_, Self> { - let mut path_type = 0; - let mut presentation = [false; 16]; - let mut invert_y = false; - let mut width = 0; - let mut mag = 1.0; - let mut angle_deg = 0.0; - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = TEXTTYPE::read(input)?; - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PRESENTATION => - {(input, presentation) = PRESENTATION::read_data(input, header.data_size)?;}, - records::RTAG_PATHTYPE => - {(input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, - records::RTAG_WIDTH => - {(input, width) = WIDTH::read_data(input, header.data_size)?;}, - records::RTAG_STRANS => { - let result = STRANS::read_data(input, header.data_size)?; - input = result.0; - invert_y = result.1[0]; - }, - records::RTAG_MAG => - {(input, mag) = MAG::read_data(input, header.data_size)?;}, - records::RTAG_ANGLE => - {(input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - } - (input, header) = RecordHeader::read(input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - - let (input, string) = STRING::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Text{ - layer: (layer, dtype), - xy: xy, - properties: properties, - string: string, - presentation: presentation, - path_type: path_type, - width: width, - invert_y: invert_y, - mag: mag, - angle_deg: angle_deg, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += TEXT::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += TEXTTYPE::write(ww, &self.layer.1)?; - if self.presentation.iter().any(|&x| x) { - size += PRESENTATION::write(ww, &self.presentation)?; - } - if self.path_type != 0 { - size += PATHTYPE::write(ww, &self.path_type)?; - } - if self.width != 0 { - size += WIDTH::write(ww, &self.width)? - } - if self.angle_deg != 0.0 || self.mag != 1.0 || self.invert_y { - let strans = { - let mut arr = [false; 16]; - arr[0] = self.invert_y; - arr - }; - size += STRANS::write(ww, &strans)?; - if self.mag != 1.0 { - size += MAG::write(ww, &self.mag)?; - } - if self.angle_deg != 0.0 { - size += ANGLE::write(ww, &self.angle_deg)?; - } - } - size += XY::write(ww, &self.xy)?; - size += STRING::write(ww, &self.string)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} -*/ diff --git a/src/lib.rs b/src/lib.rs index 0698e3e..c31b907 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,12 @@ -extern crate byteorder; - pub mod basic; pub mod record; pub mod records; -pub mod elements; +mod elements; pub mod library; pub mod review; use crate::library::{read_library, read_selected_library, scan_library}; -use byteorder::{ByteOrder, BigEndian}; use std::any::Any; use std::cell::RefCell; use std::mem::size_of; @@ -23,14 +20,26 @@ use std::str; use std::fs; use std::slice; -use arrow::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; -use arrow::array::{Array, StructArray}; +use arrow_array::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, StructArray}; thread_local! { static LAST_ERROR: RefCell> = const { RefCell::new(None) }; } +/// Stable C API version. Increment for incompatible function/ownership changes. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_abi_version() -> u32 { + 1 +} + +/// Layout of the full-read and hierarchy-scan Arrow payloads. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_arrow_schema_version() -> u32 { + 1 +} + fn clear_last_error() { LAST_ERROR.with(|last_error| *last_error.borrow_mut() = None); @@ -163,6 +172,26 @@ unsafe fn input_slice<'a, T>(data: *const T, size: usize, name: &str) -> Result< Ok(unsafe { slice::from_raw_parts(data, size) }) } +/// Read a borrowed GDSII buffer, exporting Arrow data that owns its storage. +/// +/// # Safety +/// `data` must cover `size` readable bytes (null is allowed for zero bytes). +/// Outputs must be initialized Arrow FFI structures, as for `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, +) -> i32 { + export_struct_array("read_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = read_library(input) + .map_err(|err| format!("read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + /// Scan cell ranges and hierarchy references. Return 0 on success or -1 on error. /// @@ -207,8 +236,10 @@ pub unsafe extern "C" fn read_cells_bytes( .ok_or_else(|| "range count is too large".to_string())?; let flat_ranges = unsafe { input_slice(ranges, range_size, "ranges") }?; let cell_ranges: Vec<(u64, u64)> = flat_ranges - .chunks_exact(2) - .map(|chunk| (chunk[0], chunk[1])) + .as_chunks::<2>() + .0 + .iter() + .map(|&[start, end]| (start, end)) .collect(); let (_input, struct_arr) = read_selected_library(input, &cell_ranges) .map_err(|err| format!("selected-cell read failed {}", describe_parse_error(input, err)))?; @@ -217,80 +248,60 @@ pub unsafe extern "C" fn read_cells_bytes( } -macro_rules! impl_i16be { - ( $tt:ty, $arr:ident, $size:ident ) => { - { - let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; - for xx in sl.iter_mut() { - // All source types represent the target bounds exactly in f64. - let value = *xx as f64; - if !value.is_finite() || value < i16::MIN as f64 || value > i16::MAX as f64 { - return *xx; - } - - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i16(&mut buf, *xx as i16); - *xx = <$tt>::from_ne_bytes(buf); +macro_rules! impl_intbe { + ($source:ty, $target:ty, $arr:ident, $size:ident) => {{ + let values = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; + for value in values { + // All source types represent the target bounds exactly in f64. + let number = *value as f64; + if !number.is_finite() + || number < <$target>::MIN as f64 + || number > <$target>::MAX as f64 + { + return *value; } - 0 as $tt + let mut slot = [0; size_of::<$source>()]; + slot[..size_of::<$target>()].copy_from_slice(&(*value as $target).to_be_bytes()); + *value = <$source>::from_ne_bytes(slot); } - } -} - - -macro_rules! impl_i32be { - ( $tt:ty, $arr:ident, $size:ident ) => { - { - let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; - for xx in sl.iter_mut() { - let value = *xx as f64; - if !value.is_finite() || value < i32::MIN as f64 || value > i32::MAX as f64 { - return *xx; - } - - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i32(&mut buf, *xx as i32); - *xx = <$tt>::from_ne_bytes(buf); - } - 0 as $tt - } - } + 0 as $source + }}; } #[unsafe(no_mangle)] -pub unsafe extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_i16be!(f64, arr, size) } +pub unsafe extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_i32be!(f64, arr, size) } +pub unsafe extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_i16be!(f32, arr, size) } +pub unsafe extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_i32be!(f32, arr, size) } +pub unsafe extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_i16be!(u64, arr, size) } +pub unsafe extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_i32be!(u64, arr, size) } +pub unsafe extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_i16be!(i64, arr, size) } +pub unsafe extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_i32be!(i64, arr, size) } +pub unsafe extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_i16be!(u32, arr, size) } +pub unsafe extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_i32be!(u32, arr, size) } +pub unsafe extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_i16be!(i32, arr, size) } +pub unsafe extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_i32be!(i32, arr, size) } +pub unsafe extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i32, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_i16be!(u16, arr, size) } +pub unsafe extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_intbe!(u16, i16, arr, size) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_i16be!(i16, arr, size) } +pub unsafe extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_intbe!(i16, i16, arr, size) } diff --git a/src/library.rs b/src/library.rs index 862c85f..2e0ac81 100644 --- a/src/library.rs +++ b/src/library.rs @@ -3,23 +3,23 @@ /// use std::io::Write; -//use std::collections::HashMap; -pub use crate::record; -pub use crate::record::{RecordHeader, Record}; -pub use crate::records; -pub use crate::elements; -pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; -pub use crate::basic::{IResult, OResult, take_bytes, fail}; +use crate::record::{Record, RecordFrame}; +use crate::records; +use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +use crate::basic::{ErrType, IResult, OResult, decode_name, fail}; use std::string::String; use std::collections::HashMap; use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields}; -use arrow::array::{ - BooleanBuilder, StructBuilder, StringBuilder, UInt64Builder, UInt32Builder, Int16Builder, Float64Builder, +use arrow_schema::{DataType, Field, Fields}; +use arrow_array::{ StructArray, + builder::{ + BooleanBuilder, Float64Builder, Int16Builder, StringBuilder, StructBuilder, + UInt32Builder, UInt64Builder, + }, }; @@ -129,6 +129,28 @@ fn make_top_level_builder(cells_field: Field, extra_fields: Vec) -> Struc } +fn reference_type(is_array: bool, properties: Option<&DataType>) -> DataType { + let mut fields = vec![ + Field::new("target", DataType::UInt32, false), + Field::new("invert_y", DataType::Boolean, false), + Field::new("scale", DataType::Float64, false), + Field::new("angle_rad", DataType::Float64, false), + Field::new("xy", DataType::UInt64, false), + ]; + if is_array { + fields.extend([ + Field::new("xy0", DataType::UInt64, false), + Field::new("xy1", DataType::UInt64, false), + Field::new("counts", DataType::UInt32, false), + ]); + } + if let Some(properties) = properties { + fields.push(Field::new("properties", properties.clone(), true)); + } + DataType::Struct(Fields::from(fields)) +} + + fn make_library_builder() -> StructBuilder { let property_t = DataType::Struct(Fields::from(vec![ Field::new("key", DataType::Int16, false), @@ -139,46 +161,10 @@ fn make_library_builder() -> StructBuilder { Field::new_list_field(property_t, false) )); - - let sref_struct_t = DataType::Struct(Fields::from(vec![ - Field::new("target", DataType::UInt32, false), - Field::new("invert_y", DataType::Boolean, false), - Field::new("scale", DataType::Float64, false), - Field::new("angle_rad", DataType::Float64, false), - Field::new("xy", DataType::UInt64, false), - ])); - - let aref_struct_t = DataType::Struct(Fields::from(vec![ - Field::new("target", DataType::UInt32, false), - Field::new("invert_y", DataType::Boolean, false), - Field::new("scale", DataType::Float64, false), - Field::new("angle_rad", DataType::Float64, false), - Field::new("xy", DataType::UInt64, false), - Field::new("xy0", DataType::UInt64, false), - Field::new("xy1", DataType::UInt64, false), - Field::new("counts", DataType::UInt32, false), - ])); - - let sref_prop_struct_t = DataType::Struct(Fields::from(vec![ - Field::new("target", DataType::UInt32, false), - Field::new("invert_y", DataType::Boolean, false), - Field::new("scale", DataType::Float64, false), - Field::new("angle_rad", DataType::Float64, false), - Field::new("xy", DataType::UInt64, false), - Field::new("properties", property_list_t.clone(), true), - ])); - - let aref_prop_struct_t = DataType::Struct(Fields::from(vec![ - Field::new("target", DataType::UInt32, false), - Field::new("invert_y", DataType::Boolean, false), - Field::new("scale", DataType::Float64, false), - Field::new("angle_rad", DataType::Float64, false), - Field::new("xy", DataType::UInt64, false), - Field::new("xy0", DataType::UInt64, false), - Field::new("xy1", DataType::UInt64, false), - Field::new("counts", DataType::UInt32, false), - Field::new("properties", property_list_t.clone(), true), - ])); + let sref_struct_t = reference_type(false, None); + let aref_struct_t = reference_type(true, None); + let sref_prop_struct_t = reference_type(false, Some(&property_list_t)); + let aref_prop_struct_t = reference_type(true, Some(&property_list_t)); let text_struct_t = DataType::Struct(Fields::from(vec![ Field::new("layer", DataType::UInt32, false), @@ -311,16 +297,7 @@ fn make_library_builder() -> StructBuilder { fn make_scan_builder() -> StructBuilder { - let scan_ref_struct_t = DataType::Struct(Fields::from(vec![ - Field::new("target", DataType::UInt32, false), - Field::new("invert_y", DataType::Boolean, false), - Field::new("scale", DataType::Float64, false), - Field::new("angle_rad", DataType::Float64, false), - Field::new("xy", DataType::UInt64, false), - Field::new("xy0", DataType::UInt64, false), - Field::new("xy1", DataType::UInt64, false), - Field::new("counts", DataType::UInt32, false), - ])); + let scan_ref_struct_t = reference_type(true, None); let scan_ref_list_t = DataType::List(Arc::new( Field::new_list_field(scan_ref_struct_t, false) @@ -347,13 +324,14 @@ fn make_scan_builder() -> StructBuilder { fn append_file_header( lib_builder: &mut StructBuilder, header: &FileHeader, - ) { + ) -> Result<(), ErrType> { + let name = decode_name(&header.name).map_err(ErrType::Failed)?; let dbu_builder = lib_builder.field_builder::(0).unwrap(); dbu_builder.append_value(header.meters_per_db_unit); let uu_builder = lib_builder.field_builder::(1).unwrap(); uu_builder.append_value(header.user_units_per_db_unit); let libname_builder = lib_builder.field_builder::(2).unwrap(); - libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); + libname_builder.append_value(name); let mtl_builder = lib_builder.field_builder::(3).unwrap(); let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); mt_builder.append_values(&header.mod_time, &[true; 6]); @@ -362,6 +340,7 @@ fn append_file_header( let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); at_builder.append_values(&header.acc_time, &[true; 6]); atl_builder.append(true); + Ok(()) } @@ -370,11 +349,15 @@ fn append_names( field_index: usize, names: HashMap, ) { - let mut ids: HashMap = names.into_iter().map(|(kk, vv)| (vv, kk)).collect(); + // Interned IDs are consecutive, so a vector directly represents ID order. + let mut ordered_names = vec![String::new(); names.len()]; + for (name, id) in names { + ordered_names[id as usize] = name; + } let names_builder = lib_builder.field_builder::(field_index).unwrap(); let name_builder = names_builder.values().as_any_mut().downcast_mut::().unwrap(); - for id in 0..ids.len() { - name_builder.append_value(ids.remove(&id.try_into().unwrap()).unwrap()); + for name in ordered_names { + name_builder.append_value(name); } names_builder.append(true); } @@ -384,12 +367,13 @@ fn append_layers( lib_builder: &mut StructBuilder, layers: HashMap, ) { - let mut layer_ids: HashMap = layers.into_iter().map(|(kk, vv)| (vv, kk)).collect(); + let mut ordered_layers = vec![0; layers.len()]; + for (layer, id) in layers { + ordered_layers[id as usize] = layer; + } let layers_builder = lib_builder.field_builder::(7).unwrap(); let layer_builder = layers_builder.values().as_any_mut().downcast_mut::().unwrap(); - for layer_id in 0..layer_ids.len() { - layer_builder.append_value(layer_ids.remove(&layer_id.try_into().unwrap()).unwrap()); - } + layer_builder.append_slice(&ordered_layers); layers_builder.append(true); } @@ -425,6 +409,24 @@ fn intern_name(names: &mut HashMap, name: String) -> u32 { } +fn read_cell_header<'a>(begin: &RecordFrame<'a>, input: &'a [u8]) -> IResult<'a, String> { + if begin.tag != records::RTAG_BGNSTR { + return fail( + begin.data, + format!( + "Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", + begin.tag, + records::RTAG_BGNSTR + ), + ); + } + begin.decode::()?; + let (input, name_bytes) = records::STRNAME::read(input)?; + let name = decode_name(&name_bytes).map_err(|error| (input, ErrType::Failed(error)))?; + Ok((input, name.to_owned())) +} + + fn append_full_cell<'a>( cells_builder: &mut DListBuilder, names: &mut HashMap, @@ -456,21 +458,23 @@ where F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, { let input_size = input.len(); - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDLIB { - let record_start = base_offset + input_size - input.len() - 4; - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_BGNSTR { - let name_bytes; - (input, name_bytes) = records::STRNAME::read(input)?; - let name = String::from_utf8(name_bytes).unwrap(); - let id = intern_name(names, name); - let (next_input, ()) = on_struct(input, record_start, id, names)?; - input = next_input; + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { + records::RTAG_BGNSTR => { + let record_start = base_offset + input_size - input.len(); + let (after_name, name) = read_cell_header(&record, remaining)?; + let id = intern_name(names, name); + (input, ()) = on_struct(after_name, record_start, id, names)?; + } + records::RTAG_ENDLIB => { + record.decode::()?; + return Ok((remaining, ())); + } + _ => input = remaining, } - (input, header) = RecordHeader::read(input)?; } - Ok((input, ())) } @@ -479,7 +483,7 @@ pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_library_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -515,7 +519,7 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { let mut lib_builder = make_scan_builder(); let (input, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; let base_offset = full_input_size - input.len(); let mut names = HashMap::::new(); @@ -559,8 +563,8 @@ pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { /// as its remainder; it does not advance a sequential parsing cursor. pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { let mut lib_builder = make_library_builder(); - let (_, header) = FileHeader::read(input)?; - append_file_header(&mut lib_builder, &header); + let (header_end, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (header_end, error))?; let mut names = HashMap::::new(); let mut layers = HashMap::::new(); @@ -573,10 +577,8 @@ pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IRes let start = usize::try_from(*start_u64).unwrap(); let end = usize::try_from(*end_u64).unwrap(); - let struct_input = &input[start..end]; - let (struct_input, _) = records::BGNSTR::read(struct_input)?; - let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; - let name = String::from_utf8(name_bytes).unwrap(); + let (struct_input, begin) = RecordFrame::read(&input[start..end])?; + let (struct_input, name) = read_cell_header(&begin, struct_input)?; let file_offset = end - struct_input.len(); let id = intern_name(&mut names, name); @@ -601,306 +603,3 @@ pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IRes let lib = lib_builder.finish(); Ok((input, lib)) } - -/* -/// -/// Scan through a GDS file, building a table of -/// {b'structure_name': byte_offset}. -/// The intent of this function is to enable random access -/// and/or partial (structure-by-structure) reads. -/// -/// Args: -/// input: Seekable input to read from. Should be positioned -/// before the first structure record, but possibly -/// already past the file header. -/// -pub fn scan_structs(input: &[u8]) -> IResult, usize>> { - let input_size = input.len(); - let mut positions = HashMap::new(); - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDLIB { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_BGNSTR { - let name; - (input, name) = records::STRNAME::read(input)?; - if positions.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - let position = input_size - input.len(); - positions.insert(name, position); - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, positions)) -} -*/ - -/* -#[derive(Debug, Clone)] -pub struct Cell { - name: Vec, - - boundaries: Vec, - paths: Vec, - nodes: Vec, - boxes: Vec, - texts: Vec, - refs: Vec, -} - -impl Cell { - /// Build an empty cell - pub fn new(name: Vec) -> Self { - Cell{ - name: name, - boundaries: Vec::new(), - paths: Vec::new(), - nodes: Vec::new(), - boxes: Vec::new(), - texts: Vec::new(), - refs: Vec::new(), - } - } - - /// Skip to the next structure and attempt to read it. - /// - /// Args: - /// input: Seekable input to read from. - /// - /// Returns: - /// (name, elements) if a structure was found. - /// None if no structure was found before the end of the library. - /// - pub fn read(input: &[u8]) -> IResult> { - let (input, success) = records::BGNSTR::skip_past(input)?; - if !success { - return Ok((input, None)) - } - - let (input, name) = records::STRNAME::read(input)?; - let mut cell = Cell::new(name); - let (input, _) = cell.read_elements(input)?; - Ok((input, Some(cell))) - } - - /// Read elements from the input until an ENDSTR - /// record is encountered. The ENDSTR record is also - /// consumed. - /// - /// Args: - /// input: Seekable input to read from. - /// - /// Returns: - /// List of element objects. - /// - pub fn read_elements<'a>(&mut self, input: &'a [u8]) -> IResult<'a, ()> { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - match header.tag { - records::RTAG_BOUNDARY => { - let boundary; - (input, _) = records::BOUNDARY::read(input)?; - (input, boundary) = elements::Boundary::read(input)?; - self.boundaries.push(boundary); - }, - records::RTAG_PATH => { - let path; - (input, _) = records::PATH::read(input)?; - (input, path) = elements::Path::read(input)?; - self.paths.push(path); - }, - records::RTAG_NODE => { - let node; - (input, _) = records::NODE::read(input)?; - (input, node) = elements::Node::read(input)?; - self.nodes.push(node); - }, - records::RTAG_BOX => { - let gds_box; - (input, _) = records::BOX::read(input)?; - (input, gds_box) = elements::GDSBox::read(input)?; - self.boxes.push(gds_box); - }, - records::RTAG_TEXT => { - let txt; - (input, _) = records::TEXT::read(input)?; - (input, txt) = elements::Text::read(input)?; - self.texts.push(txt); - }, - records::RTAG_SREF => { - let sref; - (input, _) = records::SREF::read(input)?; - (input, sref) = elements::Reference::read(input)?; - self.refs.push(sref); - }, - records::RTAG_AREF => { - let aref; - (input, _) = records::AREF::read(input)?; - (input, aref) = elements::Reference::read(input)?; - self.refs.push(aref); - }, - _ => { - // don't care, skip - (input, _) = take_bytes(input, header.data_size)?; - } - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, ())) - } - - /// - /// Write a structure to the provided input. - /// - /// Args: - /// name: Structure name (ascii-encoded). - /// elements: List of Elements containing the geometry and text in this struct. - /// cre_time: Creation time (optional). - /// mod_time: Modification time (optional). - /// - /// Return: - /// Number of bytes written - /// - pub fn write( - &self, - ww: &mut W, - cre_time: Option<[i16; 6]>, - mod_time: Option<[i16; 6]>, - ) -> OResult { - let mut size = 0; - size += records::BGNSTR::write(ww, &[cre_time.unwrap_or(DEFAULT_DATE), - mod_time.unwrap_or(DEFAULT_DATE)])?; - size += records::STRNAME::write(ww, &self.name)?; - size += self.write_elements(ww)?; - size += records::ENDSTR::write(ww, &())?; - Ok(size) - } - - pub fn write_elements(&self, ww: &mut W) -> OResult { - let mut size = 0; - for boundary in &self.boundaries { - size += boundary.write(ww)?; - } - for path in &self.paths { - size += path.write(ww)?; - } - for node in &self.nodes { - size += node.write(ww)?; - } - for gds_box in &self.boxes { - size += gds_box.write(ww)?; - } - for text in &self.texts { - size += text.write(ww)?; - } - for reference in &self.refs { - size += reference.write(ww)?; - } - Ok(size) - } -} -*/ - -/* -/// -/// Scan through a GDS file, building a table of instance counts -/// `{b'structure_name': {b'ref_name': count}}`. -/// -/// This is intended to provide a fast overview of the file's -/// contents without performing a full read of all elements. -/// -/// Args: -/// input: Seekable input to read from. Should be positioned -/// before the first structure record, but possibly -/// already past the file header. -/// -pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap::, u32>>> { - let mut structures = HashMap::new(); - - let mut ref_name = None; - let mut ref_count = None; - let (mut input, mut header) = RecordHeader::read(input)?; - let mut cur_structure = HashMap::new(); - while header.tag != records::RTAG_ENDLIB { - match header.tag { - records::RTAG_BGNSTR => { - (input, _) = take_bytes(input, header.data_size)?; - let result = records::STRNAME::read(input)?; - input = result.0; - let name = result.1; - if structures.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - let mut cur_structure = HashMap::new(); - structures.insert(name, cur_structure); - ref_name = None; - ref_count = None; - }, - records::RTAG_SNAME => { - let result = records::SNAME::read_data(input, header.data_size)?; - input = result.0; - ref_name = Some(result.1); - }, - records::RTAG_COLROW => { - let result = records::COLROW::read_data(input, header.data_size)?; - input = result.0; - let (col, row) = (result.1[0], result.1[1]); - ref_count = Some((col * row) as u32); - }, - records::RTAG_ENDEL => { - *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); - }, - _ => { - (input, _) = take_bytes(input, header.data_size)?; - }, - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, structures)) -} -*/ - -/* -pub fn count_ref(input: &[u8]) -> IResult, u32))> { - let (input, found_struc) = records::BGNSTR.skip_past(input)?; - if !found_struc { - return Ok((input, None)) - } - let mut cur_structure = HashMap::new(); - let (input, name) = records::STRNAME::read(input)?; - if structures.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - let mut ref_name = None; - let mut ref_count = None; - while header.tag != records::RTAG_ENDEL { - match header.tag { - records::RTAG_SNAME => { - let result = records::SNAME::read_data(input1, header.data_size)?; - input1 = result.0; - ref_name = Some(result.1); - }, - records::RTAG_COLROW => { - let result = records::COLROW::read_data(input1, header.data_size)?; - input1 = result.0; - let (col, row) = (result.1[0], result.1[1]); - ref_count = Some((col * row) as u32); - }, - _ => { - (input1, _) = take_bytes(input1, header.data_size)?; - }, - } - (input1, header) = RecordHeader::read(input1)?; - } - // got ENDEL, update count for this reference - *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); - (input1, header) = RecordHeader::read(input1)?; - } - structures.insert(name, cur_structure); - (input, header) = RecordHeader::read(input1)?; -} -*/ diff --git a/src/misc.py b/src/misc.py deleted file mode 100644 index 1e507c7..0000000 --- a/src/misc.py +++ /dev/null @@ -1,39 +0,0 @@ -''' -https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py -''' -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import ctypes -import pyarrow as pa -from pyarrow.cffi import ffi - - c_schema = ffi.new('struct ArrowSchema*') - ptr_schema = int(ffi.cast('uintptr_t', c_schema)) - c_array = ffi.new('struct ArrowArray*') - ptr_array = int(ffi.cast('uintptr_t', c_array)) - - # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) - - # import gc - # gc.collect() # Make sure no Arrow data dangles in a ref cycle - # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) - - arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) - - diff --git a/src/record.rs b/src/record.rs index c2c0149..5ad0762 100644 --- a/src/record.rs +++ b/src/record.rs @@ -2,19 +2,14 @@ /// Generic record-level read/write functionality. /// use std::io::{self, Write}; -use byteorder::{ByteOrder, BigEndian}; -use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; +use crate::basic::{ErrType, IResult, OResult, fail, incomplete, take_bytes}; use crate::records; -//#[no_mangle] -//pub extern "C" fn write_record_header( - - #[repr(C)] #[derive(Debug, Clone)] pub struct RecordHeader { @@ -24,30 +19,66 @@ pub struct RecordHeader { impl RecordHeader { pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { - let (input, size) = parse_u16(input)?; - let (input, tag) = parse_u16(input)?; + let Some((&[size_hi, size_lo, tag_hi, tag_lo], payload)) = input.split_first_chunk::<4>() else { + return incomplete(input, Some(4)); + }; + let size = u16::from_be_bytes([size_hi, size_lo]); let Some(data_size) = size.checked_sub(4) else { return fail(input, format!("Record size {size} is smaller than its header")); }; - Ok((input, RecordHeader{tag, data_size})) + let tag = u16::from_be_bytes([tag_hi, tag_lo]); + Ok((payload, RecordHeader { tag, data_size })) } - pub fn pack_into(&self) -> [u8; 4] { - let size = self.data_size.checked_add(4).expect("Record too big!"); - let vals = [size, self.tag]; - let mut buf = [0x77; 4]; - BigEndian::write_u16_into(&vals, &mut buf); - buf + pub fn pack_into(&self) -> io::Result<[u8; 4]> { + let size = self.data_size.checked_add(4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + let [size_hi, size_lo] = size.to_be_bytes(); + let [tag_hi, tag_lo] = self.tag.to_be_bytes(); + Ok([size_hi, size_lo, tag_hi, tag_lo]) } pub fn write(&self, ww: &mut W) -> OResult { - let bytes = self.pack_into(); + let bytes = self.pack_into()?; ww.write_all(&bytes)?; Ok(bytes.len()) } } +/// A complete record borrowing its payload from the input. +pub(crate) struct RecordFrame<'a> { + pub tag: u16, + pub data: &'a [u8], +} + +impl<'a> RecordFrame<'a> { + pub fn read(input: &'a [u8]) -> IResult<'a, Self> { + let (_, header) = RecordHeader::read(input)?; + let (remaining, data) = read_payload(input, header.data_size)?; + Ok((remaining, Self { tag: header.tag, data })) + } + + /// Decode the payload after the caller has matched the record tag. + pub fn decode + ?Sized, D: RecordData>( + &self, + ) -> Result, (&'a [u8], ErrType)> { + let (_, value) = R::read_data(self.data, self.data.len() as u16)?; + Ok(value) + } +} + +fn read_payload(input: &[u8], size: u16) -> IResult<'_, &[u8]> { + if !size.is_multiple_of(2) { + return fail(input, "odd record length".to_string()); + } + let Some((record, remaining)) = input.split_at_checked(4 + usize::from(size)) else { + return fail(input, "truncated record".to_string()); + }; + Ok((remaining, &record[4..])) +} + + pub trait RecordData { type BareData<'a>; type InData : ?Sized; @@ -55,7 +86,6 @@ pub trait RecordData { fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; fn pack_into(buf: &mut [u8], data: &Self::InData); - //fn size(data: &Self::BareData<'_>) -> u16; fn pack(data: &Self::InData) -> Self::ByteData; } @@ -75,22 +105,19 @@ pub trait Record { } } - fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { - RecordHeader::read(input) - } - fn write_header(ww: &mut W, data_size: u16) -> OResult { + Self::check_size(data_size) + .map_err(|message| io::Error::new(io::ErrorKind::InvalidInput, message))?; RecordHeader{tag: Self::tag(), data_size}.write(ww) } fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + if let Err(message) = Self::check_size(size) { + return fail(input, message); + } RData::read(input, size) } - fn pack_data(buf: &mut [u8], data: &RData::InData) { - RData::pack_into(buf, data) - } - /// /// Skip to the end of the next occurence of this record. /// @@ -100,26 +127,28 @@ pub trait Record { /// fn skip_past(input: &[u8]) -> IResult<'_, bool> { let original_input = input; - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_ENDLIB { - return Ok((original_input, false)) + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, true)); } - (input, header) = RecordHeader::read(input)?; + if record.tag == records::RTAG_ENDLIB { + return Ok((original_input, false)); + } + input = remaining; } - (input, _) = take_bytes(input, header.data_size)?; - Ok((input, true)) } fn skip_and_read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - (input, header) = RecordHeader::read(input)?; + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, record.decode::()?)); + } + input = remaining; } - let (input, data) = Self::read_data(input, header.data_size)?; - Ok((input, data)) } fn expect_header(input: &[u8]) -> IResult<'_, u16> { @@ -132,18 +161,17 @@ pub trait Record { } fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { - let (input, size) = Self::expect_header(input)?; - Self::check_size(size).unwrap(); - let (input, data) = Self::read_data(input, size)?; - Ok((input, data)) + let (_, size) = Self::expect_header(input)?; + let (remaining, data) = read_payload(input, size)?; + let (_, value) = Self::read_data(data, size)?; + Ok((remaining, value)) } fn write(ww: &mut W, data: &RData::InData) -> OResult { let packed_data = RData::pack(data); let data_bytes = packed_data.as_ref(); - let len = u16::try_from(data_bytes.len()).ok() - .filter(|len| *len <= u16::MAX - 4) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + let len = u16::try_from(data_bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; Self::write_header(ww, len)?; ww.write_all(data_bytes)?; Ok(usize::from(len) + 4) @@ -157,7 +185,9 @@ impl RecordData for BitArray { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_bitarray(input) } @@ -180,7 +210,9 @@ impl RecordData for Int2 { type ByteData = [u8; 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2); + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_int2(input) } @@ -202,7 +234,9 @@ impl RecordData for Int4 { type ByteData = [u8; 4]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 4); + if size != 4 { + return fail(input, format!("Expected record size {}, got {size}", 4)); + } parse_int4(input) } @@ -225,14 +259,18 @@ impl RecordData for Int2Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size % 2 == 0, "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(2) { + return fail(input, format!("Record size must be divisible by 2, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int2ArrayReader{bytes})) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - BigEndian::write_i16_into(data, buf) + assert_eq!(buf.len(), data.len() * 2); + for (bytes, value) in buf.as_chunks_mut::<2>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } } fn pack(data: &Self::InData) -> Self::ByteData { @@ -266,14 +304,18 @@ impl RecordData for Int4Array { type ByteData = Vec; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size % 4 == 0, "Record must contain an integer quantity of integers"); - //let mut input = input; + if !size.is_multiple_of(4) { + return fail(input, format!("Record size must be divisible by 4, got {size}")); + } let (input, bytes) = take_bytes(input, size)?; Ok((input, Int4ArrayReader{bytes})) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - BigEndian::write_i32_into(data, buf) + assert_eq!(buf.len(), data.len() * 4); + for (bytes, value) in buf.as_chunks_mut::<4>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } } fn pack(data: &Self::InData) -> Self::ByteData { @@ -307,7 +349,9 @@ impl RecordData for Real8 { type ByteData = [u8; 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 8); + if size != 8 { + return fail(input, format!("Expected record size {}, got {size}", 8)); + } parse_real8(input) } @@ -329,7 +373,9 @@ impl RecordData for Real8Pair { type ByteData = [u8; 2 * 8]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 8); + if size != 2 * 8 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 8)); + } let (input, data0) = parse_real8(input)?; let (input, data1) = parse_real8(input)?; Ok((input, (data0, data1))) @@ -346,10 +392,6 @@ impl RecordData for Real8Pair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct ASCII; @@ -381,7 +423,9 @@ impl RecordData for DateTimePair { type ByteData = [u8; 2 * 6 * 2]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 2 * 6 * 2); + if size != 2 * 6 * 2 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2)); + } let (input, data0) = parse_datetime(input)?; let (input, data1) = parse_datetime(input)?; Ok((input, [data0, data1])) @@ -398,10 +442,6 @@ impl RecordData for DateTimePair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct Empty; @@ -411,7 +451,9 @@ impl RecordData for Empty { type ByteData = [u8; 0]; fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { - assert!(size == 0); + if size != 0 { + return fail(input, format!("Expected record size {}, got {size}", 0)); + } Ok((input, ())) } @@ -421,6 +463,4 @@ impl RecordData for Empty { fn pack(_data: &Self::InData) -> Self::ByteData { [] } - //fn write(ww: &mut W, data: &Self::BareData) { - //} } diff --git a/src/records.rs b/src/records.rs index 3731c9f..3f6ba58 100644 --- a/src/records.rs +++ b/src/records.rs @@ -2,9 +2,11 @@ /// Record type and tag definitions /// -use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; - -//use std::io::Write; +use crate::basic::{ErrType, IResult, fail}; +use crate::record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, +}; // record tags pub const RTAG_HEADER: u16 = 0x0002; @@ -77,42 +79,66 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300; pub const RTAG_SPACERERROR: u16 = 0x4400; pub const RTAG_CONTACT: u16 = 0x4500; -/* -// data types -pub const DATA_TYPE_NONE: u16 = 0x00; -pub const DATA_TYPE_BIT: u16 = 0x01; -pub const DATA_TYPE_INT16: u16 = 0x02; -pub const DATA_TYPE_INT32: u16 = 0x03; -pub const DATA_TYPE_REAL32: u16 = 0x04; -pub const DATA_TYPE_REAL64: u16 = 0x05; -pub const DATA_TYPE_STR: u16 = 0x06; -pub const MAX_DATA_SIZE: usize = 8; -/// Returns the size of the given data type in bytes. -pub fn data_size(t: u16) -> Option { - match t { - x if x == DATA_TYPE_NONE => 0, - x if x == DATA_TYPE_BIT => 2, - x if x == DATA_TYPE_INT16 => 2, - x if x == DATA_TYPE_INT32 => 4, - x if x == DATA_TYPE_REAL32 => 4, - x if x == DATA_TYPE_REAL64 => 8, - _ => 0 - } -*/ +// Define fixed-size records once for typed decoding and raw-record validation. +macro_rules! fixed_size_records { + ($($(#[$attr:meta])* $name:ident => $tag:ident, $data:ty, $size:expr;)+) => { + $( + $(#[$attr])* + pub struct $name; + impl Record<$data> for $name { + fn tag() -> u16 { $tag } + fn expected_size() -> Option { Some($size) } + } + )+ -pub struct HEADER; -impl Record for HEADER { - fn tag() -> u16 { RTAG_HEADER } - fn expected_size() -> Option { Some(2) } + pub(crate) fn check_payload_size(tag: u16, size: u16) -> Result<(), String> { + match tag { + $($tag => $name::check_size(size),)+ + RTAG_UNITS => UNITS::check_size(size), + RTAG_XY => XY::check_size(size), + _ => Ok(()), + } + } + }; } -//impl Record for HEADER; -pub struct BGNLIB; -impl Record for BGNLIB { - fn tag() -> u16 { RTAG_BGNLIB } - fn expected_size() -> Option { Some(2 * 2 * 6) } +fixed_size_records! { + HEADER => RTAG_HEADER, Int2, 2; + BGNLIB => RTAG_BGNLIB, DateTimePair, 2 * 2 * 6; + ENDLIB => RTAG_ENDLIB, Empty, 0; + BGNSTR => RTAG_BGNSTR, DateTimePair, 2 * 6 * 2; + ENDSTR => RTAG_ENDSTR, Empty, 0; + BOUNDARY => RTAG_BOUNDARY, Empty, 0; + PATH => RTAG_PATH, Empty, 0; + SREF => RTAG_SREF, Empty, 0; + AREF => RTAG_AREF, Empty, 0; + TEXT => RTAG_TEXT, Empty, 0; + LAYER => RTAG_LAYER, Int2, 2; + DATATYPE => RTAG_DATATYPE, Int2, 2; + WIDTH => RTAG_WIDTH, Int4, 4; + ENDEL => RTAG_ENDEL, Empty, 0; + COLROW => RTAG_COLROW, Int2Array, 4; + NODE => RTAG_NODE, Empty, 0; + TEXTTYPE => RTAG_TEXTTYPE, Int2, 2; + PRESENTATION => RTAG_PRESENTATION, BitArray, 2; + STRANS => RTAG_STRANS, BitArray, 2; + MAG => RTAG_MAG, Real8, 8; + ANGLE => RTAG_ANGLE, Real8, 8; + PATHTYPE => RTAG_PATHTYPE, Int2, 2; + GENERATIONS => RTAG_GENERATIONS, Int2, 2; + ELFLAGS => RTAG_ELFLAGS, BitArray, 2; + NODETYPE => RTAG_NODETYPE, Int2, 2; + PROPATTR => RTAG_PROPATTR, Int2, 2; + BOX => RTAG_BOX, Empty, 0; + BOXTYPE => RTAG_BOXTYPE, Int2, 2; + PLEX => RTAG_PLEX, Int4, 4; + BGNEXTN => RTAG_BGNEXTN, Int4, 4; + ENDEXTN => RTAG_ENDEXTN, Int4, 4; + STRCLASS => RTAG_STRCLASS, Int2, 2; + /// End of MASKS records + ENDMASKS => RTAG_ENDMASKS, Empty, 0; } pub struct LIBNAME; @@ -126,18 +152,14 @@ impl Record for UNITS { // (user_units_per_db_unit, db_units_per_meter) fn tag() -> u16 { RTAG_UNITS } fn expected_size() -> Option { Some(2 * 8) } -} -pub struct ENDLIB; -impl Record for ENDLIB { - fn tag() -> u16 { RTAG_ENDLIB } - fn expected_size() -> Option { Some(0) } -} - -pub struct BGNSTR; -impl Record for BGNSTR { - fn tag() -> u16 { RTAG_BGNSTR } - fn expected_size() -> Option { Some(2 * 6 * 2) } + fn read_data(input: &[u8], size: u16) -> IResult<'_, (f64, f64)> { + let (remaining, (user, meters)) = Real8Pair::read(input, size)?; + if !user.is_finite() || user <= 0.0 || !meters.is_finite() || meters <= 0.0 { + return fail(input, "UNITS must be positive finite values".to_string()); + } + Ok((remaining, (user, meters))) + } } pub struct STRNAME; @@ -146,70 +168,105 @@ impl Record for STRNAME { fn expected_size() -> Option { None } } -pub struct ENDSTR; -impl Record for ENDSTR { - fn tag() -> u16 { RTAG_ENDSTR } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOUNDARY; -impl Record for BOUNDARY { - fn tag() -> u16 { RTAG_BOUNDARY } - fn expected_size() -> Option { Some(0) } -} - -pub struct PATH; -impl Record for PATH { - fn tag() -> u16 { RTAG_PATH } - fn expected_size() -> Option { Some(0) } -} - -pub struct SREF; -impl Record for SREF { - fn tag() -> u16 { RTAG_SREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct AREF; -impl Record for AREF { - fn tag() -> u16 { RTAG_AREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXT; -impl Record for TEXT { - fn tag() -> u16 { RTAG_TEXT } - fn expected_size() -> Option { Some(0) } -} - -pub struct LAYER; -impl Record for LAYER { - fn tag() -> u16 { RTAG_LAYER } - fn expected_size() -> Option { Some(2) } -} - -pub struct DATATYPE; -impl Record for DATATYPE { - fn tag() -> u16 { RTAG_DATATYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct WIDTH; -impl Record for WIDTH { - fn tag() -> u16 { RTAG_WIDTH } - fn expected_size() -> Option { Some(4) } -} - pub struct XY; impl Record for XY { fn tag() -> u16 { RTAG_XY } fn expected_size() -> Option { None } + + fn check_size(size: u16) -> Result<(), String> { + if !size.is_multiple_of(8) { + return Err("XY payload must contain whole coordinate pairs".to_string()); + } + Ok(()) + } } -pub struct ENDEL; -impl Record for ENDEL { - fn tag() -> u16 { RTAG_ENDEL } - fn expected_size() -> Option { Some(0) } +impl XY { + pub(crate) fn point([x0, x1, x2, x3, y0, y1, y2, y3]: [u8; 8]) -> [i32; 2] { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + } + + /// Iterate over a validated XY payload. + pub(crate) fn points(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .copied() + .map(Self::point) + } + + /// Bounds of a rectangle encoded as a closed XY ring. + pub(crate) fn closed_rectangle(data: &[u8]) -> Option<[i32; 4]> { + if data.len() != 40 { + return None; + } + let mut points = Self::points(data); + let corners: [[i32; 2]; 4] = std::array::from_fn(|_| points.next().unwrap()); + if points.next() != Some(corners[0]) { + return None; + } + Self::rectangle(&corners) + } + + /// Bounds of four corners in ring order, with the closing point omitted. + pub(crate) fn rectangle(corners: &[[i32; 2]]) -> Option<[i32; 4]> { + let corners: &[[i32; 2]; 4] = corners.try_into().ok()?; + let min_x = corners.iter().map(|point| point[0]).min().unwrap(); + let max_x = corners.iter().map(|point| point[0]).max().unwrap(); + let min_y = corners.iter().map(|point| point[1]).min().unwrap(); + let max_y = corners.iter().map(|point| point[1]).max().unwrap(); + if min_x == max_x || min_y == max_y { + return None; + } + + let mut seen = 0_u8; + for (index, &[x, y]) in corners.iter().enumerate() { + let [next_x, next_y] = corners[(index + 1) % corners.len()]; + if (x == next_x) == (y == next_y) { + return None; + } + if (x != min_x && x != max_x) || (y != min_y && y != max_y) { + return None; + } + let corner = u8::from(x == max_x) << 1 | u8::from(y == max_y); + seen |= 1 << corner; + } + (seen == 0b1111).then_some([min_x, min_y, max_x, max_y]) + } +} + +/// The array dimensions and its three coordinates in GDSII record order. +pub(crate) struct ArrayPlacement { + pub counts: [u16; 2], + pub points: [[i32; 2]; 3], +} + +impl ArrayPlacement { + pub fn decode<'a>(colrow: &'a [u8], xy: &'a [u8]) -> Result { + let (_, mut counts) = COLROW::read_data(colrow, colrow.len() as u16)?; + let columns = counts.next().unwrap(); + let rows = counts.next().unwrap(); + if columns <= 0 || rows <= 0 { + return Err(( + colrow, + ErrType::Failed("AREF column and row counts must be positive".to_string()), + )); + } + if xy.len() != 24 { + return Err(( + xy, + ErrType::Failed("AREF needs exactly three XY coordinate pairs".to_string()), + )); + } + let mut points = XY::points(xy); + Ok(Self { + counts: [columns as u16, rows as u16], + points: std::array::from_fn(|_| points.next().unwrap()), + }) + } } pub struct SNAME; @@ -218,30 +275,6 @@ impl Record for SNAME { fn expected_size() -> Option { None } } -pub struct COLROW; -impl Record for COLROW { - fn tag() -> u16 { RTAG_COLROW } - fn expected_size() -> Option { Some(4) } -} - -pub struct NODE; -impl Record for NODE { - fn tag() -> u16 { RTAG_NODE } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXTTYPE; -impl Record for TEXTTYPE { - fn tag() -> u16 { RTAG_TEXTTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PRESENTATION; -impl Record for PRESENTATION { - fn tag() -> u16 { RTAG_PRESENTATION } - fn expected_size() -> Option { Some(2) } -} - pub struct SPACING; impl Record for SPACING { fn tag() -> u16 { RTAG_SPACING } @@ -254,24 +287,6 @@ impl Record for STRING { fn expected_size() -> Option { None } } -pub struct STRANS; -impl Record for STRANS { - fn tag() -> u16 { RTAG_STRANS } - fn expected_size() -> Option { Some(2) } -} - -pub struct MAG; -impl Record for MAG { - fn tag() -> u16 { RTAG_MAG } - fn expected_size() -> Option { Some(8) } -} - -pub struct ANGLE; -impl Record for ANGLE { - fn tag() -> u16 { RTAG_ANGLE } - fn expected_size() -> Option { Some(8) } -} - pub struct UINTEGER; impl Record for UINTEGER { fn tag() -> u16 { RTAG_UINTEGER } @@ -288,10 +303,9 @@ pub struct REFLIBS; impl Record for REFLIBS { fn tag() -> u16 { RTAG_REFLIBS } fn expected_size() -> Option { None } -} -impl REFLIBS { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -303,10 +317,9 @@ pub struct FONTS; impl Record for FONTS { fn tag() -> u16 { RTAG_FONTS } fn expected_size() -> Option { None } -} -impl FONTS { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -314,26 +327,13 @@ impl FONTS { } } -pub struct PATHTYPE; -impl Record for PATHTYPE { - fn tag() -> u16 { RTAG_PATHTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct GENERATIONS; -impl Record for GENERATIONS { - fn tag() -> u16 { RTAG_GENERATIONS } - fn expected_size() -> Option { Some(2) } -} - pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } fn expected_size() -> Option { None } -} -impl ATTRTABLE { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -353,12 +353,6 @@ impl Record for STRTYPE { fn expected_size() -> Option { None } } -pub struct ELFLAGS; -impl Record for ELFLAGS { - fn tag() -> u16 { RTAG_ELFLAGS } - fn expected_size() -> Option { Some(2) } -} - pub struct ELKEY; impl Record for ELKEY { fn tag() -> u16 { RTAG_ELKEY } @@ -377,54 +371,12 @@ impl Record for LINKKEYS { fn expected_size() -> Option { Some(2) } } -pub struct NODETYPE; -impl Record for NODETYPE { - fn tag() -> u16 { RTAG_NODETYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PROPATTR; -impl Record for PROPATTR { - fn tag() -> u16 { RTAG_PROPATTR } - fn expected_size() -> Option { Some(2) } -} - pub struct PROPVALUE; impl Record for PROPVALUE { fn tag() -> u16 { RTAG_PROPVALUE } fn expected_size() -> Option { None } } -pub struct BOX; -impl Record for BOX { - fn tag() -> u16 { RTAG_BOX } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOXTYPE; -impl Record for BOXTYPE { - fn tag() -> u16 { RTAG_BOXTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PLEX; -impl Record for PLEX { - fn tag() -> u16 { RTAG_PLEX } - fn expected_size() -> Option { Some(4) } -} - -pub struct BGNEXTN; -impl Record for BGNEXTN { - fn tag() -> u16 { RTAG_BGNEXTN } - fn expected_size() -> Option { Some(4) } -} - -pub struct ENDEXTN; -impl Record for ENDEXTN { - fn tag() -> u16 { RTAG_ENDEXTN } - fn expected_size() -> Option { Some(4) } -} - pub struct TAPENUM; impl Record for TAPENUM { fn tag() -> u16 { RTAG_TAPENUM } @@ -437,12 +389,6 @@ impl Record for TAPECODE { fn expected_size() -> Option { Some(2 * 6) } } -pub struct STRCLASS; -impl Record for STRCLASS { - fn tag() -> u16 { RTAG_STRCLASS } - fn expected_size() -> Option { Some(2) } -} - pub struct RESERVED; impl Record for RESERVED { fn tag() -> u16 { RTAG_RESERVED } @@ -461,13 +407,6 @@ impl Record for MASK { fn expected_size() -> Option { None } } -/// End of MASKS records -pub struct ENDMASKS; -impl Record for ENDMASKS { - fn tag() -> u16 { RTAG_ENDMASKS } - fn expected_size() -> Option { Some(0) } -} - pub struct LIBDIRSIZE; impl Record for LIBDIRSIZE { fn tag() -> u16 { RTAG_LIBDIRSIZE } diff --git a/src/review.rs b/src/review.rs index 6f7f557..baf99c4 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1,26 +1,53 @@ //! Deterministic GDS object review without the normalization performed by the //! Arrow reader. Raw payloads borrow the file; only polygon keys are copied. -use crate::{basic::decode_real8, record::RecordHeader, records::*}; +use crate::{ + basic::{ErrType, decode_name, decode_real8, encode_real8, unpad_ascii}, + record::{Record as _, RecordFrame}, + records::*, +}; use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet}, - fmt::Write, + fmt::{self, Write}, }; -mod names; -pub use names::{parse_layer, parse_layer_name, write_layer_names}; -mod selection; -pub use selection::{DisplayFilter, Scope}; -mod changes; -pub use changes::Replacement; +pub mod changes; +mod hierarchy; +pub mod names; +mod pair; pub mod report; +pub mod selection; +mod text; -/// A comparison limitation with source context independent of its wording. +use changes::Replacement; +use pair::CellReuse; +pub use pair::RevisionPair; +use selection::{DisplayFilter, Scope}; +pub use text::{TextReport, TextReportOptions, TextStyle}; + +/// Input revision associated with a diagnostic. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Side { + Old, + New, +} + +impl fmt::Display for Side { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + out.write_str(match self { + Self::Old => "old", + Self::New => "new", + }) + } +} + +/// A review error or comparison limitation, with structured source context. #[derive(Clone, Debug)] pub struct Diagnostic { pub code: &'static str, pub message: String, + pub side: Option, pub offset: Option, pub cell: Option, pub layer: Option, @@ -28,20 +55,79 @@ pub struct Diagnostic { } impl Diagnostic { - fn at(code: &'static str, offset: usize, message: String) -> Self { + pub fn new(code: &'static str, message: impl Into, side: Option) -> Self { Self { code, - message: error(offset, message), - offset: Some(offset), + message: message.into(), + side, + offset: None, cell: None, layer: None, target: None, } } + + fn at(code: &'static str, offset: usize, message: String) -> Self { + Self { + offset: Some(offset), + ..Self::new(code, error(offset, message), None) + } + } } -pub const PROFILE: &str = "klamath-object-v2"; -pub const LAYER_SENTINEL: &str = "KLAMATH_LAYER_NAMES_V1"; +impl fmt::Display for Diagnostic { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(side) = self.side { + write!(out, "{side}: ")?; + } + out.write_str(&self.message) + } +} + +/// The five meaningful combinations of completeness and comparison result. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Outcome { + Equal, + Different, + Incomplete { different: bool }, + Failed, +} + +impl Outcome { + pub fn status(self) -> &'static str { + match self { + Self::Equal | Self::Different => "complete", + Self::Incomplete { .. } => "incomplete", + Self::Failed => "failed", + } + } + + pub fn result(self) -> &'static str { + match self { + Self::Equal => "equal", + Self::Different | Self::Incomplete { different: true } => "different", + Self::Incomplete { different: false } | Self::Failed => "unknown", + } + } + + pub fn exit_code(self) -> u8 { + match self { + Self::Equal => 0, + Self::Different => 1, + _ => 2, + } + } + + pub fn is_equal(self) -> bool { + self == Self::Equal + } + + pub fn is_complete(self) -> bool { + matches!(self, Self::Equal | Self::Different) + } +} + +pub const PROFILE: &str = "pheon-structural-v1"; pub type Layer = (u16, u16); pub type Bounds = [i32; 4]; @@ -59,7 +145,7 @@ struct Key<'a> { properties: Vec<(u16, &'a [u8])>, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Object<'a> { pub offset: usize, pub end_offset: usize, @@ -81,16 +167,130 @@ impl Object<'_> { .find(|(t, _)| *t == tag) .map(|(_, v)| v.as_ref()) } +} - pub fn description(&self) -> String { - let mut out = kind_name(self.kind).to_string(); - for (tag, value) in &self.key.fields { - write!(out, " {}={}", tag_name(*tag), format_data(*tag, value)).unwrap(); +fn format_object<'a, W: fmt::Write>( + out: &mut W, + kind: u16, + layer: Option, + fields: impl Iterator + Clone, + properties: impl IntoIterator, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, +) -> fmt::Result { + let field = |tag| { + fields + .clone() + .find_map(|(t, data)| (t == tag).then_some(data)) + }; + let rectangle = if is_polygon(kind) { + field(RTAG_XY).and_then(XY::closed_rectangle) + } else { + None + }; + let target = if matches!(kind, RTAG_SREF | RTAG_AREF) { + field(RTAG_SNAME) + } else { + None + }; + let dimensions = if kind == RTAG_AREF { + field(RTAG_COLROW).and_then(array_dimensions) + } else { + None + }; + heading(out, object_name(kind, rectangle.is_some()))?; + if let Some(target) = target { + write!(out, " {}", quoted(target))?; + } + if let Some([columns, rows]) = dimensions { + write!(out, " ({columns} × {rows})")?; + } + for (tag, value) in fields { + if (layer.is_some() && matches_layer_field(kind, tag)) + || (tag == RTAG_SNAME && target.is_some()) + || (tag == RTAG_COLROW && dimensions.is_some()) + { + continue; } - for (attribute, value) in &self.key.properties { - write!(out, " property[{attribute}]={}", quoted(value)).unwrap(); + if tag == RTAG_XY && value.len().is_multiple_of(8) { + if let Some([xmin, ymin, xmax, ymax]) = rectangle { + write!(out, " ({xmin}, {ymin}) to ({xmax}, {ymax})")?; + } else { + write_points(out, value)?; + } + } else { + write!(out, " {}={}", tag_name(tag), format_exact_value(tag, value))?; } - out + } + for (attribute, value) in properties { + write!(out, " property[{attribute}]={}", quoted(value))?; + } + Ok(()) +} + +fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> { + let [column_hi, column_lo, row_hi, row_lo]: [u8; 4] = data.try_into().ok()?; + Some([ + u16::from_be_bytes([column_hi, column_lo]), + u16::from_be_bytes([row_hi, row_lo]), + ]) +} + +const POINTS_PER_LINE: usize = 4; + +fn write_points(out: &mut impl fmt::Write, data: &[u8]) -> fmt::Result { + let mut points = XY::points(data); + if points.len() == 1 { + let [x, y] = points.next().unwrap(); + return write!(out, " xy=({x}, {y})"); + } + out.write_str(" xy=[")?; + for (index, [x, y]) in points.enumerate() { + if index != 0 { + out.write_str(if index % POINTS_PER_LINE == 0 { + ",\n " + } else { + ", " + })?; + } + write!(out, "({x}, {y})")?; + } + out.write_str("]") +} + +impl fmt::Display for Object<'_> { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +fn write_object_heading( + out: &mut impl fmt::Write, + layer: Option, + name: &str, +) -> fmt::Result { + if let Some((layer, datatype)) = layer { + write!(out, "{layer}/{datatype}: ")?; + } + out.write_str(name) +} + +impl Object<'_> { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { + format_object( + out, + self.kind, + self.layer, + self.key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + self.key.properties.iter().copied(), + heading, + ) } } @@ -110,7 +310,6 @@ pub struct Document<'a> { pub cells: BTreeMap>, pub layer_names: BTreeMap, /// Unsupported features or invalid reference graphs prevent an equality claim. - pub issues: Vec, pub diagnostics: Vec, header: Vec<(u16, &'a [u8])>, global_issues: Vec, @@ -123,10 +322,6 @@ fn error(offset: usize, message: impl std::fmt::Display) -> String { format!("at byte {offset}: {message}") } -fn string(data: &[u8], offset: usize) -> Result { - decoded_string(unpad(data), offset) -} - fn decoded_string(data: &[u8], offset: usize) -> Result { if data.contains(&0) { return Err(error(offset, "name or metadata contains an embedded NUL")); @@ -136,35 +331,23 @@ fn decoded_string(data: &[u8], offset: usize) -> Result { .map_err(|_| error(offset, "name or metadata is not UTF-8")) } -fn unpad(data: &[u8]) -> &[u8] { - if data.last() == Some(&0) { - &data[..data.len() - 1] - } else { - data - } -} - fn integer(data: &[u8]) -> u16 { u16::from_be_bytes(data.try_into().unwrap()) } -fn points(data: &[u8]) -> Vec<[i32; 2]> { - data.chunks_exact(8) - .map(|p| { - [ - i32::from_be_bytes(p[..4].try_into().unwrap()), - i32::from_be_bytes(p[4..].try_into().unwrap()), - ] - }) - .collect() +fn coordinate_bounds(points: impl IntoIterator) -> Bounds { + let mut points = points.into_iter(); + let [x, y] = points.next().unwrap(); + points.fold([x, y, x, y], |[xmin, ymin, xmax, ymax], [x, y]| { + [xmin.min(x), ymin.min(y), xmax.max(x), ymax.max(y)] + }) } // Booth's least-rotation algorithm avoids quadratic work on repeated vertices. -fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { - let n = points.len(); +fn least_rotation(n: usize, point: impl Fn(usize) -> [i32; 2]) -> usize { let (mut i, mut j, mut k) = (0, 1, 0); while i < n && j < n && k < n { - match points[(i + k) % n].cmp(&points[(j + k) % n]) { + match point((i + k) % n).cmp(&point((j + k) % n)) { std::cmp::Ordering::Equal => k += 1, std::cmp::Ordering::Greater => { i += k + 1; @@ -182,56 +365,64 @@ fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { } } } - let start = i.min(j); - (0..n).map(|k| points[(start + k) % n]).collect() + i.min(j) } -fn canonical_ring(data: &[u8]) -> Vec { - let mut ring = points(data); - ring.pop(); // The caller has checked closure. - let forward = least_rotation(&ring); - ring.reverse(); - let backward = least_rotation(&ring); - let mut ring = forward.min(backward); - ring.push(ring[0]); - ring.into_iter() - .flatten() - .flat_map(i32::to_be_bytes) - .collect() +fn canonical_ring(data: &[u8]) -> Option> { + let (points, _) = data.as_chunks::<8>(); + let (last, ring) = points.split_last()?; + if ring.len() < 3 || ring.first() != Some(last) { + return None; + } + let n = ring.len(); + let first = least_rotation(n, |index| XY::point(ring[index])); + let split = n - least_rotation(n, |index| XY::point(ring[n - 1 - index])); + let forward = ring[first..].iter().chain(&ring[..first]); + let backward = ring[..split].iter().rev().chain(ring[split..].iter().rev()); + let decode = |point: &[u8; 8]| XY::point(*point); + let use_forward = forward + .clone() + .map(decode) + .cmp(backward.clone().map(decode)) + .is_le(); + if use_forward && first == 0 { + return Some(Cow::Borrowed(data)); + } + let mut result = Vec::with_capacity(data.len()); + if use_forward { + result.extend(forward.flatten().copied()); + } else { + result.extend(backward.flatten().copied()); + } + result.extend_from_within(..8); + Some(Cow::Owned(result)) } fn validate_payload(record: &Record<'_>) -> Result<(), String> { - let size = record.data.len(); - let valid = match record.tag { - RTAG_BGNLIB | RTAG_BGNSTR => size == 24, - RTAG_UNITS => size == 16, - RTAG_HEADER | RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE - | RTAG_PROPATTR | RTAG_PATHTYPE | RTAG_STRANS | RTAG_PRESENTATION | RTAG_ELFLAGS - | RTAG_STRCLASS | RTAG_GENERATIONS => size == 2, - RTAG_WIDTH | RTAG_BGNEXTN | RTAG_ENDEXTN | RTAG_PLEX | RTAG_COLROW => size == 4, - RTAG_MAG | RTAG_ANGLE => size == 8, - RTAG_XY => size % 8 == 0, - RTAG_ENDLIB | RTAG_ENDSTR | RTAG_ENDEL | RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF - | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX | RTAG_ENDMASKS => size == 0, - _ => true, - }; - if !valid { - return Err(error( + let size = record.data.len() as u16; + check_payload_size(record.tag, size).map_err(|message| { + error( record.offset, - format!("invalid payload size {size} for {}", tag_name(record.tag)), - )); - } - if record.tag == RTAG_UNITS - && record.data.chunks_exact(8).any(|bytes| { - let value = decode_real8(u64::from_be_bytes(bytes.try_into().unwrap())); - !value.is_finite() || value <= 0.0 - }) - { - return Err(error(record.offset, "UNITS must be positive finite values")); + format!( + "invalid payload size {size} for {}: {message}", + tag_name(record.tag) + ), + ) + })?; + if record.tag == RTAG_UNITS { + UNITS::read_data(record.data, size) + .map_err(|(_, reason)| payload_error(record.offset, reason))?; } Ok(()) } +fn payload_error(offset: usize, reason: ErrType) -> String { + match reason { + ErrType::Failed(message) => error(offset, message), + ErrType::Incomplete(_) => error(offset, "truncated record payload"), + } +} + fn is_element(tag: u16) -> bool { matches!( tag, @@ -239,6 +430,23 @@ fn is_element(tag: u16) -> bool { ) } +fn is_polygon(kind: u16) -> bool { + matches!(kind, RTAG_BOUNDARY | RTAG_BOX) +} + +fn layer_type_tag(kind: u16) -> u16 { + match kind { + RTAG_TEXT => RTAG_TEXTTYPE, + RTAG_NODE => RTAG_NODETYPE, + RTAG_BOX => RTAG_BOXTYPE, + _ => RTAG_DATATYPE, + } +} + +fn matches_layer_field(kind: u16, tag: u16) -> bool { + tag == RTAG_LAYER || tag == layer_type_tag(kind) +} + fn allowed_field(kind: u16, tag: u16) -> bool { if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) { return true; @@ -276,6 +484,7 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result(records: &[Record<'a>], issues: &mut Vec) -> Result p.len() == 1, - RTAG_AREF => p.len() == 3, - RTAG_PATH => p.len() >= 2, - RTAG_NODE => !p.is_empty(), - RTAG_BOX => p.len() == 5, - _ => true, - }; - if !valid_count { - return Err(error(first.offset, "invalid coordinate count")); - } - object.bounds = Some([ - p.iter().map(|p| p[0]).min().unwrap(), - p.iter().map(|p| p[1]).min().unwrap(), - p.iter().map(|p| p[0]).max().unwrap(), - p.iter().map(|p| p[1]).max().unwrap(), - ]); + object.bounds = Some(if first.tag == RTAG_AREF { + let counts = object + .field(RTAG_COLROW) + .ok_or_else(|| error(first.offset, "array has no COLROW"))?; + let placement = ArrayPlacement::decode(counts, xy) + .map_err(|(_, reason)| payload_error(first.offset, reason))?; + let [columns, rows] = placement.counts; + object.multiplicity = u64::from(columns) * u64::from(rows); + coordinate_bounds(placement.points) + } else { + let points = XY::points(xy); + let valid_count = match first.tag { + RTAG_TEXT | RTAG_SREF => points.len() == 1, + RTAG_PATH => points.len() >= 2, + RTAG_NODE => points.len() > 0, + RTAG_BOX => points.len() == 5, + _ => true, + }; + if !valid_count { + return Err(error(first.offset, "invalid coordinate count")); + } + coordinate_bounds(points) + }); if matches!(first.tag, RTAG_SREF | RTAG_AREF) { - object.target = Some(decoded_string( - object - .field(RTAG_SNAME) - .ok_or_else(|| error(first.offset, "reference has no SNAME"))?, - first.offset, - )?); - if first.tag == RTAG_AREF { - let counts = object - .field(RTAG_COLROW) - .ok_or_else(|| error(first.offset, "array has no COLROW"))?; - let a = i16::from_be_bytes(counts[..2].try_into().unwrap()); - let b = i16::from_be_bytes(counts[2..].try_into().unwrap()); - if a <= 0 || b <= 0 { - return Err(error(first.offset, "array counts must be positive")); - } - object.multiplicity = a as u64 * b as u64; + if object.target.is_none() { + return Err(error(first.offset, "reference has no SNAME")); } } else { - let dtype = match first.tag { - RTAG_TEXT => RTAG_TEXTTYPE, - RTAG_BOX => RTAG_BOXTYPE, - RTAG_NODE => RTAG_NODETYPE, - _ => RTAG_DATATYPE, - }; + let dtype = layer_type_tag(first.tag); object.layer = Some(( integer( object @@ -404,29 +601,45 @@ fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result Result>, String> { + read_records_with(input, |_, _| {}) +} + +fn read_records_with<'a>( + input: &'a [u8], + mut observe: impl FnMut(usize, &Record<'a>), +) -> Result>, String> { + if input.starts_with(b"\x1f\x8b") { + return Err(error( + 0, + "gzip-compressed input. Decompress it before reading GDSII", + )); + } + if input.starts_with(b"%SEMI-OASIS\r\n") { + return Err(error(0, "OASIS input. Export the layout as GDSII")); + } + let mut records = Vec::new(); - let mut offset = 0; - while offset < input.len() { - let (payload, header) = RecordHeader::read(&input[offset..]) - .map_err(|(_, e)| error(offset, format!("invalid record header: {e:?}")))?; - let size = header.data_size as usize; - if (size + 4) % 2 != 0 { - return Err(error(offset, "odd record length")); - } - let data = payload - .get(..size) - .ok_or_else(|| error(offset, "truncated record"))?; + let mut remaining = input; + while !remaining.is_empty() { + let offset = input.len() - remaining.len(); + let (next, frame) = RecordFrame::read(remaining).map_err(|(_, reason)| match reason { + ErrType::Incomplete(_) => error(offset, "truncated record header"), + ErrType::Failed(message) => error(offset, message), + })?; let record = Record { offset, - tag: header.tag, - data, + tag: frame.tag, + data: frame.data, }; - offset += size + 4; - let ended = record.tag == RTAG_ENDLIB; + observe(records.len(), &record); records.push(record); - if ended { - if input[offset..].iter().any(|b| *b != 0) { - return Err(error(offset, "nonzero data after ENDLIB")); + remaining = next; + if frame.tag == RTAG_ENDLIB { + if remaining.iter().any(|b| *b != 0) { + return Err(error( + input.len() - remaining.len(), + "nonzero data after ENDLIB", + )); } break; } @@ -437,7 +650,16 @@ fn read_records(input: &[u8]) -> Result>, String> { impl<'a> Document<'a> { pub fn parse(input: &'a [u8]) -> Result { let records = read_records(input)?; - for record in &records { + let mut doc = Self::parse_records(&records, None)?; + doc.records = records; + Ok(doc) + } + + fn parse_records( + records: &[Record<'a>], + reuse: Option>, + ) -> Result { + for record in records { validate_payload(record)?; } if records.first().map(|r| r.tag) != Some(RTAG_HEADER) @@ -447,10 +669,9 @@ impl<'a> Document<'a> { return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string()); } let mut doc = Self { - records, + records: Vec::new(), cells: BTreeMap::new(), layer_names: BTreeMap::new(), - issues: Vec::new(), diagnostics: Vec::new(), header: Vec::new(), global_issues: Vec::new(), @@ -461,9 +682,9 @@ impl<'a> Document<'a> { let mut i = 2; let mut global_seen = BTreeSet::new(); let mut cells_started = false; - doc.header.push((RTAG_HEADER, doc.records[0].data)); - while i + 1 < doc.records.len() { - let record = &doc.records[i]; + doc.header.push((RTAG_HEADER, records[0].data)); + while i + 1 < records.len() { + let record = &records[i]; if record.tag != RTAG_BGNSTR { if cells_started { return Err(error( @@ -497,8 +718,12 @@ impl<'a> Document<'a> { } doc.header.push(( record.tag, - if record.tag & 0xff == 6 { - unpad(record.data) + if record.tag == RTAG_LIBNAME { + decode_name(record.data) + .map_err(|message| error(record.offset, message))? + .as_bytes() + } else if record.tag & 0xff == 6 { + unpad_ascii(record.data) } else { record.data }, @@ -512,15 +737,25 @@ impl<'a> Document<'a> { cells_started = true; let offset = record.offset; i += 1; - let name_record = doc - .records + let name_record = records .get(i) .filter(|r| r.tag == RTAG_STRNAME) .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; - let name = string(name_record.data, name_record.offset)?; + let name = decode_name(name_record.data) + .map_err(|message| error(name_record.offset, message))? + .to_owned(); if name.is_empty() { return Err(error(name_record.offset, "empty cell name")); } + if let Some(reuse) = &reuse + && let Some((cell, end)) = reuse.cell(&name, offset, records) + { + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + i = end + 1; + continue; + } i += 1; let mut cell = Cell { offset, @@ -530,8 +765,7 @@ impl<'a> Document<'a> { issues: Vec::new(), }; loop { - let record = doc - .records + let record = records .get(i) .ok_or_else(|| error(offset, "missing ENDSTR"))?; if record.tag == RTAG_ENDSTR { @@ -542,19 +776,19 @@ impl<'a> Document<'a> { if is_element(record.tag) { let start = i; i += 1; - while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { - if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) - || is_element(doc.records[i].tag) + while records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { + if matches!(records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) + || is_element(records[i].tag) { - return Err(error(doc.records[i].offset, "missing ENDEL")); + return Err(error(records[i].offset, "missing ENDEL")); } i += 1; } - if i == doc.records.len() { + if i == records.len() { return Err(error(offset, "missing ENDEL")); } let mut issues = Vec::new(); - let mut parsed = object(&doc.records[start..=i], &mut issues)?; + let mut parsed = object(&records[start..=i], &mut issues)?; parsed.issues = issues; cell.objects.push(parsed); i += 1; @@ -592,64 +826,10 @@ impl<'a> Document<'a> { doc.header.sort(); doc.read_metadata()?; doc.known_cells = doc.cells.keys().cloned().collect(); - doc.refresh_issues(); + doc.refresh_diagnostics(); Ok(doc) } - fn read_metadata(&mut self) -> Result<(), String> { - for (cell_name, cell) in &mut self.cells { - for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { - let text = object.field(RTAG_STRING).unwrap(); - if text.starts_with(b"KLAMATH_LAYER_NAMES_") && text != LAYER_SENTINEL.as_bytes() { - // Layer names apply across the library. An unknown version - // may describe layers used outside this carrier or cell, so - // geometry scope selection cannot establish its relevance. - let mut diagnostic = Diagnostic::at( - "unsupported_metadata_version", - object.offset, - "unsupported layer-name metadata version".to_string(), - ); - diagnostic.cell = Some(cell_name.clone()); - diagnostic.layer = object.layer; - diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); - self.global_issues.push(diagnostic); - } - } - let carriers: BTreeSet<_> = cell - .objects - .iter() - .filter(|o| { - o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) - }) - .filter_map(|o| o.layer) - .collect(); - for object in &mut cell.objects { - if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) - { - continue; - } - let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; - if text != LAYER_SENTINEL { - let ((layer, dtype), name) = - parse_layer_name(&text).map_err(|message| error(object.offset, message))?; - if self - .layer_names - .get(&(layer, dtype)) - .is_some_and(|old| old != &name) - { - return Err(error( - object.offset, - format!("conflicting names for {layer}/{dtype}"), - )); - } - self.layer_names.insert((layer, dtype), name.to_string()); - } - object.metadata = true; - } - } - Ok(()) - } - fn check_graph(&mut self) { let mut unresolved = BTreeMap::new(); for (parent, cell) in &self.cells { @@ -664,6 +844,7 @@ impl<'a> Document<'a> { for ((parent, target), offset) in unresolved { self.diagnostics.push(Diagnostic { code: "unresolved_reference", + side: None, message: format!("cell {parent:?}: unresolved reference {target:?}"), offset: Some(offset), cell: Some(parent.clone()), @@ -684,6 +865,7 @@ impl<'a> Document<'a> { Some(1) => { self.diagnostics.push(Diagnostic { code: "hierarchy_cycle", + side: None, message: format!("cyclic hierarchy involving {name:?}"), offset: self.cells.get(name).map(|c| c.offset), cell: Some(name.to_string()), @@ -708,42 +890,119 @@ impl<'a> Document<'a> { } pub fn edges(&self) -> BTreeMap<(String, String), u64> { - let mut edges = BTreeMap::new(); - for (name, cell) in &self.cells { - for object in &cell.objects { - if let Some(target) = &object.target { - let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64); - *count = count.saturating_add(object.multiplicity); + self.cells + .iter() + .flat_map(|(name, cell)| { + let mut counts = BTreeMap::new(); + for object in &cell.objects { + if let Some(target) = object.target.as_deref() { + let count = counts.entry(target).or_insert(0_u64); + *count = count.saturating_add(object.multiplicity); + } } - } - } - edges + counts + .into_iter() + .map(|(target, count)| ((name.clone(), target.to_owned()), count)) + }) + .collect() + } + + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + let revision = Revision::from_document(self); + hierarchy::render_hierarchy(&revision, &revision, None, mermaid, max_depth) } pub fn canonical_dump(&self) -> String { - let mut out = format!( - "profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n", - self.scope.description() + let mut out = TextReport::default(); + self.write_canonical_dump(&mut out); + out.into_string() + } + + /// Append the canonical object dump and its presentation roles. + pub fn write_canonical_dump(&self, out: &mut TextReport) { + if !self.present { + out.write_styled(TextStyle::Heading, format_args!("library absent")); + out.push('\n'); + return; + } + let version_index = self + .header + .iter() + .position(|(tag, _)| *tag == RTAG_HEADER) + .unwrap(); + let header = |tag| self.header.iter().find(|(t, _)| *t == tag).unwrap().1; + let (user, meters) = header(RTAG_UNITS).split_at(8); + let version = i16::from_be_bytes(self.header[version_index].1.try_into().unwrap()); + out.write_styled( + TextStyle::Heading, + format_args!( + "library {} (GDSII {version}, {} user units/dbu, {} m/dbu)", + quoted(header(RTAG_LIBNAME)), + format_exact_value(RTAG_UNITS, user), + format_exact_value(RTAG_UNITS, meters) + ), ); - for (tag, data) in &self.header { - writeln!(out, "{}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + out.push('\n'); + self.scope.write_restrictions(out); + for (index, (tag, data)) in self.header.iter().enumerate() { + if index != version_index && !matches!(*tag, RTAG_LIBNAME | RTAG_UNITS) { + writeln!(out, "{}={}", tag_name(*tag), format_exact_value(*tag, data)).unwrap(); + } } for ((layer, dtype), name) in &self.layer_names { - writeln!(out, "layer {layer}/{dtype} name={name:?}").unwrap(); + out.write_styled(TextStyle::Layer, format_args!("{layer}/{dtype}")); + writeln!(out, ": name {name:?}").unwrap(); } for (name, cell) in &self.cells { - writeln!(out, "cell {name:?}").unwrap(); + out.write_styled(TextStyle::Heading, format_args!("cell {name:?}")); + out.push('\n'); for (tag, data) in &cell.attributes { - writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + writeln!( + out, + " {}={}", + tag_name(*tag), + format_exact_value(*tag, data) + ) + .unwrap(); } - for object in cell.objects.iter().filter(|o| !o.metadata) { - writeln!(out, " {}", object.description()).unwrap(); + let mut objects = cell + .objects + .iter() + .filter(|object| !object.metadata) + .peekable(); + while let Some(object) = objects.next() { + let mut count = 1; + while objects.next_if(|next| next.key == object.key).is_some() { + count += 1; + } + out.push_str(" "); + if count != 1 { + write!(out, "{count}x ").unwrap(); + } + object + .format_with_heading(out, |out, name| { + if let Some((layer, datatype)) = object.layer { + out.write_styled(TextStyle::Layer, format_args!("{layer}/{datatype}")); + out.push_str(": "); + } + if matches!(object.kind, RTAG_SREF | RTAG_AREF) && object.layer.is_none() { + out.write_styled(TextStyle::Layer, format_args!("{name}")); + } else { + out.push_str(name); + } + Ok(()) + }) + .unwrap(); + out.push('\n'); } } - for issue in &self.issues { - writeln!(out, "INCOMPLETE: {issue}").unwrap(); + for diagnostic in &self.diagnostics { + out.write_styled( + TextStyle::Warning, + format_args!("INCOMPLETE: {}", diagnostic.message), + ); + out.push('\n'); } - out } pub fn record_dump(&self) -> String { @@ -753,23 +1012,35 @@ impl<'a> Document<'a> { /// Inspect record boundaries even when object grammar or metadata is invalid. pub fn dump_records(input: &[u8]) -> Result { - Ok(format_records(&read_records(input)?)) + let mut out = TextReport::default(); + write_record_dump(&mut out, input)?; + Ok(out.into_string()) +} + +/// Append the raw record dump and its presentation roles. +pub fn write_record_dump(out: &mut TextReport, input: &[u8]) -> Result<(), String> { + write_records(out, &read_records(input)?); + Ok(()) } fn format_records(records: &[Record<'_>]) -> String { - let mut out = String::new(); + let mut out = TextReport::default(); + write_records(&mut out, records); + out.into_string() +} + +fn write_records(out: &mut TextReport, records: &[Record<'_>]) { for record in records { + write!(out, "{:010} {:04x} ", record.offset, record.tag).unwrap(); + out.write_styled(TextStyle::Layer, format_args!("{}", tag_name(record.tag))); writeln!( out, - "{:010} {:04x} {} size={} {}", - record.offset, - record.tag, - tag_name(record.tag), + " size={} {}", record.data.len() + 4, format_data( record.tag, if record.tag & 0xff == 6 { - unpad(record.data) + unpad_ascii(record.data) } else { record.data } @@ -777,7 +1048,6 @@ fn format_records(records: &[Record<'_>]) -> String { ) .unwrap(); } - out } /// A retained comparison field. Polygon coordinates and string padding follow @@ -809,11 +1079,37 @@ pub struct ObjectChange { pub origin: Option<[i32; 2]>, pub width: Option, pub counts: Option<[u16; 2]>, - pub description: String, pub fields: Vec, pub properties: Vec<(u16, Vec)>, } +impl fmt::Display for ObjectChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +impl ObjectChange { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { + format_object( + out, + self.kind, + self.layer, + self.fields + .iter() + .map(|field| (field.tag, field.data.as_slice())), + self.properties + .iter() + .map(|(attribute, data)| (*attribute, data.as_slice())), + heading, + ) + } +} + impl From<&Object<'_>> for ObjectChange { fn from(object: &Object<'_>) -> Self { Self { @@ -823,24 +1119,19 @@ impl From<&Object<'_>> for ObjectChange { layer: object.layer, bounds: object.bounds, target: object.target.clone(), - origin: object.field(RTAG_XY).map(|data| { - [ - i32::from_be_bytes(data[..4].try_into().unwrap()), - i32::from_be_bytes(data[4..8].try_into().unwrap()), - ] - }), + origin: object + .field(RTAG_XY) + .and_then(|data| XY::points(data).next()), width: object .field(RTAG_WIDTH) .map(|data| i32::from_be_bytes(data.try_into().unwrap())), - counts: object.target.as_ref().map(|_| { + counts: object.target.as_ref().and_then(|_| { if object.kind == RTAG_AREF { - let data = object.field(RTAG_COLROW).unwrap(); - [integer(&data[..2]), integer(&data[2..])] + object.field(RTAG_COLROW).and_then(array_dimensions) } else { - [1, 1] + Some([1, 1]) } }), - description: object.description(), fields: owned_fields( object .key @@ -858,19 +1149,39 @@ impl From<&Object<'_>> for ObjectChange { } } +/// An existing cell's source location and retained attributes. +#[derive(Debug)] +pub struct CellRevision { + pub offset: usize, + pub attributes: Vec, +} + +impl From<&Cell<'_>> for CellRevision { + fn from(cell: &Cell<'_>) -> Self { + Self { + offset: cell.offset, + attributes: owned_fields(cell.attributes.iter().copied()), + } + } +} + #[derive(Debug)] pub struct CellChange { pub name: String, - pub old_offset: Option, - pub new_offset: Option, - pub attributes_changed: bool, - pub old_attributes: Vec, - pub new_attributes: Vec, + pub old: Option, + pub new: Option, pub removed: Vec, pub added: Vec, pub replacements: Vec, } +impl CellChange { + pub fn attributes_changed(&self) -> bool { + self.old.as_ref().map(|cell| &cell.attributes) + != self.new.as_ref().map(|cell| &cell.attributes) + } +} + #[derive(Debug)] pub struct LayerNameChange { pub layer: Layer, @@ -878,59 +1189,167 @@ pub struct LayerNameChange { pub new: Option, } +/// Library presence or exact record payloads, formatted only when rendered. #[derive(Debug)] -pub struct LibraryFieldChange { - pub field: String, - pub tag: Option, - pub old_fields: Vec, - pub new_fields: Vec, - pub old: Vec, - pub new: Vec, +pub enum LibraryChange { + Presence { + old: bool, + new: bool, + }, + Record { + tag: u16, + old: Vec>, + new: Vec>, + }, +} + +impl fmt::Display for LibraryChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Presence { old, new } => out.write_str(match (old, new) { + (false, true) => "added", + (true, false) => "removed", + (true, true) => "present", + (false, false) => "absent", + }), + Self::Record { tag, old, new } => { + let values = if old.len() <= 1 && new.len() <= 1 { + format_field_change( + *tag, + old.first().map(Vec::as_slice), + new.first().map(Vec::as_slice), + ) + } else { + let list = |items: &[Vec]| { + if items.is_empty() { + return "absent".to_string(); + } + let values = items + .iter() + .map(|data| format_exact_value(*tag, data)) + .collect::>() + .join(", "); + format!("[{values}]") + }; + format!("{} -> {}", list(old), list(new)) + }; + write!(out, "{} {values}", tag_name(*tag)) + } + } + } +} + +/// Owned revision metadata and reference counts captured before documents are dropped. +#[derive(Debug)] +pub struct Revision { + pub scope: Scope, + pub present: bool, + pub units: Option>, + cells: BTreeSet, + known_cells: BTreeSet, + edges: BTreeMap<(String, String), u64>, +} + +impl Revision { + fn from_document(doc: &Document<'_>) -> Self { + Self { + scope: doc.scope.clone(), + present: doc.present, + units: doc + .header + .iter() + .find(|(tag, _)| *tag == RTAG_UNITS && doc.present) + .map(|(_, data)| data.to_vec()), + cells: doc.cells.keys().cloned().collect(), + known_cells: doc.known_cells.clone(), + edges: doc.edges(), + } + } } #[derive(Debug)] pub struct Comparison { - pub scope: Scope, - /// Unchanged names are displayed once when objects on that layer change. + pub old: Revision, + pub new: Revision, + /// Unchanged layer names available to the text report. pub layer_legend: BTreeMap, - pub library_fields: Vec, + pub library_changes: Vec, pub layer_names: Vec, pub cells: Vec, - pub issues: Vec, + pub diagnostics: Vec, +} + +/// A comparison for text and hierarchy reports. +#[derive(Debug)] +pub struct TextComparison { + comparison: Comparison, +} + +impl TextComparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new()) + } + + fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self { + Self { + comparison: Comparison::collect(old, new, reused_cells, false), + } + } + + pub fn outcome(&self) -> Outcome { + self.comparison.outcome() + } + + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + self.comparison.hierarchy(mermaid, max_depth) + } } impl Comparison { pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new(), true) + } + + fn collect( + old: &Document<'_>, + new: &Document<'_>, + reused_cells: &BTreeSet<&str>, + include_whole_cell_objects: bool, + ) -> Self { let mut result = Self { - scope: old.scope.clone(), + old: Revision::from_document(old), + new: Revision::from_document(new), layer_legend: old .layer_names .iter() .filter(|(pair, name)| new.layer_names.get(pair) == Some(*name)) .map(|(pair, name)| (*pair, name.clone())) .collect(), - library_fields: Vec::new(), + library_changes: Vec::new(), layer_names: Vec::new(), cells: Vec::new(), - issues: Vec::new(), + diagnostics: Vec::new(), }; - if old.scope != new.scope { - result.issues.push("comparison scopes differ".to_string()); - } if old.present != new.present { - result.library_fields.push(LibraryFieldChange { - field: "presence".to_string(), - tag: None, - old_fields: Vec::new(), - new_fields: Vec::new(), - old: vec![if old.present { "present" } else { "absent" }.to_string()], - new: vec![if new.present { "present" } else { "absent" }.to_string()], + result.library_changes.push(LibraryChange::Presence { + old: old.present, + new: new.present, }); } - for (side, doc) in [("old", old), ("new", new)] { + for (side, doc) in [(Side::Old, old), (Side::New, new)] { result - .issues - .extend(doc.issues.iter().map(|i| format!("{side}: {i}"))); + .diagnostics + .extend(doc.diagnostics.iter().cloned().map(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic + })); + } + if old.scope != new.scope { + result.diagnostics.push(Diagnostic::new( + "scope_mismatch", + "comparison scopes differ", + None, + )); } let header_tags: BTreeSet<_> = old .header @@ -952,13 +1371,10 @@ impl Comparison { .map(|(_, value)| *value) .collect(); if a != b { - result.library_fields.push(LibraryFieldChange { - field: tag_name(tag), - tag: Some(tag), - old_fields: owned_fields(a.iter().map(|data| (tag, *data))), - new_fields: owned_fields(b.iter().map(|data| (tag, *data))), - old: a.iter().map(|data| format_data(tag, data)).collect(), - new: b.iter().map(|data| format_data(tag, data)).collect(), + result.library_changes.push(LibraryChange::Record { + tag, + old: a.into_iter().map(<[u8]>::to_vec).collect(), + new: b.into_iter().map(<[u8]>::to_vec).collect(), }); } } @@ -979,23 +1395,36 @@ impl Comparison { } let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect(); for name in names { + if reused_cells.contains(name.as_str()) { + continue; + } let a = old.cells.get(name); let b = new.cells.get(name); let mut change = CellChange { name: name.clone(), - old_offset: a.map(|c| c.offset), - new_offset: b.map(|c| c.offset), - attributes_changed: a.map(|c| &c.attributes) != b.map(|c| &c.attributes), - old_attributes: owned_fields( - a.into_iter().flat_map(|c| c.attributes.iter().copied()), - ), - new_attributes: owned_fields( - b.into_iter().flat_map(|c| c.attributes.iter().copied()), - ), + old: a.map(CellRevision::from), + new: b.map(CellRevision::from), removed: Vec::new(), added: Vec::new(), replacements: Vec::new(), }; + if !include_whole_cell_objects && (a.is_none() || b.is_none()) { + let details = if a.is_some() { + &mut change.removed + } else { + &mut change.added + }; + // The hierarchy uses the changed reference targets. + details.extend( + a.or(b) + .into_iter() + .flat_map(|cell| &cell.objects) + .filter(|object| !object.metadata && object.target.is_some()) + .map(ObjectChange::from), + ); + result.cells.push(change); + continue; + } let a: Vec<_> = a .into_iter() .flat_map(|c| &c.objects) @@ -1028,11 +1457,7 @@ impl Comparison { } } } - if change.old_offset.is_none() - || change.new_offset.is_none() - || change.attributes_changed - || !change.removed.is_empty() - || !change.added.is_empty() + if change.attributes_changed() || !change.removed.is_empty() || !change.added.is_empty() { change.replacements = changes::replacements(&change.removed, &change.added); result.cells.push(change); @@ -1041,338 +1466,41 @@ impl Comparison { result } - pub fn is_equal(&self) -> bool { - self.issues.is_empty() - && self.library_fields.is_empty() - && self.layer_names.is_empty() - && self.cells.is_empty() - } - - pub fn exit_code(&self) -> u8 { - if !self.issues.is_empty() { - 2 - } else if self.is_equal() { - 0 + pub fn outcome(&self) -> Outcome { + let different = !self.library_changes.is_empty() + || !self.layer_names.is_empty() + || !self.cells.is_empty(); + if !self.diagnostics.is_empty() { + Outcome::Incomplete { different } + } else if different { + Outcome::Different } else { - 1 + Outcome::Equal } } - /// Limit presentation only: the complete comparison has already been made. - pub fn text(&self, max_objects: usize) -> String { - self.text_filtered(max_objects, &DisplayFilter::default()) - } - - pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { - let mut out = format!("profile {PROFILE}\n{}\n", self.scope.description()); - if show.cell.is_some() || show.layer.is_some() { - writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap(); - } - for change in &self.library_fields { - writeln!( - out, - "library {}: {:?} -> {:?}", - change.field, change.old, change.new - ) - .unwrap(); - } - if self.library_fields.iter().any(|c| c.field == "UNITS") { - out.push_str("UNITS changed; coordinates remain in each file's database units\n"); - } - for change in &self.layer_names { - writeln!( - out, - "layer {}/{} name: {:?} -> {:?}", - change.layer.0, change.layer.1, change.old, change.new - ) - .unwrap(); - } - let changed_layers: BTreeSet<_> = self - .cells - .iter() - .filter(|cell| show.cell.as_ref().is_none_or(|name| name == &cell.name)) - .flat_map(|cell| cell.removed.iter().chain(&cell.added)) - .filter_map(|object| object.layer) - .filter(|layer| show.layer.is_none_or(|selected| selected == *layer)) - .collect(); - for layer in changed_layers { - if let Some(name) = self.layer_legend.get(&layer) { - writeln!(out, "layer {}/{} name={name:?}", layer.0, layer.1).unwrap(); - } - } - let total: usize = self - .cells - .iter() - .map(|c| c.removed.len() + c.added.len()) - .sum(); - let mut shown: usize = 0; - for cell in &self.cells { - if show.cell.as_ref().is_some_and(|name| name != &cell.name) { - continue; - } - writeln!( - out, - "cell {:?} old@{:?} new@{:?}{}", - cell.name, - cell.old_offset, - cell.new_offset, - if cell.attributes_changed { - " [cell/header changed]" - } else { - "" - } - ) - .unwrap(); - let mut paired_old = BTreeSet::new(); - let mut paired_new = BTreeSet::new(); - for replacement in &cell.replacements { - let a = &cell.removed[replacement.removed]; - let b = &cell.added[replacement.added]; - if shown.saturating_add(2) <= max_objects - && show - .layer - .is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer)) - { - writeln!(out, " ~ {}", replacement.explanation).unwrap(); - writeln!( - out, - " - @{} bbox={:?} {}", - a.offset, a.bounds, a.description - ) - .unwrap(); - writeln!( - out, - " + @{} bbox={:?} {}", - b.offset, b.bounds, b.description - ) - .unwrap(); - paired_old.insert(replacement.removed); - paired_new.insert(replacement.added); - shown += 2; - } - } - for (sign, objects, paired) in [ - ("-", &cell.removed, &paired_old), - ("+", &cell.added, &paired_new), - ] { - for (index, object) in objects.iter().enumerate() { - if paired.contains(&index) { - continue; - } - if show.layer.is_some_and(|layer| object.layer != Some(layer)) { - continue; - } - if shown < max_objects { - writeln!( - out, - " {sign} @{} bbox={:?} {}", - object.offset, object.bounds, object.description - ) - .unwrap(); - shown += 1; - } - } - } - } - writeln!( - out, - "{} changed cells; {shown} of {total} object changes shown; {} layer-name changes", - self.cells.len(), - self.layer_names.len() - ) - .unwrap(); - for issue in &self.issues { - writeln!(out, "INCOMPLETE: {issue}").unwrap(); - } - if self.is_equal() { - out.push_str("equal under this profile\n"); - } - out + /// Render the affected hierarchy. ASCII output shows relationships between cells. + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + hierarchy::render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) } } -/// The graph joins both revisions, retaining removed branches and old callers. -pub fn hierarchy( - old: &Document<'_>, - new: &Document<'_>, - diff: Option<&Comparison>, - mermaid: bool, - max_depth: usize, -) -> String { - let old_edges = old.edges(); - let new_edges = new.edges(); - let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); - let direct: BTreeSet<_> = diff - .into_iter() - .flat_map(|d| d.cells.iter().map(|c| c.name.clone())) - .collect(); - let changed_refs: BTreeSet<_> = diff - .into_iter() - .flat_map(|d| &d.cells) - .flat_map(|c| { - c.removed - .iter() - .chain(&c.added) - .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) - }) - .collect(); - let mut parents = BTreeMap::<&str, Vec<&str>>::new(); - for (parent, child) in &edges { - parents.entry(child).or_default().push(parent); +fn cell_marker(old: bool, new: bool) -> char { + match (old, new) { + (true, false) => '-', + (false, true) => '+', + _ => '*', } - let mut affected = direct.clone(); - let mut pending: Vec<_> = direct.iter().cloned().collect(); - while let Some(child) = pending.pop() { - for parent in parents.get(child.as_str()).into_iter().flatten() { - if affected.insert((*parent).to_string()) { - pending.push((*parent).to_string()); - } - } - } - let visible: BTreeSet<_> = if diff.is_some() { - affected - .iter() - .cloned() - .chain(changed_refs.iter().map(|(_, t)| t.clone())) - .collect() +} + +fn cell_label(name: &str) -> Cow<'_, str> { + if name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'?' | b'.' | b'-')) + { + Cow::Borrowed(name) } else { - old.cells - .keys() - .chain(new.cells.keys()) - .cloned() - .chain(edges.iter().map(|(_, t)| t.clone())) - .collect() - }; - let mut children = BTreeMap::>::new(); - let mut targets = BTreeSet::new(); - for (parent, target) in &edges { - if visible.contains(parent) && visible.contains(target) { - children - .entry(parent.clone()) - .or_default() - .push(target.clone()); - targets.insert(target.clone()); - } - } - let roots: Vec<_> = visible.difference(&targets).cloned().collect(); - // Distinguish rootless cycles from descendants hidden by the depth limit. - let mut reachable = BTreeSet::new(); - let mut pending = roots.clone(); - while let Some(name) = pending.pop() { - if reachable.insert(name.clone()) { - pending.extend(children.get(&name).into_iter().flatten().cloned()); - } - } - let mut included = BTreeSet::new(); - let mut rows = Vec::new(); - let mut stack: Vec<_> = roots - .iter() - .rev() - .map(|n| (n.clone(), 0, None::)) - .collect(); - while let Some((name, depth, parent)) = stack.pop() { - let seen = !included.insert(name.clone()); - let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); - rows.push(( - name.clone(), - depth, - parent, - seen, - depth >= max_depth && !descendants.is_empty(), - )); - if !seen && depth < max_depth { - for child in descendants.iter().rev() { - stack.push((child.clone(), depth + 1, Some(name.clone()))); - } - } - } - for name in visible.difference(&reachable) { - rows.push((name.clone(), 0, None, false, true)); - } - let status = |name: &String| { - if !old.cells.contains_key(name) && !new.cells.contains_key(name) { - if old.known_cells.contains(name) || new.known_cells.contains(name) { - " [outside scope]" - } else { - " [unresolved]" - } - } else if direct.contains(name) { - " [cell changed]" - } else if affected.contains(name) { - " [affected]" - } else if diff.is_some() { - " [cell unchanged]" - } else { - "" - } - }; - let edge_label = |parent: &String, target: &String| { - let key = (parent.clone(), target.clone()); - let a = old_edges.get(&key).copied().unwrap_or(0); - let b = new_edges.get(&key).copied().unwrap_or(0); - let count = if a == b { - format!("x{b}") - } else { - format!("x{a} -> x{b}") - }; - format!( - "{count}{}", - if changed_refs.contains(&key) { - " [reference changed]" - } else { - "" - } - ) - }; - if mermaid { - let ids: BTreeMap<_, _> = rows - .iter() - .map(|r| &r.0) - .collect::>() - .into_iter() - .enumerate() - .map(|(i, n)| (n, i)) - .collect(); - let mut out = format!("flowchart TD\n %% {}\n", new.scope.description()); - for (name, id) in &ids { - let label = format!("{name}{}", status(name)); - let label = label - .replace('&', "&") - .replace('"', """) - .replace('<', "<") - .replace('>', ">") - .replace(['\r', '\n'], " "); - writeln!(out, " n{id}[\"{label}\"]").unwrap(); - } - for (parent, target) in &edges { - if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { - writeln!(out, " n{a} -->|\"{}\"| n{b}", edge_label(parent, target)).unwrap(); - } - } - if rows.is_empty() { - out.push_str(" empty[\"No changed hierarchy\"]\n"); - } - out - } else { - let mut out = format!("{}\n", new.scope.description()); - for (name, depth, parent, seen, limited) in rows { - let count = parent - .map(|p| edge_label(&p, &name)) - .unwrap_or_else(|| "x1".to_string()); - writeln!( - out, - "{}{name:?} {count}{}{}{}", - " ".repeat(depth), - status(&name), - if seen { " [shown above]" } else { "" }, - if limited { " [children omitted]" } else { "" } - ) - .unwrap(); - } - if visible.is_empty() { - out.push_str("No changed hierarchy\n"); - } - out + Cow::Owned(format!("{name:?}")) } } @@ -1387,6 +1515,57 @@ fn quoted(data: &[u8]) -> String { out } +fn real8_words(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .copied() + .map(u64::from_be_bytes) +} + +fn format_real8(data: &[u8], show_bytes: bool) -> String { + let mut values = real8_words(data).map(decode_real8); + let mut out = if values.len() == 1 { + format!("{:?}", values.next().unwrap()) + } else { + format!("{:?}", values.collect::>()) + }; + if show_bytes { + write!(out, " raw={}", hex(data)).unwrap(); + } + out +} + +fn is_real8(tag: u16, data: &[u8]) -> bool { + tag & 0xff == 5 && data.len().is_multiple_of(8) +} + +fn format_exact_value(tag: u16, data: &[u8]) -> String { + if is_real8(tag, data) { + let show_bytes = + real8_words(data).any(|bits| encode_real8(decode_real8(bits)).ok() != Some(bits)); + format_real8(data, show_bytes) + } else { + format_data(tag, data) + } +} + +fn format_field_change(tag: u16, before: Option<&[u8]>, after: Option<&[u8]>) -> String { + let show_bytes = before.zip(after).is_some_and(|(a, b)| { + is_real8(tag, a) + && is_real8(tag, b) + && real8_words(a) + .zip(real8_words(b)) + .any(|(a, b)| a != b && decode_real8(a).to_bits() == decode_real8(b).to_bits()) + }); + let format_value = |data: Option<&[u8]>| match data { + Some(data) if is_real8(tag, data) => format_real8(data, show_bytes), + Some(data) => format_data(tag, data), + None => "absent".to_string(), + }; + format!("{} -> {}", format_value(before), format_value(after)) +} + fn format_data(tag: u16, data: &[u8]) -> String { if matches!( tag, @@ -1397,23 +1576,27 @@ fn format_data(tag: u16, data: &[u8]) -> String { } match tag & 0xff { 0 if data.is_empty() => String::new(), - 2 if data.len() % 2 == 0 => format!( + 2 if data.len().is_multiple_of(2) => format!( "{:?}", - data.chunks_exact(2) - .map(|b| i16::from_be_bytes(b.try_into().unwrap())) + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) .collect::>() ), - 3 if data.len() % 4 == 0 => format!( + 3 if data.len().is_multiple_of(4) => format!( "{:?}", - data.chunks_exact(4) - .map(|b| i32::from_be_bytes(b.try_into().unwrap())) + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) .collect::>() ), - 5 if data.len() % 8 == 0 => format!( + 5 if data.len().is_multiple_of(8) => format!( "{:?} raw={}", - data.chunks_exact(8) - .map(|b| decode_real8(u64::from_be_bytes(b.try_into().unwrap()))) - .collect::>(), + real8_words(data).map(decode_real8).collect::>(), hex(data) ), 6 => quoted(data), @@ -1421,6 +1604,20 @@ fn format_data(tag: u16, data: &[u8]) -> String { } } +fn object_name(kind: u16, rectangular: bool) -> &'static str { + match kind { + RTAG_BOUNDARY if rectangular => "rectangle", + RTAG_BOUNDARY => "polygon", + RTAG_BOX => "box", + RTAG_PATH => "path", + RTAG_SREF => "reference", + RTAG_AREF => "array reference", + RTAG_TEXT => "label", + RTAG_NODE => "node", + _ => kind_name(kind), + } +} + fn hex(data: &[u8]) -> String { let mut out = "0x".to_string(); for byte in data { diff --git a/src/review/changes.rs b/src/review/changes.rs index 22e202a..157a349 100644 --- a/src/review/changes.rs +++ b/src/review/changes.rs @@ -1,10 +1,11 @@ -use super::*; +//! Unique replacement matches that explain removed and added objects. + +use super::{text::CellText, *}; #[derive(Debug)] pub struct Replacement { pub removed: usize, pub added: usize, - pub explanation: String, /// Explanation matching is never object identity or geometric equivalence. pub match_rule: &'static str, } @@ -24,49 +25,95 @@ fn group(object: &ObjectChange, bounds: bool) -> Group { ) } +fn unique_groups<'a, K: Ord>( + objects: &'a [ObjectChange], + key: impl Fn(usize, &'a ObjectChange) -> Option, +) -> BTreeMap> { + let mut groups = BTreeMap::new(); + for (index, object) in objects.iter().enumerate() { + if let Some(key) = key(index, object) { + groups + .entry(key) + .and_modify(|value| *value = None) + .or_insert(Some(index)); + } + } + groups +} + +type ShapeKey<'a> = (u16, Vec<(u16, &'a [u8])>, &'a [(u16, Vec)]); + +fn shape_without_layer(object: &ObjectChange) -> Option> { + if object.layer.is_none() || !(is_polygon(object.kind) || object.kind == RTAG_PATH) { + return None; + } + let fields = object + .fields + .iter() + .filter(|field| !matches_layer_field(object.kind, field.tag)) + .map(|field| (field.tag, field.data.as_slice())) + .collect(); + Some((object.kind, fields, &object.properties)) +} + +fn layer_replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + // Most small edits touch a single numeric pair. + let Some(layer) = old.iter().chain(new).find_map(|object| object.layer) else { + return Vec::new(); + }; + if !old + .iter() + .chain(new) + .any(|object| object.layer.is_some_and(|pair| pair != layer)) + { + return Vec::new(); + } + let a = unique_groups(old, |_, object| shape_without_layer(object)); + let b = unique_groups(new, |_, object| shape_without_layer(object)); + a.into_iter() + .filter_map(|(key, removed)| { + let removed = removed?; + let added = b.get(&key).copied().flatten()?; + (old[removed].layer != new[added].layer).then_some(Replacement { + removed, + added, + match_rule: "unique_layer_change", + }) + }) + .collect() +} + /// Pair only unique candidates in the remaining multisets. Ambiguous groups /// remain additions/removals; no identity or geometric equivalence is inferred. pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { - let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new()); - let mut result = Vec::new(); + if old.is_empty() || new.is_empty() { + return Vec::new(); + } + let mut result = layer_replacements(old, new); + let (mut used_old, mut used_new): (BTreeSet<_>, BTreeSet<_>) = + result.iter().map(|pair| (pair.removed, pair.added)).unzip(); for bounds in [true, false] { - let (mut a, mut b) = ( - BTreeMap::>::new(), - BTreeMap::>::new(), - ); - for (i, object) in old - .iter() - .enumerate() - .filter(|(i, _)| !used_old.contains(i)) - { - a.entry(group(object, bounds)).or_default().push(i); - } - for (i, object) in new - .iter() - .enumerate() - .filter(|(i, _)| !used_new.contains(i)) - { - b.entry(group(object, bounds)).or_default().push(i); - } - for (key, indices) in a { - if let (1, Some(others)) = (indices.len(), b.get(&key)) { - if others.len() == 1 { - let (i, j) = (indices[0], others[0]); - result.push(Replacement { - removed: i, - added: j, - explanation: explain(&old[i], &new[j]), - match_rule: if old[i].target.is_some() { - "unique_kind_target" - } else if bounds { - "unique_kind_layer_bounds" - } else { - "unique_kind_layer" - }, - }); - used_old.insert(i); - used_new.insert(j); - } + let a = unique_groups(old, |i, object| { + (!used_old.contains(&i)).then(|| group(object, bounds)) + }); + let b = unique_groups(new, |i, object| { + (!used_new.contains(&i)).then(|| group(object, bounds)) + }); + for (key, index) in a { + if let (Some(i), Some(&Some(j))) = (index, b.get(&key)) { + result.push(Replacement { + removed: i, + added: j, + match_rule: if old[i].target.is_some() { + "unique_kind_target" + } else if bounds { + "unique_kind_layer_bounds" + } else { + "unique_kind_layer" + }, + }); + used_old.insert(i); + used_new.insert(j); } } } @@ -74,53 +121,542 @@ pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec String { - let mut parts = vec![format!( - "{} modified{}", - kind_name(old.kind), - old.target - .as_ref() - .map(|t| format!(" target={t:?}")) - .unwrap_or_default() - )]; - if old.target.is_some() && old.origin != new.origin { - if let (Some(a), Some(b)) = (old.origin, new.origin) { - parts.push(format!( - "origin {a:?} -> {b:?}, delta=({}, {}) dbu", - i64::from(b[0]) - i64::from(a[0]), - i64::from(b[1]) - i64::from(a[1]) - )); +pub(super) fn field_changes<'a>( + old: &'a ObjectChange, + new: &'a ObjectChange, +) -> impl Iterator, Option<&'a [u8]>)> { + let mut fields = BTreeMap::, Option<&[u8]>)>::new(); + for field in &old.fields { + fields.entry(field.tag).or_default().0 = Some(&field.data); + } + for field in &new.fields { + fields.entry(field.tag).or_default().1 = Some(&field.data); + } + fields + .into_iter() + .filter_map(|(tag, (before, after))| (before != after).then_some((tag, before, after))) +} + +fn object_heading(object: &ObjectChange, kind: &str, with_layer: bool) -> String { + match object.layer.filter(|_| with_layer) { + Some((layer, datatype)) => format!("{layer}/{datatype}: {kind}"), + None => kind.to_string(), + } +} + +fn object_xy(object: &ObjectChange) -> &[u8] { + object + .fields + .iter() + .find(|field| field.tag == RTAG_XY) + .map_or(&[], |field| field.data.as_slice()) +} + +fn write_cell_heading(out: &mut CellText<'_>, layer: Option, name: &str) { + if let Some(layer) = layer { + out.write_layer(layer, None); + out.push_str(": "); + } + out.push_str(name); +} + +pub(super) fn write_unmatched(out: &mut CellText<'_>, object: &ObjectChange, details: bool) { + if !details + && object.kind == RTAG_PATH + && let Some([xmin, ymin, xmax, ymax]) = object.bounds + { + let vertices = XY::points(object_xy(object)).len(); + if vertices > POINTS_PER_LINE { + write_cell_heading(out, object.layer, "path"); + let width = object + .width + .map_or_else(|| "absent".to_string(), |width| format!("{width} dbu")); + write!( + out, + " · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" + ) + .unwrap(); + return; } } - if old.counts != new.counts { - let counts = |c: Option<[u16; 2]>| { - c.map(|[a, b]| format!("{a} x {b}")) - .unwrap_or_else(|| "absent".to_string()) + object + .format_with_heading(out, |out, name| { + write_cell_heading(out, object.layer, name); + Ok(()) + }) + .unwrap(); +} + +fn vertex_count(xy: &[u8]) -> usize { + XY::points(xy).len().saturating_sub(1) +} + +fn rectangle_size(xy: &[u8]) -> Option<[i64; 2]> { + let [xmin, ymin, xmax, ymax] = XY::closed_rectangle(xy)?; + Some([ + i64::from(xmax) - i64::from(xmin), + i64::from(ymax) - i64::from(ymin), + ]) +} + +fn translation(before: &[u8], after: &[u8]) -> Option<[i64; 2]> { + let mut old = XY::points(before); + let mut new = XY::points(after); + if old.len() != new.len() { + return None; + } + let (Some(a), Some(b)) = (old.next(), new.next()) else { + return None; + }; + let displacement = |[x0, y0]: [i32; 2], [x1, y1]: [i32; 2]| { + [i64::from(x1) - i64::from(x0), i64::from(y1) - i64::from(y0)] + }; + let shift = displacement(a, b); + (shift != [0, 0] && old.zip(new).all(|(a, b)| displacement(a, b) == shift)).then_some(shift) +} + +#[derive(Default)] +struct ShapeCounts { + rectangles: usize, + polygons: usize, + boxes: usize, +} + +impl ShapeCounts { + fn total(&self) -> usize { + self.rectangles + self.polygons + self.boxes + } + + fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) { + let is_rectangle = + |object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some(); + match object.kind { + RTAG_BOX => self.boxes += 1, + _ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => { + self.rectangles += 1 + } + _ => self.polygons += 1, + } + } +} + +#[derive(Default)] +struct LayerPolygonChanges { + changed: ShapeCounts, + moved: ShapeCounts, + removed: ShapeCounts, + added: ShapeCounts, + displacement: Option<[i64; 2]>, +} + +#[derive(Default)] +pub(super) struct PolygonChanges(BTreeMap, LayerPolygonChanges>); + +impl PolygonChanges { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) { + let counts = self.0.entry(old.layer).or_default(); + let displacement = if old.properties == new.properties { + translation(object_xy(old), object_xy(new)) + .filter(|_| field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY)) + } else { + None + }; + if let Some(shift) = displacement { + counts.displacement = Some(shift); + counts.moved.add(old, Some(new)); + } else { + counts.changed.add(old, Some(new)); + } + } + + pub fn unmatched(&mut self, object: &ObjectChange, removed: bool) { + let counts = self.0.entry(object.layer).or_default(); + let shapes = if removed { + &mut counts.removed + } else { + &mut counts.added }; - parts.push(format!( - "columns/rows {} -> {}", - counts(old.counts), - counts(new.counts) - )); - } - if old.width != new.width { - let width = |n: Option| { - n.map(|n| n.to_string()) - .unwrap_or_else(|| "absent".to_string()) + shapes.add(object, None); + } + + pub fn write(&self, out: &mut CellText<'_>) { + for (layer, counts) in &self.0 { + out.push_str(" "); + out.style(Some(TextStyle::Layer)); + if let Some(layer) = layer { + out.write_layer(*layer, None); + } else { + out.push_str("unknown layer"); + } + out.style(None); + out.push_str(": "); + let groups = [ + ("changed", TextStyle::Changed, &counts.changed), + ("moved", TextStyle::Changed, &counts.moved), + ("removed", TextStyle::Removed, &counts.removed), + ("added", TextStyle::Added, &counts.added), + ]; + if let Some([dx, dy]) = counts.displacement + && groups + .iter() + .map(|(_, _, shapes)| shapes.total()) + .sum::() + == 1 + { + let shape = if counts.moved.rectangles == 1 { + "rectangle" + } else if counts.moved.boxes == 1 { + "box" + } else { + "polygon" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "{shape} moved by ({dx}, {dy}) dbu").unwrap(); + continue; + } + let mut separator = ""; + for (action, style, shapes) in groups { + let terms = [ + (shapes.rectangles, "rectangle", "rectangles"), + (shapes.polygons, "polygon", "polygons"), + (shapes.boxes, "box", "boxes"), + ] + .into_iter() + .filter(|(count, _, _)| *count != 0) + .map(|(count, singular, plural)| { + let shape = if count == 1 { singular } else { plural }; + format!("{count} {shape}") + }) + .collect::>(); + if !terms.is_empty() { + out.push_str(separator); + out.style(Some(style)); + write!(out, "{} {action}", terms.join(", ")).unwrap(); + out.style(None); + separator = " · "; + } + } + out.push('\n'); + } + } +} + +#[derive(Default)] +pub(super) struct ReferenceCounts { + singles: usize, + arrays: usize, + placements: u128, +} + +impl ReferenceCounts { + pub fn add(&mut self, object: &ObjectChange) { + if object.kind == RTAG_AREF { + self.arrays += 1; + } else { + self.singles += 1; + } + self.placements += reference_placements(object); + } + + pub fn write( + &self, + out: &mut CellText<'_>, + sign: char, + target: &str, + first: &ObjectChange, + ) -> usize { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if self.len() == 1 { + write!(out, "{first}").unwrap(); + } else { + self.write_counts(out, ""); + write!(out, " to {}", cell_label(target)).unwrap(); + self.write_placements(out); + } + let placements = out.reserve_placements(); + out.push('\n'); + placements + } + + fn len(&self) -> usize { + self.singles + self.arrays + } + + pub fn write_change(&self, out: &mut CellText<'_>, target: &str, added: &Self) -> usize { + write!(out, " {} · ", cell_label(target)).unwrap(); + out.style(Some(TextStyle::Removed)); + self.write_counts(out, "-"); + self.write_placements(out); + out.style(None); + out.push_str(", "); + out.style(Some(TextStyle::Added)); + added.write_counts(out, "+"); + added.write_placements(out); + out.style(None); + let placements = out.reserve_placements(); + out.push('\n'); + placements + } + + fn write_counts(&self, out: &mut CellText<'_>, sign: &str) { + if self.singles > 0 { + let suffix = if self.singles == 1 { "" } else { "s" }; + write!(out, "{sign}{} ref{suffix}", self.singles).unwrap(); + } + if self.arrays > 0 { + if self.singles > 0 { + out.push_str(", "); + } + let suffix = if self.arrays == 1 { "" } else { "s" }; + write!(out, "{sign}{} array ref{suffix}", self.arrays).unwrap(); + } + } + + fn write_placements(&self, out: &mut CellText<'_>) { + if self.arrays > 0 { + let suffix = if self.placements == 1 { "" } else { "s" }; + write!(out, " ({} placement{suffix})", self.placements).unwrap(); + } + } +} + +fn reference_placements(object: &ObjectChange) -> u128 { + match object.counts { + Some([columns, rows]) => u128::from(columns) * u128::from(rows), + None => 0, + } +} + +#[derive(Default)] +pub(super) struct ShownReferences { + shown: usize, + total: usize, + removed: u128, + added: u128, + placement_slot: usize, +} + +impl ShownReferences { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange, placement_slot: usize) { + self.shown += 2; + self.removed += reference_placements(old); + self.added += reference_placements(new); + self.placement_slot = placement_slot; + } + + pub fn unmatched( + &mut self, + removed: &ReferenceCounts, + added: &ReferenceCounts, + placement_slot: usize, + ) { + self.shown += removed.len() + added.len(); + self.removed += removed.placements; + self.added += added.placements; + self.placement_slot = placement_slot; + } + + pub fn individual( + &mut self, + object: &ObjectChange, + removed: bool, + count: usize, + placement_slot: usize, + ) { + let placements = if removed { + &mut self.removed + } else { + &mut self.added }; - parts.push(format!( - "width {} -> {} dbu", - width(old.width), - width(new.width) - )); - } - if old.bounds == new.bounds { - parts.push("coordinate bounds unchanged".to_string()); + *placements += reference_placements(object) * count as u128; + self.shown += count; + self.placement_slot = placement_slot; + } + + pub fn count_change(&mut self) { + self.total += 1; + } + + pub fn complete(&self) -> bool { + self.shown == self.total + } + + pub fn write_placements(&self, out: &mut CellText<'_>, old: u64, new: u64) { + if (u128::from(old), u128::from(new)) != (self.removed, self.added) { + out.set_placements(self.placement_slot, old, new); + } + } +} + +pub(super) fn write_layer_transition( + out: &mut CellText<'_>, + old: &ObjectChange, + new: &ObjectChange, +) { + out.write_layer(old.layer.unwrap(), Some(Side::Old)); + out.push_str(" → "); + out.write_layer(new.layer.unwrap(), Some(Side::New)); + write!(out, " · {}", transition_shape(old)).unwrap(); +} + +fn transition_shape(old: &ObjectChange) -> &'static str { + let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some(); + object_name(old.kind, rectangular) +} + +fn polygon_replacement(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let before = object_xy(old); + let after = object_xy(new); + let old_size = rectangle_size(before); + let new_size = rectangle_size(after); + let name = object_name(old.kind, old_size.is_some() && new_size.is_some()); + let mut out = object_heading(old, name, with_layer); + if before == after { + out.push_str(" modified"); } else { - parts.push(format!( - "coordinate bounds {:?} -> {:?}", - old.bounds, new.bounds - )); + match (old_size, new_size) { + (Some(a), Some(b)) if a == b => out.push_str(" moved"), + (Some([w0, h0]), Some([w1, h1])) => { + write!(out, " dimensions changed ({w0} x {h0} -> {w1} x {h1} dbu)").unwrap(); + } + _ if translation(before, after).is_some() => out.push_str(" moved"), + _ => { + out.push_str(" vertices changed"); + let (a, b) = (vertex_count(before), vertex_count(after)); + if a != b { + write!(out, " ({a} -> {b} vertices)").unwrap(); + } + } + } + } + out +} + +fn replacement_heading(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let rectangular = is_polygon(old.kind) + && rectangle_size(object_xy(old)).is_some() + && rectangle_size(object_xy(new)).is_some(); + let mut heading = object_heading(old, object_name(old.kind, rectangular), with_layer); + if let Some(target) = &old.target { + write!(heading, " {target:?}").unwrap(); + } + heading +} + +pub(super) fn write_replacement_objects( + out: &mut CellText<'_>, + old: &ObjectChange, + new: &ObjectChange, +) { + for (sign, object) in [('-', old), ('+', new)] { + let description = object.to_string(); + let mut lines = description.split('\n'); + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + writeln!(out, "{sign} {}", lines.next().unwrap()).unwrap(); + for line in lines { + writeln!(out, " {line}").unwrap(); + } + } +} + +pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { + if let (Some((from, from_type)), Some((to, to_type))) = (old.layer, new.layer) + && old.layer != new.layer + { + return format!( + "{from}/{from_type} → {to}/{to_type} · {}", + transition_shape(old) + ); + } + replacement_fields(old, new, true) +} + +pub(super) fn replacement_description( + old: &ObjectChange, + new: &ObjectChange, + details: bool, +) -> String { + if details { + replacement_heading(old, new, false) + } else { + replacement_fields(old, new, false) + } +} + +fn replacement_fields(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let polygon = is_polygon(old.kind); + let mut out = if polygon { + polygon_replacement(old, new, with_layer) + } else { + replacement_heading(old, new, with_layer) + }; + + let mut separator = if polygon { "\n " } else { " " }; + for (tag, before, after) in field_changes(old, new) { + if polygon && tag == RTAG_XY { + continue; + } + let change = match (tag, before, after) { + (RTAG_XY, Some(a), Some(b)) => { + if let Some([dx, dy]) = translation(a, b) { + format!("moved by ({dx}, {dy}) dbu") + } else if matches!(old.kind, RTAG_PATH | RTAG_NODE) { + let (a, b) = (XY::points(a).len(), XY::points(b).len()); + if a == b { + "vertices changed".to_string() + } else { + format!("vertices changed ({a} -> {b} vertices)") + } + } else { + let points = |data: &[u8]| XY::points(data).collect::>(); + format!("coordinates {:?} -> {:?} dbu", points(a), points(b)) + } + } + (RTAG_COLROW, _, _) if old.kind == RTAG_AREF => { + let counts = |dimensions: Option<[u16; 2]>| { + dimensions.map_or_else(|| "absent".to_string(), |[a, b]| format!("{a} x {b}")) + }; + format!( + "columns/rows {} -> {}", + counts(old.counts), + counts(new.counts) + ) + } + (RTAG_WIDTH, _, _) => { + let width = + |n: Option| n.map_or_else(|| "absent".to_string(), |n| n.to_string()); + format!("width {} -> {} dbu", width(old.width), width(new.width)) + } + _ => { + let name = if tag == RTAG_STRING { + "text".to_string() + } else { + tag_name(tag) + }; + format!("{name} {}", format_field_change(tag, before, after)) + } + }; + write!(out, "{separator}{change}").unwrap(); + separator = "\n "; + } + if old.properties != new.properties { + let properties = |items: &[(u16, Vec)]| { + items + .iter() + .map(|(attribute, value)| format!("{attribute}={}", quoted(value))) + .collect::>() + .join(", ") + }; + write!( + out, + "{separator}properties [{}] -> [{}]", + properties(&old.properties), + properties(&new.properties) + ) + .unwrap(); } - parts.join("; ") + out } diff --git a/src/review/hierarchy.rs b/src/review/hierarchy.rs new file mode 100644 index 0000000..4d5342e --- /dev/null +++ b/src/review/hierarchy.rs @@ -0,0 +1,495 @@ +//! Hierarchy analysis, traversal, and presentation. + +use super::*; + +/// Graph facts shared by text, Mermaid, and JSON renderers. +pub(super) struct HierarchyAnalysis<'a> { + pub(super) old_edges: &'a BTreeMap<(String, String), u64>, + pub(super) new_edges: &'a BTreeMap<(String, String), u64>, + pub(super) edges: BTreeSet<(String, String)>, + pub(super) direct: BTreeSet, + pub(super) affected: BTreeSet, + pub(super) changed_refs: BTreeSet<(String, String)>, + old: &'a Revision, + new: &'a Revision, + comparison: bool, +} + +impl<'a> HierarchyAnalysis<'a> { + pub(super) fn between( + old: &'a Revision, + new: &'a Revision, + changes: Option<&[CellChange]>, + ) -> Self { + let old_edges = &old.edges; + let new_edges = &new.edges; + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = changes + .into_iter() + .flatten() + .map(|cell| cell.name.clone()) + .collect(); + let changed_refs: BTreeSet<_> = changes + .into_iter() + .flatten() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + Self { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + old, + new, + comparison: changes.is_some(), + } + } + + fn visible_rows( + &self, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, + ) -> Option { + let mut visible: BTreeSet<_> = if self.comparison { + self.affected + .iter() + .cloned() + .chain(self.changed_refs.iter().map(|(_, t)| t.clone())) + .collect() + } else { + self.old + .cells + .iter() + .chain(self.new.cells.iter()) + .cloned() + .chain(self.edges.iter().map(|(_, t)| t.clone())) + .collect() + }; + let mut children = BTreeMap::>::new(); + let mut targets = BTreeSet::new(); + let mut shared = BTreeSet::new(); + let fold_paths = self.comparison && !mermaid; + for (parent, target) in &self.edges { + if visible.contains(parent) && visible.contains(target) { + children + .entry(parent.clone()) + .or_default() + .push(target.clone()); + if !targets.insert(target.clone()) && fold_paths { + shared.insert(target.clone()); + } + } + } + if let Some(covered) = covered_edges { + if !covered.is_empty() { + // Keep changed or shared descendants, branches, and target status notes. + let redundant: BTreeSet<_> = covered + .iter() + .map(|(_, target)| target) + .filter(|&target| { + targets.contains(target) + && !shared.contains(target) + && !children.contains_key(target) + && !self.affected.contains(target) + && self.old.cells.contains(target) + && self.new.cells.contains(target) + }) + .collect(); + children.retain(|_, children| { + children.retain(|target| !redundant.contains(target)); + !children.is_empty() + }); + targets.retain(|target| !redundant.contains(target)); + } + visible.retain(|name| children.contains_key(name) || targets.contains(name)); + } + if fold_paths && children.is_empty() { + return None; + } + let rows = traverse(&visible, &children, &targets, max_depth); + Some(VisibleHierarchy { + rows, + children, + shared, + }) + } + + fn single_placement(&self, parent: &str, target: &str) -> bool { + let key = (parent.to_string(), target.to_string()); + self.old_edges.get(&key) == Some(&1) + && self.new_edges.get(&key) == Some(&1) + && !self.changed_refs.contains(&key) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RowNote { + ChildrenOmitted, + SubtreeElided, +} + +struct HierarchyRow { + name: String, + depth: usize, + parent: Option, + note: Option, +} + +fn traverse( + visible: &BTreeSet, + children: &BTreeMap>, + targets: &BTreeSet, + max_depth: usize, +) -> Vec { + let roots: Vec<_> = visible.difference(targets).cloned().collect(); + // Distinguish rootless cycles from descendants hidden by the depth limit. + let mut reachable = BTreeSet::new(); + let mut pending = roots.clone(); + while let Some(name) = pending.pop() { + if reachable.insert(name.clone()) { + pending.extend(children.get(&name).into_iter().flatten().cloned()); + } + } + let mut included = BTreeSet::new(); + let mut rows = Vec::new(); + let mut stack: Vec<_> = roots + .into_iter() + .rev() + .map(|name| (name, 0, None::)) + .collect(); + while let Some((name, depth, parent)) = stack.pop() { + let seen = !included.insert(name.clone()); + let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); + let note = if descendants.is_empty() { + None + } else if depth >= max_depth { + Some(RowNote::ChildrenOmitted) + } else if seen { + Some(RowNote::SubtreeElided) + } else { + None + }; + if !seen && depth < max_depth { + for child in descendants.iter().rev() { + stack.push((child.clone(), depth + 1, Some(name.clone()))); + } + } + rows.push(HierarchyRow { + name, + depth, + parent, + note, + }); + } + for name in visible.difference(&reachable) { + rows.push(HierarchyRow { + name: name.clone(), + depth: 0, + parent: None, + note: Some(RowNote::ChildrenOmitted), + }); + } + rows +} + +struct VisibleHierarchy { + rows: Vec, + children: BTreeMap>, + shared: BTreeSet, +} + +struct AsciiPath<'a> { + indent: usize, + rows: &'a [HierarchyRow], +} + +impl VisibleHierarchy { + fn unbranched(&self, name: &str) -> bool { + !self.shared.contains(name) + && self + .children + .get(name) + .is_none_or(|descendants| descendants.len() == 1) + } + + fn can_join( + &self, + graph: &HierarchyAnalysis<'_>, + parent: &HierarchyRow, + child: &HierarchyRow, + ) -> bool { + child.parent.as_deref() == Some(&parent.name) + && parent.note.is_none() + && !graph.direct.contains(&parent.name) + && graph.affected.contains(&parent.name) + && self.unbranched(&child.name) + && (graph.old.cells.contains(&child.name) || graph.new.cells.contains(&child.name)) + && graph.single_placement(&parent.name, &child.name) + } + + fn ascii_paths<'a>( + &'a self, + graph: &'a HierarchyAnalysis<'_>, + ) -> impl Iterator> { + let mut index = 0; + let mut ancestor_depths = Vec::new(); + std::iter::from_fn(move || { + let first = self.rows.get(index)?; + let mut end = index; + if graph.comparison + && self.unbranched(&first.name) + && first + .parent + .as_ref() + .is_none_or(|parent| graph.single_placement(parent, &first.name)) + { + while self + .rows + .get(end + 1) + .is_some_and(|next| self.can_join(graph, &self.rows[end], next)) + { + end += 1; + } + // Fold paths with at least two edges. + if end - index < 2 { + end = index; + } + } + while ancestor_depths + .last() + .is_some_and(|&depth| depth >= first.depth) + { + ancestor_depths.pop(); + } + let rows = &self.rows[index..=end]; + let indent = ancestor_depths.len(); + ancestor_depths.push(self.rows[end].depth); + index = end + 1; + Some(AsciiPath { indent, rows }) + }) + } +} + +/// The graph joins both revisions, retaining removed branches and old callers. +pub(super) fn render_hierarchy( + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, +) -> String { + let mut out = TextReport::default(); + write_hierarchy(&mut out, old, new, changes, mermaid, max_depth, None); + out.into_string() +} + +pub(super) fn write_hierarchy( + out: &mut TextReport, + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, +) { + if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { + return; + } + let graph = HierarchyAnalysis::between(old, new, changes); + let Some(view) = graph.visible_rows(mermaid, max_depth, covered_edges) else { + return; + }; + if mermaid { + graph.write_mermaid(out, &view.rows); + } else { + graph.write_ascii(out, view.ascii_paths(&graph)); + } +} + +impl HierarchyAnalysis<'_> { + fn status(&self, name: &str, mermaid: bool) -> &'static str { + if !self.old.cells.contains(name) && !self.new.cells.contains(name) { + if self.old.known_cells.contains(name) || self.new.known_cells.contains(name) { + " [outside scope]" + } else { + " [unresolved]" + } + } else if self.direct.contains(name) { + if mermaid { " [cell changed]" } else { "" } + } else if self.affected.contains(name) { + " [affected]" + } else if mermaid && self.comparison { + " [cell unchanged]" + } else { + "" + } + } + + fn edge_label(&self, parent: &str, target: &str, mermaid: bool) -> String { + let key = (parent.to_string(), target.to_string()); + let a = self.old_edges.get(&key).copied().unwrap_or(0); + let b = self.new_edges.get(&key).copied().unwrap_or(0); + let count = if a == b { + format!("x{b}") + } else { + format!("x{a} -> x{b}") + }; + format!( + "{count}{}", + if (mermaid || a == b) && self.changed_refs.contains(&key) { + " [reference changed]" + } else { + "" + } + ) + } + + fn write_mermaid(&self, out: &mut TextReport, rows: &[HierarchyRow]) { + let ids: BTreeMap<_, _> = rows + .iter() + .map(|row| &row.name) + .collect::>() + .into_iter() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + out.push_str("flowchart TD\n"); + if !self.comparison && self.new.scope != Scope::default() { + writeln!(out, " %% {}", self.new.scope.description()).unwrap(); + } + for (name, id) in &ids { + let label = format!("{name}{}", self.status(name, true)); + let label = label + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") + .replace(['\r', '\n'], " "); + writeln!(out, " n{id}[\"{label}\"]").unwrap(); + } + for (parent, target) in &self.edges { + if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { + writeln!( + out, + " n{a} -->|\"{}\"| n{b}", + self.edge_label(parent, target, true) + ) + .unwrap(); + } + } + if rows.is_empty() { + out.push_str(" empty[\"No changed hierarchy\"]\n"); + } + } + + fn write_ascii<'a>(&self, out: &mut TextReport, paths: impl Iterator>) { + out.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + out.push('\n'); + if !self.comparison { + self.new.scope.write_restrictions(out); + } + let mut paths = paths.peekable(); + if paths.peek().is_none() { + out.push_str("No changed hierarchy\n"); + } + for path in paths { + out.push_str(&" ".repeat(path.indent)); + let mut change = None; + for (i, row) in path.rows.iter().enumerate() { + let name = &row.name; + if i != 0 { + out.push_str(" → "); + } + if self.direct.contains(name) { + let marker = + cell_marker(self.old.cells.contains(name), self.new.cells.contains(name)); + change = Some((out.len(), TextStyle::change(marker))); + write!(out, "{marker} ").unwrap(); + } + if self.comparison { + out.push_str(&cell_label(name)); + } else { + write!(out, "{name:?}").unwrap(); + } + if i == 0 + && let Some(parent) = &row.parent + { + write!(out, " {}", self.edge_label(parent, name, false)).unwrap(); + } + } + let last = path.rows.last().unwrap(); + let note = match last.note { + Some(RowNote::ChildrenOmitted) => " [children omitted]", + Some(RowNote::SubtreeElided) => " [subtree elided]", + None => "", + }; + write!(out, "{}{note}", self.status(&last.name, false)).unwrap(); + if let Some((start, style)) = change { + out.mark(start, style); + } + out.push('\n'); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn traversal_distinguishes_depth_limits_from_rootless_components() { + let children = BTreeMap::from([ + ("root".into(), vec!["branch".into()]), + ("branch".into(), vec!["leaf".into()]), + ("cycle_a".into(), vec!["cycle_b".into()]), + ("cycle_b".into(), vec!["cycle_a".into()]), + ]); + let visible = ["root", "branch", "leaf", "cycle_a", "cycle_b"] + .map(String::from) + .into_iter() + .collect(); + let targets = children.values().flatten().cloned().collect(); + let rows = traverse(&visible, &children, &targets, 1); + let rows: Vec<_> = rows + .iter() + .map(|row| { + ( + row.name.as_str(), + row.depth, + row.parent.as_deref(), + row.note, + ) + }) + .collect(); + assert_eq!( + rows, + [ + ("root", 0, None, None), + ("branch", 1, Some("root"), Some(RowNote::ChildrenOmitted)), + ("cycle_a", 0, None, Some(RowNote::ChildrenOmitted)), + ("cycle_b", 0, None, Some(RowNote::ChildrenOmitted)), + ] + ); + } +} diff --git a/src/review/names.rs b/src/review/names.rs index 956bbdc..c551526 100644 --- a/src/review/names.rs +++ b/src/review/names.rs @@ -1,5 +1,66 @@ +//! Embedded layer names: parsing, recognition, and record-preserving updates. + use super::*; +const LAYER_PREFIX: &[u8] = b"PHEON_LAYER_NAMES_"; +pub const LAYER_SENTINEL: &str = "PHEON_LAYER_NAMES_V1"; + +impl Document<'_> { + pub(super) fn read_metadata(&mut self) -> Result<(), String> { + for (cell_name, cell) in &mut self.cells { + for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { + let text = object.field(RTAG_STRING).unwrap(); + if text.starts_with(LAYER_PREFIX) && text != LAYER_SENTINEL.as_bytes() { + // Layer names apply across the library. An unknown version + // may describe layers used outside this carrier or cell, so + // geometry scope selection cannot establish its relevance. + let mut diagnostic = Diagnostic::at( + "unsupported_metadata_version", + object.offset, + "unsupported layer-name metadata version".to_string(), + ); + diagnostic.cell = Some(cell_name.clone()); + diagnostic.layer = object.layer; + diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); + self.global_issues.push(diagnostic); + } + } + let carriers: BTreeSet<_> = cell + .objects + .iter() + .filter(|o| { + o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) + }) + .filter_map(|o| o.layer) + .collect(); + for object in &mut cell.objects { + if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) + { + continue; + } + let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; + if text != LAYER_SENTINEL { + let ((layer, dtype), name) = + parse_layer_name(&text).map_err(|message| error(object.offset, message))?; + if self + .layer_names + .get(&(layer, dtype)) + .is_some_and(|old| old != &name) + { + return Err(error( + object.offset, + format!("conflicting names for {layer}/{dtype}"), + )); + } + self.layer_names.insert((layer, dtype), name.to_string()); + } + object.metadata = true; + } + } + Ok(()) + } +} + pub fn parse_layer(value: &str) -> Result { let (a, b) = value .split_once('/') @@ -7,6 +68,30 @@ pub fn parse_layer(value: &str) -> Result { Ok((decimal(a)?, decimal(b)?)) } +/// Find the unique numeric pair for an exact name across the supplied revisions. +pub fn resolve_layer_name<'a>( + name: &str, + revisions: impl IntoIterator>, +) -> Result { + let matches: BTreeSet<_> = revisions + .into_iter() + .flatten() + .filter_map(|(&pair, label)| (label == name).then_some(pair)) + .collect(); + match matches.len() { + 0 => Err(format!("unknown layer name {name:?}")), + 1 => Ok(*matches.first().unwrap()), + _ => { + let choices = matches + .into_iter() + .map(|(layer, datatype)| format!("{layer}/{datatype}")) + .collect::>() + .join(", "); + Err(format!("ambiguous layer name {name:?} (choose {choices})")) + } + } +} + fn decimal(value: &str) -> Result { if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { return Err("layer numbers must be unsigned decimal integers".to_string()); @@ -28,7 +113,7 @@ pub fn parse_layer_name(value: &str) -> Result<(Layer, String), String> { } fn record(output: &mut Vec, tag: u16, payload: &[u8]) -> Result<(), String> { - let padding = usize::from(tag & 0xff == 6 && payload.len() % 2 != 0); + let padding = usize::from(tag & 0xff == 6 && !payload.len().is_multiple_of(2)); let size = payload .len() .checked_add(padding + 4) diff --git a/src/review/pair.rs b/src/review/pair.rs new file mode 100644 index 0000000..3b247e3 --- /dev/null +++ b/src/review/pair.rs @@ -0,0 +1,223 @@ +//! Adaptive parsing for unscoped comparisons of two revisions. + +use super::*; +use std::{sync::Mutex, thread}; + +struct CellSpan<'a> { + body: &'a [u8], + end_record: usize, +} + +struct FramedRevision<'a> { + records: Vec>, + spans: Option>>, +} + +impl<'a> FramedRevision<'a> { + fn read(input: &'a [u8]) -> Result { + let mut markers = Vec::new(); + let records = read_records_with(input, |index, record| { + if matches!(record.tag, RTAG_BGNSTR | RTAG_ENDSTR) { + markers.push(index); + } + })?; + let spans = index_cells(input, &records, markers); + Ok(Self { records, spans }) + } +} + +// Failure to index a layout leaves error reporting to the regular parser. +fn index_cells<'a>( + input: &'a [u8], + records: &[Record<'a>], + markers: Vec, +) -> Option>> { + let mut cells = BTreeMap::new(); + let mut start = None; + for index in markers { + let record = &records[index]; + match record.tag { + RTAG_BGNSTR => { + if start.is_some() { + return None; + } + let name_record = records.get(index + 1).filter(|r| r.tag == RTAG_STRNAME)?; + let name = decode_name(name_record.data) + .ok() + .filter(|name| !name.is_empty())?; + start = Some((name, name_record.offset)); + } + RTAG_ENDSTR => { + let (name, begin) = start.take()?; + let body = &input[begin..record.offset + 4]; + let span = CellSpan { + body, + end_record: index, + }; + if cells.insert(name, span).is_some() { + return None; + } + } + _ => (), + } + } + start.is_none().then_some(cells) +} + +fn matching_cells<'a>( + old: &FramedRevision<'a>, + new: &FramedRevision<'a>, + input_size: usize, +) -> BTreeMap<&'a str, usize> { + let mut matches = BTreeMap::new(); + if let (Some(old), Some(new)) = (&old.spans, &new.spans) { + let mut bytes = 0; + for (&name, cell) in new { + if old.get(name).is_some_and(|old| old.body == cell.body) { + bytes += cell.body.len(); + matches.insert(name, cell.end_record); + } + } + // Reuse serializes some parsing work, so require substantial overlap. + if bytes < input_size - input_size / 10 { + matches.clear(); + } + } + matches +} + +pub(super) struct CellReuse<'a, 'b> { + old: &'b Document<'a>, + matches: &'b BTreeMap<&'a str, usize>, +} + +impl<'a> CellReuse<'a, '_> { + pub(super) fn cell( + &self, + name: &str, + offset: usize, + records: &[Record<'a>], + ) -> Option<(Cell<'a>, usize)> { + let end = *self.matches.get(name)?; + let previous = &self.old.cells[name]; + // Re-run library metadata and reference checks with the new offsets. + let objects = previous + .objects + .iter() + .filter(|object| object.target.is_some() || object.kind == RTAG_TEXT) + .cloned() + .map(|mut object| { + object.offset = offset + (object.offset - previous.offset); + object.end_offset = offset + (object.end_offset - previous.offset); + object + }) + .collect(); + Some(( + Cell { + offset, + end_offset: records[end].offset, + objects, + attributes: previous.attributes.clone(), + issues: Vec::new(), + }, + end, + )) + } +} + +/// Two unscoped GDSII revisions prepared for comparison. +#[derive(Debug)] +pub struct RevisionPair<'a> { + old: Document<'a>, + new: Document<'a>, + reused_cells: BTreeSet<&'a str>, +} + +impl<'a> RevisionPair<'a> { + pub fn parse(old: &'a [u8], new: &'a [u8]) -> Result { + let invalid = |message, side| Diagnostic::new("invalid_input", message, Some(side)); + let input_size = old.len().max(new.len()); + let parallel = thread::available_parallelism().is_ok_and(|count| count.get() > 1); + let (old, new) = run_together(parallel, old, new, FramedRevision::read); + let old = old.map_err(|message| invalid(message, Side::Old))?; + let new = match new { + Ok(new) => new, + Err(message) => { + Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + return Err(invalid(message, Side::New)); + } + }; + let mut matches = matching_cells(&old, &new, input_size); + let (old, new) = if matches.is_empty() { + let (old, new) = run_together(parallel, old, new, |revision| { + Document::parse_records(&revision.records, None) + }); + (old.map_err(|message| invalid(message, Side::Old))?, new) + } else { + let old_doc = Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + drop(old); + // The regular parser gives unsupported records their own offsets + // and messages, so parse those cells independently on each side. + matches.retain(|name, _| { + old_doc.cells.get(*name).is_some_and(|cell| { + cell.issues.is_empty() + && cell.objects.iter().all(|object| object.issues.is_empty()) + }) + }); + let reuse = CellReuse { + old: &old_doc, + matches: &matches, + }; + let new = Document::parse_records(&new.records, Some(reuse)); + (old_doc, new) + }; + Ok(Self { + old, + new: new.map_err(|message| invalid(message, Side::New))?, + reused_cells: matches.into_keys().collect(), + }) + } + + pub fn compare(self) -> Comparison { + Comparison::collect(&self.old, &self.new, &self.reused_cells, true) + } + + /// Resolve a layer name from the embedded metadata in either revision. + pub fn resolve_layer_name(&self, name: &str) -> Result { + names::resolve_layer_name(name, [&self.old.layer_names, &self.new.layer_names]) + } + + pub fn compare_text(self) -> TextComparison { + TextComparison::collect(&self.old, &self.new, &self.reused_cells) + } +} + +fn run_together( + parallel: bool, + old: I, + new: I, + process: impl Fn(I) -> T + Copy + Send, +) -> (T, T) { + if !parallel { + return (process(old), process(new)); + } + // Transfer ownership only once a worker starts, so a spawn failure can + // process the same input on the calling thread. + let input = Mutex::new(Some(old)); + let run_old = { + let input = &input; + move || { + let old = input.lock().unwrap().take().unwrap(); + process(old) + } + }; + thread::scope(|scope| { + if let Ok(worker) = thread::Builder::new().spawn_scoped(scope, run_old) { + let new = process(new); + return (worker.join().unwrap(), new); + } + (run_old(), process(new)) + }) +} diff --git a/src/review/report.rs b/src/review/report.rs index aae5c63..8004f48 100644 --- a/src/review/report.rs +++ b/src/review/report.rs @@ -1,6 +1,11 @@ //! Versioned machine-readable diff reports. No consumer needs to parse prose. -use super::*; +use super::{hierarchy::HierarchyAnalysis, *}; +use serde::{ + Serialize, Serializer, + ser::{SerializeMap, SerializeSeq}, +}; use serde_json::{Value, json}; +use std::io; use std::path::{Path, PathBuf}; pub const SCHEMA_VERSION: u32 = 1; @@ -27,32 +32,37 @@ fn scope(value: &Scope) -> Value { json!({"top": value.top, "cell": value.cell, "layer": value.layer.map(layer)}) } -fn input(path: Option<&Path>, doc: Option<&Document<'_>>) -> Value { - let units = doc - .filter(|d| d.present) - .and_then(|d| d.header.iter().find(|(tag, _)| *tag == RTAG_UNITS)) - .map(|(tag, data)| field(*tag, data)); +fn input(path: Option<&Path>, revision: Option<&Revision>) -> Value { + let units = revision + .and_then(|r| r.units.as_deref()) + .map(|data| field(RTAG_UNITS, data)); json!({ "path": path.and_then(Path::to_str), "path_encoded_hex": path.filter(|p| p.to_str().is_none()) .map(|p| hex(p.as_os_str().as_encoded_bytes())), - "present": doc.map(|d| d.present), + "present": revision.map(|r| r.present), "units": units }) } -fn envelope(inputs: &ReportInputs, selection: &Scope, options: &ReportOptions) -> Value { +fn envelope( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + outcome: Outcome, + revisions: Option<(&Revision, &Revision)>, +) -> Value { json!({ "schema_version": SCHEMA_VERSION, "kind": "diff", - "producer": {"name": "klamath", "version": env!("CARGO_PKG_VERSION")}, + "producer": {"name": "pheon", "version": env!("CARGO_PKG_VERSION")}, "comparison_profile": PROFILE, - "inputs": {"old": input(inputs.old.as_deref(), None), "new": input(inputs.new.as_deref(), None)}, + "inputs": {"old": input(inputs.old.as_deref(), revisions.map(|r| r.0)), "new": input(inputs.new.as_deref(), revisions.map(|r| r.1))}, "scope": scope(selection), "coordinates": {"unit": "database_unit", "frame": "cell_local", "bounds": "coordinate_envelope"}, - "status": "failed", - "result": "unknown", - "exit_code": 2, + "status": outcome.status(), + "result": outcome.result(), + "exit_code": outcome.exit_code(), "summary": null, "report_selection": { "cell": options.filter.cell, @@ -74,21 +84,25 @@ pub fn failure_report( inputs: &ReportInputs, selection: &Scope, options: &ReportOptions, - code: &str, - message: &str, - side: Option<&str>, + error: &Diagnostic, ) -> Value { - let mut report = envelope(inputs, selection, options); - report["diagnostics"] = json!([{ - "code": code, "severity": "error", "message": message, - "side": side, "byte_offset": null, "cell": null, "layer": null, "target": null - }]); + let mut report = envelope(inputs, selection, options, Outcome::Failed, None); + report["diagnostics"] = json!([diagnostic(error)]); report } +fn diagnostic(value: &Diagnostic) -> Value { + json!({ + "code": value.code, "severity": "error", "message": value.message, + "side": value.side.map(|side| side.to_string()), + "byte_offset": value.offset.map(|offset| offset.to_string()), + "cell": value.cell, "layer": value.layer.map(layer), "target": value.target + }) +} + fn field(tag: u16, data: &[u8]) -> Value { let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { - ("xy", json!(points(data))) + ("xy", json!(XY::points(data).collect::>())) } else if matches!( tag, RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE @@ -96,36 +110,58 @@ fn field(tag: u16, data: &[u8]) -> Value { { ( "uint16", - json!(data.chunks_exact(2).map(integer).collect::>()), + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), ) } else { match tag & 0xff { 0 if data.is_empty() => ("none", Value::Null), 1 if data.len().is_multiple_of(2) => ( "bits", - json!(data.chunks_exact(2).map(integer).collect::>()), + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), ), 2 if data.len().is_multiple_of(2) => ( "int16", json!( - data.chunks_exact(2) - .map(|v| i16::from_be_bytes(v.try_into().unwrap())) + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) .collect::>() ), ), 3 if data.len().is_multiple_of(4) => ( "int32", json!( - data.chunks_exact(4) - .map(|v| i32::from_be_bytes(v.try_into().unwrap())) + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) .collect::>() ), ), 5 if data.len().is_multiple_of(8) => ( "real8", json!( - data.chunks_exact(8) - .map(|v| decode_real8(u64::from_be_bytes(v.try_into().unwrap()))) + data.as_chunks::<8>() + .0 + .iter() + .map(|&v| decode_real8(u64::from_be_bytes(v))) .collect::>() ), ), @@ -154,59 +190,153 @@ fn object(value: &ObjectChange, id: &str, side: &str) -> Value { } fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { - let a: BTreeMap<_, _> = old.fields.iter().map(|f| (f.tag, &f.data)).collect(); - let b: BTreeMap<_, _> = new.fields.iter().map(|f| (f.tag, &f.data)).collect(); - a.keys() - .chain(b.keys()) - .copied() - .collect::>() - .into_iter() - .filter(|tag| a.get(tag) != b.get(tag)) - .map(|tag| { + changes::field_changes(old, new) + .map(|(tag, before, after)| { json!({ "tag": tag, - "before": a.get(&tag).map(|data| field(tag, data)), - "after": b.get(&tag).map(|data| field(tag, data)) + "before": before.map(|data| field(tag, data)), + "after": after.map(|data| field(tag, data)) }) }) .collect() } +struct SelectedObjects<'a> { + // Up to the last emitted object. A layer filter can leave gaps in this prefix. + objects: &'a [ObjectChange], + layer: Option, + cell_index: usize, + side: &'static str, +} + +impl SelectedObjects<'_> { + fn id(&self, index: usize) -> String { + format!("c{}:{}:{index}", self.cell_index, self.side) + } + + fn contains(&self, index: usize) -> bool { + self.objects + .get(index) + .is_some_and(|object| self.layer.is_none_or(|layer| object.layer == Some(layer))) + } +} + +impl Serialize for SelectedObjects<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(None)?; + for (index, change) in self.objects.iter().enumerate() { + if self.layer.is_none_or(|layer| change.layer == Some(layer)) { + sequence.serialize_element(&object(change, &self.id(index), self.side))?; + } + } + sequence.end() + } +} + +struct SelectedCell<'a> { + cell: &'a CellChange, + removed: SelectedObjects<'a>, + added: SelectedObjects<'a>, +} + +impl Serialize for SelectedCell<'_> { + fn serialize(&self, serializer: S) -> Result { + let revision = |revision: &CellRevision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)}); + let mut map = serializer.serialize_map(Some(7))?; + map.serialize_entry("added", &self.added)?; + map.serialize_entry("after", &self.cell.new.as_ref().map(revision))?; + map.serialize_entry("attributes_changed", &self.cell.attributes_changed())?; + map.serialize_entry("before", &self.cell.old.as_ref().map(revision))?; + map.serialize_entry("name", &self.cell.name)?; + map.serialize_entry("removed", &self.removed)?; + map.serialize_entry("replacements", &SelectedReplacements(self))?; + map.end() + } +} + +struct SelectedReplacements<'a>(&'a SelectedCell<'a>); + +impl Serialize for SelectedReplacements<'_> { + fn serialize(&self, serializer: S) -> Result { + let selection = self.0; + let mut sequence = serializer.serialize_seq(None)?; + for replacement in &selection.cell.replacements { + if selection.removed.contains(replacement.removed) + && selection.added.contains(replacement.added) + { + let old = &selection.cell.removed[replacement.removed]; + let new = &selection.cell.added[replacement.added]; + sequence.serialize_element(&json!({ + "removed": selection.removed.id(replacement.removed), + "added": selection.added.id(replacement.added), + "match_rule": replacement.match_rule, + "explanation": changes::replacement_summary(old, new), + "field_changes": field_changes(old, new), + "properties_changed": old.properties != new.properties + }))?; + } + } + sequence.end() + } +} + +struct JsonReport<'a> { + metadata: Value, + cells: Vec>, +} + +impl Serialize for JsonReport<'_> { + fn serialize(&self, serializer: S) -> Result { + let metadata = self + .metadata + .as_object() + .expect("report metadata is an object"); + let mut map = serializer.serialize_map(Some(metadata.len()))?; + for (key, value) in metadata { + if key == "cell_changes" { + map.serialize_entry(key, &self.cells)?; + } else { + map.serialize_entry(key, value)?; + } + } + map.end() + } +} + impl Comparison { - /// Render evidence from the same documents used to construct this comparison. + /// Render the captured comparison evidence without retaining its source documents. /// IDs are local to this report, not persistent object identities. - pub fn json_report( + pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value { + serde_json::to_value(self.prepare_report(inputs, options)) + .expect("serialize comparison report") + } + + /// Stream a pretty-printed report to a writer. + pub fn write_json_report( &self, - old: &Document<'_>, - new: &Document<'_>, + writer: impl io::Write, inputs: &ReportInputs, options: &ReportOptions, - ) -> Value { - let mut report = envelope(inputs, &self.scope, options); - report["inputs"] = json!({"old": input(inputs.old.as_deref(), Some(old)), "new": input(inputs.new.as_deref(), Some(new))}); - let different = !self.cells.is_empty() - || !self.library_fields.is_empty() - || !self.layer_names.is_empty(); - report["status"] = json!(if self.issues.is_empty() { - "complete" - } else { - "incomplete" - }); - report["result"] = json!(if different { - "different" - } else if self.is_equal() { - "equal" - } else { - "unknown" - }); - report["exit_code"] = json!(self.exit_code()); + ) -> serde_json::Result<()> { + serde_json::to_writer_pretty(writer, &self.prepare_report(inputs, options)) + } + + fn prepare_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> JsonReport<'_> { + let outcome = self.outcome(); + let mut report = envelope( + inputs, + &self.old.scope, + options, + outcome, + Some((&self.old, &self.new)), + ); report["summary"] = json!({ - "counts": if self.issues.is_empty() { "exact" } else { "observed" }, + "counts": if outcome.is_complete() { "exact" } else { "observed" }, "changed_cells": self.cells.len().to_string(), "removed_objects": self.cells.iter().map(|c| c.removed.len()).sum::().to_string(), "added_objects": self.cells.iter().map(|c| c.added.len()).sum::().to_string(), "layer_name_changes": self.layer_names.len().to_string(), - "library_field_changes": self.library_fields.len().to_string() + "library_field_changes": self.library_changes.len().to_string() }); report["layer_legend"] = json!( self.layer_legend @@ -222,11 +352,17 @@ impl Comparison { })) .collect::>() ); - report["library_changes"] = json!(self.library_fields.iter().map(|c| { - if c.tag.is_none() { - json!({"field": "presence", "tag": null, "before": old.present, "after": new.present}) - } else { - json!({"field": c.field, "tag": c.tag, "before": fields(&c.old_fields), "after": fields(&c.new_fields)}) + report["library_changes"] = json!(self.library_changes.iter().map(|change| { + match change { + LibraryChange::Presence { old, new } => { + json!({"field": "presence", "tag": null, "before": old, "after": new}) + } + LibraryChange::Record { tag, old, new } => { + let values = |data: &[Vec]| { + data.iter().map(|bytes| field(*tag, bytes)).collect::>() + }; + json!({"field": tag_name(*tag), "tag": tag, "before": values(old), "after": values(new)}) + } } }).collect::>()); let (mut emitted, mut filtered_objects, mut limited, mut filtered_cells) = @@ -243,12 +379,8 @@ impl Comparison { filtered_objects += cell.removed.len() + cell.added.len(); continue; } - let (mut removed, mut added) = (Vec::new(), Vec::new()); - let mut shown = BTreeSet::new(); - for (side, objects, output) in [ - ("old", &cell.removed, &mut removed), - ("new", &cell.added, &mut added), - ] { + let mut select = |objects: &[ObjectChange]| { + let mut end = 0; for (index, change) in objects.iter().enumerate() { if options .filter @@ -259,32 +391,29 @@ impl Comparison { } else if options.max_objects.is_some_and(|max| emitted >= max) { limited += 1; } else { - let id = format!("c{cell_index}:{side}:{index}"); - output.push(object(change, &id, side)); - shown.insert(id); + end = index + 1; emitted += 1; } } - } - let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| { - let a = format!("c{cell_index}:old:{}", r.removed); - let b = format!("c{cell_index}:new:{}", r.added); - (shown.contains(&a) && shown.contains(&b)).then(|| json!({ - "removed": a, "added": b, "match_rule": r.match_rule, - "explanation": r.explanation, - "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), - "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties - })) - }).collect(); - cells.push(json!({ - "name": cell.name, - "before": cell.old_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.old_attributes)})), - "after": cell.new_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.new_attributes)})), - "attributes_changed": cell.attributes_changed, - "removed": removed, "added": added, "replacements": replacements - })); + end + }; + let layer = options.filter.layer; + cells.push(SelectedCell { + cell, + removed: SelectedObjects { + objects: &cell.removed[..select(&cell.removed)], + layer, + cell_index, + side: "old", + }, + added: SelectedObjects { + objects: &cell.added[..select(&cell.added)], + layer, + cell_index, + side: "new", + }, + }); } - report["cell_changes"] = json!(cells); report["report_selection"]["truncated"] = json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0); for (key, count) in [ @@ -295,37 +424,33 @@ impl Comparison { ] { report["report_selection"][key] = json!(count.to_string()); } - let mut diagnostics: Vec<_> = [("old", old), ("new", new)] - .into_iter() - .flat_map(|(side, doc)| { - doc.diagnostics.iter().map(move |d| { - json!({ - "code": d.code, "severity": "error", "message": d.message, "side": side, - "byte_offset": d.offset.map(|n| n.to_string()), "cell": d.cell, - "layer": d.layer.map(layer), "target": d.target - }) - }) - }) - .collect(); - if old.scope != new.scope { - diagnostics.push(json!({"code": "scope_mismatch", "severity": "error", "message": "comparison scopes differ", "side": null, "byte_offset": null, "cell": null, "layer": null, "target": null})); - } - report["diagnostics"] = json!(diagnostics); + report["diagnostics"] = json!(self.diagnostics.iter().map(diagnostic).collect::>()); if options.hierarchy { - report["hierarchy"] = graph(old, new, self); + report["hierarchy"] = graph(self); + } + JsonReport { + metadata: report, + cells, } - report } } -fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { - let a = old.edges(); - let b = new.edges(); - let edges: BTreeSet<_> = a.keys().chain(b.keys()).cloned().collect(); +fn graph(diff: &Comparison) -> Value { + let old = &diff.old; + let new = &diff.new; + let HierarchyAnalysis { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + .. + } = HierarchyAnalysis::between(old, new, Some(&diff.cells)); let names: BTreeSet<_> = old .cells - .keys() - .chain(new.cells.keys()) + .iter() + .chain(new.cells.iter()) .cloned() .chain( edges @@ -333,34 +458,10 @@ fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { .flat_map(|(parent, child)| [parent.clone(), child.clone()]), ) .collect(); - let direct: BTreeSet<_> = diff.cells.iter().map(|c| c.name.clone()).collect(); - let mut affected = direct.clone(); - let mut parents = BTreeMap::<&str, Vec<&str>>::new(); - for (parent, child) in &edges { - parents.entry(child).or_default().push(parent); - } - let mut pending: Vec<_> = direct.iter().cloned().collect(); - while let Some(child) = pending.pop() { - for parent in parents.get(child.as_str()).into_iter().flatten() { - if affected.insert((*parent).to_string()) { - pending.push((*parent).to_string()); - } - } - } - let changed_refs: BTreeSet<_> = diff - .cells - .iter() - .flat_map(|c| { - c.removed - .iter() - .chain(&c.added) - .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) - }) - .collect(); - let a_targets: BTreeSet<_> = a.keys().map(|(_, t)| t.as_str()).collect(); - let b_targets: BTreeSet<_> = b.keys().map(|(_, t)| t.as_str()).collect(); - let state = |doc: &Document<'_>, targets: &BTreeSet<&str>, name: &String| { - if doc.cells.contains_key(name) { + let old_targets: BTreeSet<_> = old_edges.keys().map(|(_, t)| t.as_str()).collect(); + let new_targets: BTreeSet<_> = new_edges.keys().map(|(_, t)| t.as_str()).collect(); + let state = |doc: &Revision, targets: &BTreeSet<&str>, name: &String| { + if doc.cells.contains(name) { "included" } else if doc.known_cells.contains(name) { "outside_scope" @@ -372,13 +473,13 @@ fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { }; json!({ "nodes": names.iter().map(|name| json!({ - "cell": name, "before": state(old, &a_targets, name), "after": state(new, &b_targets, name), + "cell": name, "before": state(old, &old_targets, name), "after": state(new, &new_targets, name), "change": if direct.contains(name) { "direct" } else if affected.contains(name) { "affected" } else { "unchanged" } })).collect::>(), "edges": edges.iter().map(|edge| json!({ "parent": edge.0, "target": edge.1, - "before_count": a.get(edge).copied().unwrap_or(0).to_string(), - "after_count": b.get(edge).copied().unwrap_or(0).to_string(), + "before_count": old_edges.get(edge).copied().unwrap_or(0).to_string(), + "after_count": new_edges.get(edge).copied().unwrap_or(0).to_string(), "reference_changed": changed_refs.contains(edge) })).collect::>() }) diff --git a/src/review/selection.rs b/src/review/selection.rs index bab79a1..4e3491f 100644 --- a/src/review/selection.rs +++ b/src/review/selection.rs @@ -1,3 +1,5 @@ +//! Comparison scope and display-only filters. + use super::*; #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -14,7 +16,124 @@ pub struct DisplayFilter { pub layer: Option, } +#[derive(Default)] +pub(super) struct TextSelection<'a> { + pub removed_cells: Vec<&'a str>, + pub added_cells: Vec<&'a str>, + pub modified_cells: Vec>, + pub filtered_objects: usize, + pub limited_objects: usize, + pub filtered_cells: usize, +} + +pub(super) struct TextCellSelection<'a> { + pub cell: &'a CellChange, + pub attributes_changed: bool, + pub replacements: Vec<(&'a ObjectChange, &'a ObjectChange)>, + pub removed: Vec<&'a ObjectChange>, + pub added: Vec<&'a ObjectChange>, +} + +impl<'a> TextSelection<'a> { + pub fn new(cells: &'a [CellChange], show: &DisplayFilter, mut remaining: usize) -> Self { + let mut selection = Self::default(); + let visible = + |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); + for cell in cells { + let marker = cell_marker(cell.old.is_some(), cell.new.is_some()); + let attributes_changed = marker == '*' && cell.attributes_changed(); + if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + selection.filtered_cells += usize::from(marker != '*' || attributes_changed); + if marker == '*' { + selection.filtered_objects += cell.removed.len() + cell.added.len(); + } + continue; + } + match marker { + '-' => { + selection.removed_cells.push(&cell.name); + continue; + } + '+' => { + selection.added_cells.push(&cell.name); + continue; + } + _ => (), + } + + let mut selected = TextCellSelection { + cell, + attributes_changed, + replacements: Vec::new(), + removed: Vec::new(), + added: Vec::new(), + }; + let mut paired_old = BTreeSet::new(); + let mut paired_new = BTreeSet::new(); + for pair in &cell.replacements { + let old = &cell.removed[pair.removed]; + let new = &cell.added[pair.added]; + if visible(old) && visible(new) { + paired_old.insert(pair.removed); + paired_new.insert(pair.added); + if remaining >= 2 { + selected.replacements.push((old, new)); + remaining -= 2; + } else { + selection.limited_objects += 2; + } + } + } + // A layer filter may leave one side of a pair eligible on its own. + for (objects, paired, selected) in [ + (&cell.removed, &paired_old, &mut selected.removed), + (&cell.added, &paired_new, &mut selected.added), + ] { + for (index, object) in objects.iter().enumerate() { + if !visible(object) { + selection.filtered_objects += 1; + } else if !paired.contains(&index) { + if remaining == 0 { + selection.limited_objects += 1; + } else { + selected.push(object); + remaining -= 1; + } + } + } + } + if attributes_changed + || !selected.replacements.is_empty() + || !selected.removed.is_empty() + || !selected.added.is_empty() + { + selection.modified_cells.push(selected); + } + } + selection + } +} + impl Scope { + pub(super) fn write_restrictions(&self, out: &mut TextReport) { + if self == &Self::default() { + return; + } + let start = out.len(); + out.push_str("Scope"); + if let Some(top) = &self.top { + write!(out, " top={top:?}").unwrap(); + } + if let Some(cell) = &self.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = self.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + pub fn description(&self) -> String { let top = self .top @@ -35,21 +154,23 @@ impl Scope { pub fn validate_pair(&self, old: &Document<'_>, new: &Document<'_>) -> Result<(), String> { for (kind, name) in [("top", &self.top), ("cell", &self.cell)] { - if let Some(name) = name { - if !old.cells.contains_key(name) && !new.cells.contains_key(name) { - return Err(format!( - "selected {kind} {name:?} exists in neither revision" - )); - } - } - } - if let (Some(top), Some(cell)) = (&self.top, &self.cell) { - if !old.reachable(top).contains(cell) && !new.reachable(top).contains(cell) { + if let Some(name) = name + && !old.cells.contains_key(name) + && !new.cells.contains_key(name) + { return Err(format!( - "selected cell {cell:?} is not reachable from {top:?}" + "selected {kind} {name:?} exists in neither revision" )); } } + if let (Some(top), Some(cell)) = (&self.top, &self.cell) + && !old.reachable(top).contains(cell) + && !new.reachable(top).contains(cell) + { + return Err(format!( + "selected cell {cell:?} is not reachable from {top:?}" + )); + } Ok(()) } } @@ -62,7 +183,6 @@ impl<'a> Document<'a> { records: Vec::new(), cells: BTreeMap::new(), layer_names: BTreeMap::new(), - issues: Vec::new(), diagnostics: Vec::new(), header: self.header.clone(), global_issues: Vec::new(), @@ -75,16 +195,16 @@ impl<'a> Document<'a> { let mut found = BTreeSet::new(); let mut pending = vec![root.to_string()]; while let Some(name) = pending.pop() { - if let Some(cell) = self.cells.get(&name) { - if found.insert(name) { - pending.extend(cell.objects.iter().filter_map(|o| o.target.clone())); - } + if let Some(cell) = self.cells.get(&name) + && found.insert(name) + { + pending.extend(cell.objects.iter().filter_map(|o| o.target.clone())); } } found } - pub(super) fn refresh_issues(&mut self) { + pub(super) fn refresh_diagnostics(&mut self) { self.diagnostics = self.global_issues.clone(); for (name, cell) in &self.cells { for issue in &cell.issues { @@ -104,7 +224,6 @@ impl<'a> Document<'a> { } } self.check_graph(); - self.issues = self.diagnostics.iter().map(|d| d.message.clone()).collect(); } /// Project a parsed document. Missing selectors are allowed only for one @@ -225,7 +344,7 @@ impl<'a> Document<'a> { self.layer_names.retain(|pair, _| layers.contains(pair)); } self.scope = scope.clone(); - self.refresh_issues(); + self.refresh_diagnostics(); Ok(self) } } diff --git a/src/review/text.rs b/src/review/text.rs new file mode 100644 index 0000000..d47ed16 --- /dev/null +++ b/src/review/text.rs @@ -0,0 +1,699 @@ +//! Text comparison reports and presentation roles shared with the terminal. + +use super::{hierarchy::write_hierarchy, *}; +use std::ops::Range; + +/// Additional presentation options for a text comparison. +#[derive(Clone, Copy, Debug, Default)] +pub struct TextReportOptions { + /// Print individual object evidence, including both sides of replacements. + pub details: bool, + /// Append the ASCII hierarchy at this depth when provided. + pub ascii_depth: Option, +} + +/// A presentation role for a span of report text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextStyle { + Heading, + Layer, + Added, + Removed, + Changed, + Success, + Warning, + Error, +} + +impl TextStyle { + pub(super) fn change(marker: char) -> Self { + match marker { + '+' => Self::Added, + '-' => Self::Removed, + _ => Self::Changed, + } + } +} + +/// A plain text report with optional style spans. +#[derive(Default)] +pub struct TextReport { + text: String, + spans: Option, TextStyle)>>, +} + +impl TextReport { + pub fn new(styled: bool) -> Self { + Self { + text: String::new(), + spans: styled.then(Vec::new), + } + } + + pub fn as_str(&self) -> &str { + &self.text + } + + pub fn into_string(self) -> String { + self.text + } + + /// Byte ranges are in output order and never overlap. + pub fn styles(&self) -> &[(Range, TextStyle)] { + self.spans.as_deref().unwrap_or(&[]) + } + + pub(super) fn is_styled(&self) -> bool { + self.spans.is_some() + } + + pub fn write_styled(&mut self, style: TextStyle, text: fmt::Arguments<'_>) { + let start = self.len(); + self.write_fmt(text).unwrap(); + self.mark(start, style); + } + + pub(super) fn push_str(&mut self, text: &str) { + self.text.push_str(text); + } + + pub(super) fn push(&mut self, ch: char) { + self.text.push(ch); + } + + pub(super) fn len(&self) -> usize { + self.text.len() + } + + pub(super) fn mark(&mut self, start: usize, style: TextStyle) { + let end = self.text.len(); + if start < end + && let Some(spans) = &mut self.spans + { + if let Some((range, previous)) = spans.last_mut() + && range.end == start + && *previous == style + { + range.end = end; + } else { + spans.push((start..end, style)); + } + } + } +} + +impl fmt::Write for TextReport { + fn write_str(&mut self, text: &str) -> fmt::Result { + self.push_str(text); + Ok(()) + } +} + +enum Part { + Text(String), + Layer(Layer), + Placements(Option<(u64, u64)>), +} + +/// Cell details awaiting inline layer labels and reference placement totals. +pub(super) struct CellText<'a> { + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + layers: BTreeMap, + parts: Vec<(Option, Part)>, + styled: bool, + style: Option, +} + +impl<'a> CellText<'a> { + pub fn new( + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + styled: bool, + ) -> Self { + Self { + names, + name_changes, + layers: BTreeMap::new(), + parts: Vec::new(), + styled, + style: None, + } + } + + pub fn style(&mut self, style: Option) { + self.style = style.filter(|_| self.styled); + } + + pub fn write_layer(&mut self, layer: Layer, side: Option) { + if self.names.contains_key(&layer) { + self.layers + .entry(layer) + .and_modify(|repeated| *repeated = true) + .or_insert(false); + self.parts.push((self.style, Part::Layer(layer))); + } else { + write!(self, "{}/{}", layer.0, layer.1).unwrap(); + if let Some(name) = self.transition_name(layer, side) { + write!(self, " ({})", cell_label(name)).unwrap(); + } + } + } + + pub fn reserve_placements(&mut self) -> usize { + let index = self.parts.len(); + self.parts.push((self.style, Part::Placements(None))); + index + } + + pub fn set_placements(&mut self, index: usize, old: u64, new: u64) { + self.parts[index].1 = Part::Placements(Some((old, new))); + } + + pub fn push_str(&mut self, text: &str) { + self.write_str(text).unwrap(); + } + + pub fn push(&mut self, ch: char) { + self.write_char(ch).unwrap(); + } + + fn append(&mut self, text: &str) { + if text.is_empty() { + return; + } + if let Some((style, Part::Text(current))) = self.parts.last_mut() + && *style == self.style + { + current.push_str(text); + } else { + self.parts.push((self.style, Part::Text(text.to_string()))); + } + } + + fn transition_name(&self, layer: Layer, side: Option) -> Option<&'a str> { + let side = side?; + let at = self + .name_changes + .binary_search_by_key(&layer, |change| change.layer) + .ok()?; + let change = &self.name_changes[at]; + match side { + Side::Old => change.old.as_deref(), + Side::New => change.new.as_deref(), + } + } + + pub fn write_legend(&self, out: &mut TextReport) { + for (&(layer, datatype), &repeated) in &self.layers { + if repeated { + writeln!( + out, + "{layer}/{datatype}: name {:?}", + self.names[&(layer, datatype)] + ) + .unwrap(); + } + } + } + + pub fn write_body(self, out: &mut TextReport) { + for (style, part) in self.parts { + let start = out.len(); + match part { + Part::Text(text) => out.push_str(&text), + Part::Layer((layer, datatype)) => { + write!(out, "{layer}/{datatype}").unwrap(); + if !self.layers[&(layer, datatype)] { + write!(out, " ({})", cell_label(&self.names[&(layer, datatype)])).unwrap(); + } + } + Part::Placements(Some((old, new))) if old == new => { + let suffix = if new == 1 { "" } else { "s" }; + write!(out, " · {new} total placement{suffix}").unwrap(); + } + Part::Placements(Some((old, new))) => { + write!(out, " · placements {old} -> {new}").unwrap(); + } + Part::Placements(None) => (), + } + if let Some(style) = style { + out.mark(start, style); + } + } + } +} + +impl fmt::Write for CellText<'_> { + fn write_str(&mut self, text: &str) -> fmt::Result { + if self.style.is_some() + && let Some((first, rest)) = text.split_once('\n') + { + self.append(first); + self.style = None; + self.append("\n"); + self.append(rest); + } else { + self.append(text); + } + Ok(()) + } +} + +impl TextComparison { + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.comparison + .render_text(max_objects, show, show_library_presence) + } + + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + self.comparison + .render_text_with_options(max_objects, show, show_library_presence, options) + } + + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + self.comparison + .write_text_report(out, max_objects, show, show_library_presence, options); + } + + /// Render the cell changes and their ASCII hierarchy together. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.comparison.render_text_with_ascii_hierarchy( + max_objects, + show, + show_library_presence, + max_depth, + ) + } +} + +impl Comparison { + /// Limit presentation only: the complete comparison has already been made. + pub fn text(&self, max_objects: usize) -> String { + self.text_filtered(max_objects, &DisplayFilter::default()) + } + + pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { + self.render_text(max_objects, show, true) + } + + /// Render a text report. Set `show_library_presence` to false when a file + /// header already announces the library addition or deletion. + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions::default(), + ) + } + + /// Render the cell changes and omit simple hierarchy edges already shown there. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions { + ascii_depth: Some(max_depth), + ..TextReportOptions::default() + }, + ) + } + + /// Render the selected changes with optional object details and ASCII hierarchy. + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + let mut out = TextReport::default(); + self.write_text_report(&mut out, max_objects, show, show_library_presence, options); + out.into_string() + } + + /// Append the selected changes and their presentation roles to a text report. + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + let selection = selection::TextSelection::new(&self.cells, show, max_objects); + let outcome = self.outcome(); + let headline = out.len(); + let (result, style) = match outcome { + Outcome::Equal => ("Equal", TextStyle::Success), + Outcome::Different => ("Different", TextStyle::Warning), + Outcome::Incomplete { .. } => ("Incomplete", TextStyle::Warning), + Outcome::Failed => ("Failed", TextStyle::Error), + }; + out.push_str(result); + let has_omissions = selection.filtered_objects != 0 + || selection.limited_objects != 0 + || selection.filtered_cells != 0; + if !self.cells.is_empty() + && (self.cells.len() > 3 || !outcome.is_complete() || has_omissions) + { + let (mut modified, mut added, mut removed) = (0, 0, 0); + for cell in &self.cells { + match cell_marker(cell.old.is_some(), cell.new.is_some()) { + '-' => removed += 1, + '+' => added += 1, + _ => modified += 1, + } + } + let mut counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + write!(out, " · {count} {noun} {action}").unwrap(); + for (count, action) in counts { + write!(out, ", {count} {action}").unwrap(); + } + } + } + let library_changes = self.library_changes.iter().filter(|change| { + show_library_presence || !matches!(change, LibraryChange::Presence { .. }) + }); + for (count, label) in [ + (library_changes.clone().count(), "library change"), + (self.layer_names.len(), "layer name change"), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural}").unwrap(); + } + } + out.mark(headline, style); + out.push('\n'); + self.old.scope.write_restrictions(out); + if show.cell.is_some() || show.layer.is_some() { + let start = out.len(); + out.push_str("Display filter"); + if let Some(cell) = &show.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = show.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + if has_omissions { + let start = out.len(); + out.push_str("Omitted"); + for (count, label, reason) in [ + (selection.filtered_objects, "object change", "filtered out"), + (selection.limited_objects, "object change", "over the limit"), + ( + selection.filtered_cells, + "cell-level change", + "filtered out", + ), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural} {reason}").unwrap(); + } + } + out.mark(start, TextStyle::Warning); + out.push('\n'); + } + for change in library_changes { + writeln!(out, "library {change}").unwrap(); + } + if self.library_changes.iter().any(|change| { + matches!( + change, + LibraryChange::Record { + tag: RTAG_UNITS, + .. + } + ) + }) { + out.push_str("Coordinates use each file's database units\n"); + } + for change in &self.layer_names { + let name = |value: Option<&str>| { + value.map_or_else(|| "absent".to_string(), |value| format!("{value:?}")) + }; + writeln!( + out, + "{}/{}: name {} -> {}", + change.layer.0, + change.layer.1, + name(change.old.as_deref()), + name(change.new.as_deref()) + ) + .unwrap(); + } + + let mut body = CellText::new(&self.layer_legend, &self.layer_names, out.is_styled()); + let mut covered_edges = BTreeSet::new(); + for selected in &selection.modified_cells { + self.write_cell_details( + &mut body, + selected, + options.details, + (options.ascii_depth.is_some() && outcome.is_complete()) + .then_some(&mut covered_edges), + ); + } + body.write_legend(out); + if !selection.modified_cells.is_empty() + || !selection.removed_cells.is_empty() + || !selection.added_cells.is_empty() + { + out.write_styled(TextStyle::Heading, format_args!("Cells:")); + out.push('\n'); + for name in selection.removed_cells { + out.write_styled(TextStyle::Removed, format_args!("- {}", cell_label(name))); + out.push('\n'); + } + for name in selection.added_cells { + out.write_styled(TextStyle::Added, format_args!("+ {}", cell_label(name))); + out.push('\n'); + } + body.write_body(out); + } + // General comparison errors precede revision-specific errors in text. + if !self.diagnostics.is_empty() { + out.write_styled(TextStyle::Heading, format_args!("Diagnostics")); + out.push('\n'); + for diagnostic in self + .diagnostics + .iter() + .filter(|d| d.side.is_none()) + .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) + { + writeln!(out, " {diagnostic}").unwrap(); + } + } + if let Some(max_depth) = options.ascii_depth { + write_hierarchy( + out, + &self.old, + &self.new, + Some(&self.cells), + false, + max_depth, + Some(&covered_edges), + ); + } + } + + fn write_cell_details( + &self, + out: &mut CellText<'_>, + selected: &selection::TextCellSelection<'_>, + details: bool, + mut covered_edges: Option<&mut BTreeSet<(String, String)>>, + ) { + let cell = selected.cell; + let name = cell_label(&cell.name); + let note = if selected.attributes_changed { + " (attributes changed)" + } else { + "" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "* {name}{note}").unwrap(); + let mut polygons = changes::PolygonChanges::default(); + let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); + for &(a, b) in &selected.replacements { + if !details && a.layer == b.layer && is_polygon(a.kind) { + polygons.replacement(a, b); + continue; + } + out.push_str(" "); + out.style(Some(TextStyle::Changed)); + out.push_str("~ "); + if a.layer != b.layer { + changes::write_layer_transition(out, a, b); + out.push('\n'); + } else { + if let Some(layer) = a.layer { + out.write_layer(layer, None); + out.push_str(": "); + } + let description = changes::replacement_description(a, b, details); + let (heading, rest) = match description.split_once('\n') { + Some((heading, rest)) => (heading, Some(rest)), + None => (description.as_str(), None), + }; + out.push_str(heading); + if let Some(target) = &a.target { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .replacement(a, b, placements); + } + out.push('\n'); + if let Some(rest) = rest { + writeln!(out, "{rest}").unwrap(); + } + } + if details { + changes::write_replacement_objects(out, a, b); + } + } + let mut removed_refs = BTreeMap::new(); + let mut added_refs = BTreeMap::new(); + for (removed, objects, references) in [ + (true, &selected.removed, &mut removed_refs), + (false, &selected.added, &mut added_refs), + ] { + let sign = if removed { '-' } else { '+' }; + let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); + for &object in objects { + if !details && is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if !details && let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; + } else { + unmatched.push((object, 1)); + } + } + // The comparison orders these objects by kind, fields, and properties. + for (object, count) in unmatched { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if count > 1 { + write!(out, "{count}× ").unwrap(); + } + changes::write_unmatched(out, object, details); + if let Some(target) = object.target.as_deref() { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .individual(object, removed, count, placements); + } + out.push('\n'); + } + } + let no_refs = changes::ReferenceCounts::default(); + for (target, (first, counts)) in removed_refs { + if let Some((_, added)) = added_refs.remove(target) { + let placements = counts.write_change(out, target, &added); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &added, placements); + } else { + let placements = counts.write(out, '-', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &no_refs, placements); + } + } + for (target, (first, counts)) in added_refs { + let placements = counts.write(out, '+', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&no_refs, &counts, placements); + } + if !shown_refs.is_empty() { + for target in cell + .removed + .iter() + .chain(&cell.added) + .filter_map(|o| o.target.as_deref()) + { + if let Some(shown) = shown_refs.get_mut(target) { + shown.count_change(); + } + } + for (target, shown) in shown_refs { + if shown.complete() { + let edge = (cell.name.clone(), target.to_owned()); + let old = self.old.edges.get(&edge).copied().unwrap_or(0); + let new = self.new.edges.get(&edge).copied().unwrap_or(0); + shown.write_placements(out, old, new); + if let Some(covered) = covered_edges.as_deref_mut() { + covered.insert(edge); + } + } + } + } + polygons.write(out); + } +} diff --git a/test.py b/test.py index 5421f47..448e8e8 100644 --- a/test.py +++ b/test.py @@ -1,41 +1,8 @@ -import ctypes -import pyarrow -from pyarrow.cffi import ffi - -#c_schema = ffi.new('struct ArrowSchema*') -#c_array = ffi.new('struct ArrowArray*') -#ptr_schema = int(ffi.cast('uintptr_t', c_schema)) -#ptr_array = int(ffi.cast('uintptr_t', c_array)) - - -path = '/home/jan/projects/masque/test.gds' - -#clib = ctypes.CDLL('target/debug/libklamath_rs_ext.so') -clib = ffi.dlopen('target/debug/libklamath_rs_ext.so') - - -ret_ptr_array = ffi.new('struct ArrowArray[]', 1) -ret_ptr_schema = ffi.new('struct ArrowSchema[]', 1) -ffi.cdef(''' - int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); - const char* last_error_message(void); -''') -print(f'{ret_ptr_array[0]=}, {ret_ptr_schema[0]=}') - -status = clib.read_path(path.encode(), ret_ptr_array, ret_ptr_schema) -if status != 0: - message = ffi.string(clib.last_error_message()).decode('utf-8', errors='replace') - raise RuntimeError(message) - -ptr_schema = int(ffi.cast('uintptr_t', ret_ptr_schema)) -ptr_array = int(ffi.cast('uintptr_t', ret_ptr_array)) - - -print(f'{ret_ptr_array[0]=}, {ret_ptr_schema[0]=}') -print(f'python {ptr_array=:x} {ptr_schema=:x}') -#print(f'{ret_ptr_array[0].buffers=} {ret_ptr_schema[0][0]=}') - -arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) +"""Read a GDSII file through Pheon's supported Python API.""" +import sys +from pheon import arrow +if __name__ == "__main__": + print(arrow.read_path(sys.argv[1])) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index e0cc662..993caa8 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,4 +1,4 @@ -use klamath_rs_ext::{basic::encode_real8, records}; +use pheon::{basic::encode_real8, records}; pub fn record(out: &mut Vec, tag: u16, data: &[u8]) { out.extend_from_slice(&u16::try_from(data.len() + 4).unwrap().to_be_bytes()); diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs index 68186e2..3b81821 100644 --- a/tests/ffi_errors.rs +++ b/tests/ffi_errors.rs @@ -1,14 +1,14 @@ mod common; -use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; -use klamath_rs_ext::{last_error_message, read_path, read_cells_bytes, scan_bytes, records}; +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{last_error_message, read_path, read_cells_bytes, scan_bytes, records}; use std::{ffi::{CStr, CString}, fs, path::PathBuf, ptr}; struct TestFile(PathBuf); impl TestFile { fn new(name: &str, contents: &[u8]) -> Self { - let path = std::env::temp_dir().join(format!("klamath-{name}-{}.gds", std::process::id())); + let path = std::env::temp_dir().join(format!("pheon-{name}-{}.gds", std::process::id())); fs::write(&path, contents).unwrap(); Self(path) } @@ -31,20 +31,20 @@ fn error_message() -> String { } #[test] -fn path_errors_preserve_outputs_and_convert_parser_panics() { +fn path_errors_preserve_outputs_and_clear_after_success() { let mut array = FFI_ArrowArray::empty(); let mut schema = FFI_ArrowSchema::empty(); assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); assert!(error_message().contains("path pointer was null")); assert!(array.is_released()); - // A mismatched timestamp size currently makes Record::read panic. + // A mismatched timestamp size returns a parse error with a byte offset. let mut bytes = Vec::new(); common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); let file = TestFile::new("invalid-timestamps", &bytes); assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); - assert!(error_message().contains("Expected record size")); + assert!(error_message().contains("read failed at byte 10: Expected record size")); assert!(array.is_released()); let mut bytes = common::library_header(); diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json index 239f183..2ef560a 100644 --- a/tests/golden/report-equal.json +++ b/tests/golden/report-equal.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", @@ -46,7 +46,7 @@ "layer_name_changes": [], "library_changes": [], "producer": { - "name": "klamath", + "name": "pheon", "version": "test" }, "report_selection": { diff --git a/tests/golden/report-failed.json b/tests/golden/report-failed.json index 3b08309..b4de07a 100644 --- a/tests/golden/report-failed.json +++ b/tests/golden/report-failed.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", @@ -39,7 +39,7 @@ "layer_name_changes": [], "library_changes": [], "producer": { - "name": "klamath", + "name": "pheon", "version": "test" }, "report_selection": { diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json index 58ae71f..c5c5fe7 100644 --- a/tests/golden/report-geometry.json +++ b/tests/golden/report-geometry.json @@ -160,7 +160,7 @@ "replacements": [ { "added": "c0:new:0", - "explanation": "BOUNDARY modified; coordinate bounds unchanged", + "explanation": "1/0: polygon vertices changed (4 -> 3 vertices)", "field_changes": [ { "after": { @@ -225,7 +225,7 @@ ] } ], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", @@ -328,7 +328,7 @@ "layer_name_changes": [], "library_changes": [], "producer": { - "name": "klamath", + "name": "pheon", "version": "test" }, "report_selection": { diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json index 09d4ad3..b1871dd 100644 --- a/tests/golden/report-incomplete.json +++ b/tests/golden/report-incomplete.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", @@ -73,7 +73,7 @@ "layer_name_changes": [], "library_changes": [], "producer": { - "name": "klamath", + "name": "pheon", "version": "test" }, "report_selection": { diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json index 3a60532..2b219da 100644 --- a/tests/golden/report-metadata.json +++ b/tests/golden/report-metadata.json @@ -1,6 +1,6 @@ { "cell_changes": [], - "comparison_profile": "klamath-object-v2", + "comparison_profile": "pheon-structural-v1", "coordinates": { "bounds": "coordinate_envelope", "frame": "cell_local", @@ -55,7 +55,7 @@ ], "library_changes": [], "producer": { - "name": "klamath", + "name": "pheon", "version": "test" }, "report_selection": { diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt index 8811a25..8546367 100644 --- a/tests/golden/review-canonical.txt +++ b/tests/golden/review-canonical.txt @@ -1,9 +1,4 @@ -profile klamath-object-v2 -scope top=all cell=all layer=all -coordinates: integer database units; bounds: coordinate envelopes -HEADER=[600] -LIBNAME="test" -UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54 -layer 1/0 name="WG" +library "test" (GDSII 600, 0.001 user units/dbu, 1e-9 m/dbu) +1/0: name "WG" cell "CHIP" - BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] + 1/0: rectangle (0, 0) to (10, 10) diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt index 57e6c09..60cb697 100644 --- a/tests/golden/review-diff.txt +++ b/tests/golden/review-diff.txt @@ -1,7 +1,4 @@ -profile klamath-object-v2 -scope top=all cell=all layer=all -cell "CHIP" old@Some(62) new@Some(62) - ~ BOUNDARY modified; coordinate bounds unchanged - - @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] - + @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 0, 0, 0] -1 changed cells; 2 of 2 object changes shown; 0 layer-name changes +Different +Cells: +* CHIP + 1/0: 1 polygon changed diff --git a/tests/golden/review-hierarchy.mmd b/tests/golden/review-hierarchy.mmd index dcc963b..86cffd6 100644 --- a/tests/golden/review-hierarchy.mmd +++ b/tests/golden/review-hierarchy.mmd @@ -1,5 +1,4 @@ flowchart TD - %% scope top=all cell=all layer=all n0["CHIP [affected]"] n1["leaf [cell changed]"] n2["left [affected]"] diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt index fa40d79..60542a2 100644 --- a/tests/golden/review-hierarchy.txt +++ b/tests/golden/review-hierarchy.txt @@ -1,6 +1,6 @@ -scope top=all cell=all layer=all -"CHIP" x1 [affected] - "left" x1 [affected] - "leaf" x2 [cell changed] - "right" x1 [affected] - "leaf" x3 [cell changed] [shown above] +Hierarchy +CHIP [affected] + left x1 [affected] + * leaf x2 + right x1 [affected] + * leaf x3 diff --git a/tests/numeric_packing.rs b/tests/numeric_packing.rs index 4001514..14a6109 100644 --- a/tests/numeric_packing.rs +++ b/tests/numeric_packing.rs @@ -1,4 +1,4 @@ -use klamath_rs_ext::*; +use pheon::*; use std::{mem::size_of, slice}; // Read the packed prefix of each source slot, as the Python wrapper does. diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs new file mode 100644 index 0000000..be3ede4 --- /dev/null +++ b/tests/parser_errors.rs @@ -0,0 +1,287 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{ + basic::{ErrType, encode_real8}, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_bytes, read_cells_bytes, + record::RecordHeader, + records, + review::{Comparison, Document, dump_records}, + scan_bytes, +}; +use std::ffi::CStr; + +fn fixture() -> (Vec, (u64, u64)) { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_SREF, &[]); + common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_TEXT, &[]); + common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]); + common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_STRING, b"label\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let end = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + (bytes, (start, end)) +} + +fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range { + let mut offset = 0; + loop { + let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap(); + let start = offset + 4; + let end = start + usize::from(header.data_size); + if header.tag == tag { + return start..end; + } + offset = end; + } +} + +fn check_ffi_error( + expected: &str, + call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32, +) { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(call(&mut array, &mut schema), -1); + let pointer = last_error_message(); + assert!(!pointer.is_null()); + let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); + assert!(error.contains(expected), "{error}"); + assert!( + error.len() < 256, + "error should not dump the input: {error}" + ); + assert!(array.is_released()); +} + +fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) { + let ranges = [range]; + let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)]; + if scan { + results.push(scan_library(bytes)); + } else { + // Hierarchy scans deliberately skip text geometry and its string payloads. + assert!(scan_library(bytes).is_ok()); + } + for result in results { + let (remaining, error) = result.unwrap_err(); + assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr()); + assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message))); + } + let expected = format!("at byte {offset}: {message}"); + check_ffi_error(&expected, |a, s| unsafe { + read_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + let offsets = [range.0, range.1]; + check_ffi_error(&expected, |a, s| unsafe { + read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s) + }); + if scan { + check_ffi_error(&expected, |a, s| unsafe { + scan_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + } +} + +#[test] +fn invalid_utf8_returns_located_parse_errors_in_each_reader() { + let (bytes, range) = fixture(); + assert!(read_library(&bytes).is_ok()); + assert!(read_selected_library(&bytes, &[range]).is_ok()); + assert!(scan_library(&bytes).is_ok()); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + records::RTAG_STRING, + records::RTAG_PROPVALUE, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0xff; + // UTF-8 is checked at the current parser cursor, after the record/header. + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + check_read_errors( + &malformed, + range, + offset, + "Invalid UTF-8 string:", + tag != records::RTAG_STRING, + ); + } +} + +#[test] +fn extra_name_padding_is_ignored_by_comparison_and_arrow_readers() { + let (mut bytes, (start, end)) = fixture(); + let target = payload_range(&bytes, records::RTAG_SNAME); + bytes[target].copy_from_slice(b"leaf"); + bytes.truncate(end as usize); + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let mut padded = bytes.clone(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let payload = payload_range(&padded, tag); + let mut data = padded[payload.clone()].to_vec(); + data.extend_from_slice(&[0; 2]); + let mut record = Vec::new(); + common::record(&mut record, tag, &data); + padded.splice(payload.start - 4..payload.end, record); + } + let baseline = Document::parse(&bytes).unwrap(); + let document = Document::parse(&padded).unwrap(); + assert_eq!(document.cells["töp"].objects.len(), 2); + assert!( + Comparison::between(&baseline, &document) + .outcome() + .is_equal() + ); + + let full = read_library(&padded).unwrap().1; + let scan = scan_library(&padded).unwrap().1; + let ranges = [(start + 2, end + 6), (end + 6, padded.len() as u64 - 4)]; + let selected = read_selected_library(&padded, &ranges).unwrap().1; + let expected = read_library(&bytes).unwrap().1; + for actual in [&full, &scan, &selected] { + for field in ["lib_name", "cell_names"] { + assert_eq!(actual.column_by_name(field), expected.column_by_name(field)); + } + } + assert_eq!(&full, &selected); +} + +#[test] +fn interior_nuls_in_names_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0; + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + let message = "name contains an embedded NUL"; + check_read_errors(&malformed, range, offset, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", payload.start - 4) + ); + } +} + +#[test] +fn malformed_record_frames_are_rejected_at_the_same_offset() { + let (bytes, range) = fixture(); + // STRING also exercises framing in the Arrow scan's geometry skip path. + for tag in [ + records::RTAG_STRNAME, + records::RTAG_XY, + records::RTAG_STRING, + ] { + let start = payload_range(&bytes, tag).start - 4; + for (size, message) in [ + (2_u16, "Record size 2 is smaller than its header"), + (5, "odd record length"), + (u16::MAX - 1, "truncated record"), + ] { + let mut malformed = bytes.clone(); + malformed[start..start + 2].copy_from_slice(&size.to_be_bytes()); + check_read_errors(&malformed, range, start, message, true); + assert_eq!( + dump_records(&malformed).unwrap_err(), + format!("at byte {start}: {message}") + ); + } + } +} + +#[test] +fn nonpositive_unit_scales_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + let units = payload_range(&bytes, records::RTAG_UNITS); + let message = "UNITS must be positive finite values"; + for field in [units.start, units.start + 8] { + for scale in [0.0, -1.0] { + let mut malformed = bytes.clone(); + malformed[field..field + 8] + .copy_from_slice(&encode_real8(scale).unwrap().to_be_bytes()); + check_read_errors(&malformed, range, units.start, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", units.start - 4) + ); + } + } +} + +#[test] +fn coordinate_payloads_require_whole_pairs() { + let (mut bytes, (start, end)) = fixture(); + let after_text = payload_range(&bytes, records::RTAG_TEXT).end; + let xy = payload_range(&bytes[after_text..], records::RTAG_XY); + let payload_start = after_text + xy.start; + let mut replacement = Vec::new(); + common::record(&mut replacement, records::RTAG_XY, &[0; 12]); + bytes.splice(payload_start - 4..after_text + xy.end, replacement); + + let message = "XY payload must contain whole coordinate pairs"; + check_read_errors(&bytes, (start, end + 4), payload_start, message, false); + assert!(Document::parse(&bytes).unwrap_err().contains(message)); +} + +#[test] +fn large_invalid_utf8_strings_do_not_dump_their_contents() { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, records::RTAG_PROPVALUE); + let mut replacement = Vec::new(); + let mut value = vec![b'a'; 60_000]; + value[30_000] = 0xff; + common::record(&mut replacement, records::RTAG_PROPVALUE, &value); + let added = replacement.len() - (payload.end - payload.start + 4); + bytes.splice(payload.start - 4..payload.end, replacement); + let range = (range.0, range.1 + added as u64); + let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end; + check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true); +} + +#[test] +fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() { + for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] { + for timestamp in [0, 1] { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, tag); + let offset = payload.start + timestamp * 12; + bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes()); + check_read_errors(&bytes, range, offset, "Timestamp year", true); + } + } +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py new file mode 100644 index 0000000..64db3f2 --- /dev/null +++ b/tests/python/test_arrow.py @@ -0,0 +1,141 @@ +"""Public Arrow readers, lifetime management and input validation.""" + +import gc +import gzip +import io +import mmap +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy + +from pheon import PheonError, arrow +from pheon._native import ffi +from test_ffi import layout + + +class ArrowTests(unittest.TestCase): + def test_full_scan_and_selected_reads_own_their_data(self): + data = bytearray(layout()) + full = arrow.read_bytes(data) + scan = arrow.scan_bytes(data) + cells = scan.field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + ) + ) + selected = arrow.read_cells_bytes(data, ranges) + data[:] = b"\0" * len(data) + del data + gc.collect() + self.assertEqual(full.to_pylist(), selected.to_pylist()) + self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"]) + self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"]) + + def test_paths_gzip_magic_and_closed_mmap(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "layout with spaces.gds" + path.write_bytes(layout()) + expected = arrow.read_path(path).to_pylist() + with path.open("rb") as stream: + with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer: + result = arrow.read_bytes(buffer) + self.assertEqual(result.to_pylist(), expected) + path.write_bytes(gzip.compress(layout())) + self.assertEqual(arrow.read_path(path).to_pylist(), expected) + + def test_path_reader_keeps_using_the_opened_stream(self): + for payload in (layout(), gzip.compress(layout())): + with self.subTest(compressed=payload.startswith(b"\x1f\x8b")): + source = io.BytesIO(payload) + with patch("builtins.open", return_value=source) as opened: + result = arrow.read_path("opened-stream-only.gds") + opened.assert_called_once_with("opened-stream-only.gds", "rb") + self.assertTrue(source.closed) + self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"]) + + def test_empty_and_noncontiguous_cell_selections(self): + data = layout() + empty = arrow.read_cells_bytes(data, []) + self.assertEqual(empty.field("cells")[0].as_py(), []) + values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2] + self.assertFalse(values.flags.c_contiguous) + self.assertEqual( + arrow.read_cells_bytes(data, values).to_pylist(), + arrow.read_bytes(data).to_pylist(), + ) + + def test_bad_inputs_fail_before_importing_arrow(self): + with self.assertRaises(PheonError): + arrow.read_bytes(b"invalid") + with self.assertRaises(PheonError): + arrow.scan_bytes(b"") + with self.assertRaises(PheonError): + arrow.read_cells_bytes(layout(), [[0, 10]]) + with self.assertRaises(ValueError): + arrow.read_bytes(memoryview(layout())[::2]) + for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]: + with self.assertRaises(ValueError): + arrow.read_cells_bytes(layout(), ranges) + with self.assertRaises(TypeError): + arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + + def test_malformed_strings_and_years_raise_located_parse_errors(self): + data = layout() + cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0] + start = cell["struct_start_offset"] + ranges = [[start, cell["struct_end_offset"]]] + cases = [ + (38, b"\xff", "Invalid UTF-8"), + (start + 32, b"\xff", "Invalid UTF-8"), + ] + cases += [ + (offset, b"\x7f\xff", "Timestamp year") + for offset in (10, 22, start + 4, start + 16) + ] + for offset, value, message in cases: + malformed = bytearray(data) + malformed[offset:offset + len(value)] = value + for read in ( + arrow.read_bytes, + arrow.scan_bytes, + lambda payload: arrow.read_cells_bytes(payload, ranges), + ): + with self.subTest(offset=offset, reader=read): + with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"): + read(malformed) + + def test_import_failure_releases_both_native_outputs(self): + releases = [] + + @ffi.callback("void(struct ArrowArray *)") + def release_array(value): + releases.append("array") + value.release = ffi.NULL + + @ffi.callback("void(struct ArrowSchema *)") + def release_schema(value): + releases.append("schema") + value.release = ffi.NULL + + def export(array, schema): + array.release = release_array + schema.release = release_schema + return 0 + + def fail_import(*args): + raise ValueError("import failed") + + with patch.object( + arrow, + "pyarrow", + SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)), + ): + with self.assertRaisesRegex(ValueError, "import failed"): + arrow._export(export) + self.assertEqual(releases, ["array", "schema"]) diff --git a/tests/python/test_ffi.py b/tests/python/test_ffi.py index bf95534..4dba750 100644 --- a/tests/python/test_ffi.py +++ b/tests/python/test_ffi.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path import unittest -from klamath_rs_ext.klamath_rs_ext import ffi, lib +from pheon._native import ffi, lib def record(tag, data=b""): @@ -47,6 +47,7 @@ class FfiTests(unittest.TestCase): def test_scan_selected_read_and_path_exports(self): data = layout() self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s)) + self.check_export(lambda a, s: lib.read_bytes(data, len(data), a, s)) # The library header is 62 bytes; the one cell ends just before ENDLIB. ranges = ffi.new("uint64_t[]", [62, len(data) - 4]) self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s)) diff --git a/tests/python/test_loader.py b/tests/python/test_loader.py new file mode 100644 index 0000000..4724495 --- /dev/null +++ b/tests/python/test_loader.py @@ -0,0 +1,36 @@ +"""Reject incompatible native libraries before exposing callable functions.""" + +from pathlib import Path +import runpy +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import pheon._native + + +class LoaderTests(unittest.TestCase): + def load(self, library): + with patch("cffi.FFI.dlopen", return_value=library): + return runpy.run_path(str(Path(pheon._native.__file__))) + + def test_matching_versions_are_accepted(self): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: 1, pheon_arrow_schema_version=lambda: 1 + ) + ) + + def test_incompatible_abi_or_schema_is_rejected(self): + for abi, schema in [(2, 1), (1, 2)]: + with self.assertRaisesRegex(ImportError, "Incompatible Pheon library"): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: abi, + pheon_arrow_schema_version=lambda: schema, + ) + ) + + def test_unversioned_library_is_rejected(self): + with self.assertRaisesRegex(ImportError, "version API"): + self.load(SimpleNamespace()) diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py index 43f6487..75ce0cd 100644 --- a/tests/python/test_packing.py +++ b/tests/python/test_packing.py @@ -2,7 +2,7 @@ import unittest import numpy as np -from klamath_rs_ext import pack_int2, pack_int4 +from pheon import pack_int2, pack_int4 class PackingTests(unittest.TestCase): @@ -45,13 +45,14 @@ class PackingTests(unittest.TestCase): def test_failed_conversions_preserve_caller_data(self): for pack, width in [(pack_int2, 2), (pack_int4, 4)]: limit = 1 << (8 * width - 1) - for invalid in [limit, -limit - 1]: - data = np.array([[1, 2], [3, invalid]], dtype=np.int64) - before = data.copy() - with self.subTest(width=width, invalid=invalid): - with self.assertRaises(ValueError): - pack(data) - np.testing.assert_array_equal(data, before) + for dtype in [np.dtype("i8"), np.dtype("i8").newbyteorder("S")]: + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, dtype=dtype, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) for dtype in [np.float32, np.float64]: for invalid in [np.nan, np.inf, -np.inf]: data = np.array([[1, 2], [3, invalid]], dtype=dtype) diff --git a/tests/real8.rs b/tests/real8.rs index 63c4821..fc632a1 100644 --- a/tests/real8.rs +++ b/tests/real8.rs @@ -1,4 +1,4 @@ -use klamath_rs_ext::basic::{decode_real8, encode_real8, pack_real8}; +use pheon::basic::{decode_real8, encode_real8, pack_real8}; #[test] fn normalized_real8_has_known_encodings_and_exact_values() { diff --git a/tests/record_validation.rs b/tests/record_validation.rs new file mode 100644 index 0000000..5fd55d3 --- /dev/null +++ b/tests/record_validation.rs @@ -0,0 +1,122 @@ +mod common; + +use pheon::{ + basic::{ErrType, parse_datetime}, + record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, + }, + records, +}; + +#[test] +fn fixed_record_sizes_are_checked_by_every_read_entry_point() { + let payload = [0; 4]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &payload); + let mut prefixed = Vec::new(); + common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip"); + prefixed.extend_from_slice(&bytes); + + for result in [ + records::HEADER::read(&bytes), + records::HEADER::skip_and_read(&prefixed), + records::HEADER::read_data(&payload, 4), + ] { + let (remaining, reason) = result.unwrap_err(); + assert_eq!(remaining, payload); + assert!(matches!(reason, ErrType::Failed(message) + if message == "Expected record size 2, got 4")); + } + + // The underlying integer-array payload is aligned, but COLROW needs two values. + let payload = [0; 2]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_COLROW, &payload); + assert!(records::COLROW::read(&bytes).is_err()); + assert!(records::COLROW::skip_and_read(&bytes).is_err()); + assert!(records::COLROW::read_data(&payload, 2).is_err()); +} + +#[test] +fn timestamp_year_conversion_checks_the_int16_boundaries() { + for stored_year in [i16::MIN, -1900, 0, i16::MAX - 1900] { + let mut payload = [0; 12]; + payload[..2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, date) = parse_datetime(&payload).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(i32::from(date[0]), i32::from(stored_year) + 1900); + } + for stored_year in [i16::MAX - 1899, i16::MAX] { + for timestamp in [0, 1] { + let mut payload = [0; 24]; + let offset = timestamp * 12; + payload[offset..offset + 2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, error) = DateTimePair::read(&payload, 24).unwrap_err(); + assert_eq!(remaining.as_ptr(), payload[offset..].as_ptr()); + assert!( + matches!(error, ErrType::Failed(message) if message.contains("Timestamp year")) + ); + } + } +} + +#[test] +fn metadata_size_constraints_apply_to_every_read_entry_point() { + macro_rules! check { + ($record:ident, $tag:ident) => {{ + let payload = [b'x'; 44]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + let mut prefixed = common::library_header(); + prefixed.extend_from_slice(&bytes); + assert_eq!(records::$record::read(&bytes).unwrap().1, payload); + assert_eq!( + records::$record::skip_and_read(&prefixed).unwrap().1, + payload + ); + assert_eq!( + records::$record::read_data(&payload, 44).unwrap().1, + payload + ); + + let payload = [b'x'; 46]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + assert!(records::$record::read(&bytes).is_err()); + assert!(records::$record::skip_and_read(&bytes).is_err()); + assert!(records::$record::read_data(&payload, 46).is_err()); + }}; + } + + check!(REFLIBS, RTAG_REFLIBS); + check!(FONTS, RTAG_FONTS); + check!(ATTRTABLE, RTAG_ATTRTABLE); +} + +#[test] +fn malformed_payload_sizes_return_errors_without_panicking() { + let payload = [0; 32]; + assert!(BitArray::read(&payload, 4).is_err()); + assert!(Int2::read(&payload, 4).is_err()); + assert!(Int4::read(&payload, 2).is_err()); + assert!(Real8::read(&payload, 4).is_err()); + assert!(Real8Pair::read(&payload, 8).is_err()); + assert!(DateTimePair::read(&payload, 12).is_err()); + assert!(Empty::read(&payload, 2).is_err()); + assert!(Int2Array::read(&payload, 3).is_err()); + assert!(Int4Array::read(&payload, 6).is_err()); + + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_XY, &payload[..6]); + assert!(records::XY::read(&bytes).is_err()); + assert!(records::XY::skip_and_read(&bytes).is_err()); + assert!(records::XY::read_data(&payload, 6).is_err()); + + // Valid sizes still reject a truncated buffer; an empty string remains valid. + assert!(Int4::read(&payload[..2], 4).is_err()); + assert!(Real8Pair::read(&payload[..8], 16).is_err()); + assert!(DateTimePair::read(&payload[..12], 24).is_err()); + assert!(Int4Array::read(&payload[..4], 8).is_err()); + assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::::new())); +} diff --git a/tests/review.rs b/tests/review.rs index cebe600..ad625bb 100644 --- a/tests/review.rs +++ b/tests/review.rs @@ -3,9 +3,12 @@ mod common; mod reports; use common::{library_header, record}; -use klamath_rs_ext::{ +use pheon::{ records::*, - review::{Comparison, Document, LAYER_SENTINEL, hierarchy}, + review::{ + Comparison, Document, TextComparison, TextReportOptions, names::LAYER_SENTINEL, + selection::DisplayFilter, + }, }; use std::{fs, path::PathBuf, process::Command}; @@ -43,11 +46,23 @@ fn layout(cells: &[(&str, Vec>)]) -> Vec { bytes } +fn gzip(data: &[u8]) -> Vec { + use std::io::Write; + + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + fn polygon(points: &[[i32; 2]]) -> Vec { + ring(RTAG_BOUNDARY, RTAG_DATATYPE, 1, points) +} + +fn ring(kind: u16, type_tag: u16, layer: u16, points: &[[i32; 2]]) -> Vec { let mut bytes = Vec::new(); - record(&mut bytes, RTAG_BOUNDARY, &[]); - record(&mut bytes, RTAG_LAYER, &1_u16.to_be_bytes()); - record(&mut bytes, RTAG_DATATYPE, &0_u16.to_be_bytes()); + record(&mut bytes, kind, &[]); + record(&mut bytes, RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, type_tag, &0_u16.to_be_bytes()); xy(&mut bytes, points); record(&mut bytes, RTAG_ENDEL, &[]); bytes @@ -91,6 +106,19 @@ fn label(carrier: (u16, u16), text: &str) -> Vec { bytes } +fn path(width: Option, points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_PATH, &[]); + record(&mut bytes, RTAG_LAYER, &[0, 1]); + record(&mut bytes, RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, RTAG_WIDTH, &width.to_be_bytes()); + } + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + fn with_property(mut object: Vec, value: &str) -> Vec { object.truncate(object.len() - 4); record(&mut object, RTAG_PROPATTR, &17_u16.to_be_bytes()); @@ -251,7 +279,12 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { let old = Document::parse(&pair.old).unwrap(); let new = Document::parse(&pair.new).unwrap(); let diff = Comparison::between(&old, &new); - assert!(diff.issues.is_empty(), "{}: {:?}", pair.name, diff.issues); + assert!( + diff.diagnostics.is_empty(), + "{}: {:?}", + pair.name, + diff.diagnostics + ); assert_eq!(diff.cells.len(), pair.changed_cells, "{}", pair.name); assert_eq!( diff.cells.iter().map(|c| c.removed.len()).sum::(), @@ -267,14 +300,29 @@ fn ten_fixture_pairs_have_explicit_expected_changes() { ); assert_eq!(diff.layer_names.len(), pair.metadata, "{}", pair.name); let equal = pair.changed_cells + pair.metadata == 0; - assert_eq!(diff.is_equal(), equal, "{}", pair.name); + assert_eq!(diff.outcome().is_equal(), equal, "{}", pair.name); assert_eq!( old.canonical_dump() == new.canonical_dump(), equal, "{}", pair.name ); - assert_eq!(diff.exit_code(), if equal { 0 } else { 1 }); + assert_eq!(diff.outcome().exit_code(), if equal { 0 } else { 1 }); + } +} + +#[test] +fn review_identifies_gzip_and_oasis_input() { + for (bytes, expected) in [ + (&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"), + (&b"%SEMI-OASIS\r\n"[..], "OASIS input"), + ] { + let error = Document::parse(bytes).unwrap_err(); + assert!( + error.starts_with(&format!("at byte 0: {expected}")), + "{error}" + ); + assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error); } } @@ -284,9 +332,7 @@ fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { let doc = Document::parse(&bytes).unwrap(); let dump = doc.canonical_dump(); assert!( - dump.ends_with( - "cell \"CHIP\"\n BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0]\n" - ), + dump.ends_with("cell \"CHIP\"\n 1/0: rectangle (0, 0) to (10, 10)\n"), "{dump}" ); assert!(!dump.contains("BGNSTR")); @@ -295,6 +341,44 @@ fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { .contains("0000000000 0002 HEADER size=6 [600]") ); assert!(doc.record_dump().contains("BGNSTR")); + let notched = polygon(&[[0, 0], [0, 3], [2, 3], [2, 2], [3, 2], [3, 0], [0, 0]]); + let bytes = layout(&[("CHIP", vec![notched])]); + let dump = Document::parse(&bytes).unwrap().canonical_dump(); + assert!( + dump.ends_with(concat!( + " 1/0: polygon xy=[(0, 0), (0, 3), (2, 3), (2, 2),\n", + " (3, 2), (3, 0), (0, 0)]\n", + )), + "{dump}" + ); +} + +#[test] +fn canonical_dump_counts_duplicates_by_normalized_object_within_each_cell() { + let test_layout = |copies: usize| { + let mut objects = vec![square(); copies - 1]; + objects.push(polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])); + objects.push(with_property(square(), "marked")); + layout(&[("A", objects), ("B", vec![square()])]) + }; + let a = test_layout(3); + let b = test_layout(2); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let dump = a.canonical_dump(); + assert!( + dump.ends_with(concat!( + "cell \"A\"\n", + " 3x 1/0: rectangle (0, 0) to (10, 10)\n", + " 1/0: rectangle (0, 0) to (10, 10) property[17]=\"marked\"\n", + "cell \"B\"\n", + " 1/0: rectangle (0, 0) to (10, 10)\n", + )), + "{dump}" + ); + assert!(b.canonical_dump().contains(" 2x 1/0: rectangle")); + assert_ne!(dump, b.canonical_dump()); + assert_eq!(Comparison::between(&a, &b).outcome().exit_code(), 1); } #[test] @@ -317,11 +401,11 @@ fn canonical_dump_and_hierarchy_match_versioned_goldens() { let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); assert_eq!( - hierarchy(&a, &b, Some(&diff), false, 8), + diff.hierarchy(false, 8), include_str!("golden/review-hierarchy.txt") ); assert_eq!( - hierarchy(&a, &b, Some(&diff), true, 8), + diff.hierarchy(true, 8), include_str!("golden/review-hierarchy.mmd") ); } @@ -352,7 +436,8 @@ fn cell_and_property_order_do_not_affect_comparison() { let b = layout(&[("a", vec![]), ("z", vec![b])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); - assert!(Comparison::between(&a, &b).is_equal()); + assert_ne!(a.cells["z"].offset, b.cells["z"].offset); + assert!(Comparison::between(&a, &b).outcome().is_equal()); assert_eq!(a.canonical_dump(), b.canonical_dump()); } @@ -389,13 +474,16 @@ fn metadata_requires_a_sentinel_in_the_same_cell_and_carrier_pair() { .contains("conflicting names") ); assert!( - klamath_rs_ext::review::dump_records(&conflicts) + pheon::review::dump_records(&conflicts) .unwrap() .contains("1:0:B") ); - let future = layout(&[("CHIP", vec![label((9, 0), "KLAMATH_LAYER_NAMES_V2")])]); + let future = layout(&[("CHIP", vec![label((9, 0), "PHEON_LAYER_NAMES_V2")])]); let future = Document::parse(&future).unwrap(); - assert_eq!(Comparison::between(&future, &future).exit_code(), 2); + assert_eq!( + Comparison::between(&future, &future).outcome().exit_code(), + 2 + ); } #[test] @@ -407,37 +495,123 @@ fn unknown_records_never_produce_a_successful_equality_claim() { let bytes = layout(&[("CHIP", vec![unknown])]); let doc = Document::parse(&bytes).unwrap(); let diff = Comparison::between(&doc, &doc); - assert!(!diff.is_equal()); - assert_eq!(diff.exit_code(), 2); + assert!(!diff.outcome().is_equal()); + assert_eq!(diff.outcome().exit_code(), 2); assert!(doc.record_dump().contains("7f00")); - assert!(diff.text(0).contains("INCOMPLETE")); + let text = diff.text(0); + assert!(text.starts_with("Incomplete\nDiagnostics\n")); + assert!(text.contains("unsupported record 0x7f00")); } #[test] -fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { +fn shared_hierarchy_reports_ancestors_and_each_leaf_placement_count() { let a = shared(square()); let b = shared(triangle()); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); assert_eq!( - hierarchy(&a, &b, Some(&diff), false, 8), + diff.hierarchy(false, 8), concat!( - "scope top=all cell=all layer=all\n", - "\"CHIP\" x1 [affected]\n", - " \"left\" x1 [affected]\n", - " \"leaf\" x2 [cell changed]\n", - " \"right\" x1 [affected]\n", - " \"leaf\" x3 [cell changed] [shown above]\n", + "Hierarchy\n", + "CHIP [affected]\n", + " left x1 [affected]\n", + " * leaf x2\n", + " right x1 [affected]\n", + " * leaf x3\n", ) ); - let graph = hierarchy(&a, &b, Some(&diff), true, 8); + let graph = diff.hierarchy(true, 8); assert_eq!(graph.matches("leaf [cell changed]").count(), 1); assert!(graph.contains("x2")); assert!(graph.contains("x3")); - let limited = hierarchy(&a, &b, Some(&diff), false, 1); - assert!(!limited.contains("\"leaf\"")); + let limited = diff.hierarchy(false, 1); + assert!(!limited.contains("leaf")); assert!(limited.contains("children omitted")); + assert_eq!( + diff.hierarchy(false, 0), + "Hierarchy\nCHIP [affected] [children omitted]\n" + ); +} + +#[test] +fn ascii_diff_folds_single_placement_paths_up_to_branches_and_changed_edges() { + let with_branch = |count, shape| { + layout(&[ + ("CHIP", vec![reference("block", 0, 1)]), + ("block", vec![reference("routing", 0, 1)]), + ("routing", vec![reference("fanout", 0, 1)]), + ( + "fanout", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("target", 0, count)]), + ("target", vec![]), + ("right", vec![reference("unit", 0, 1)]), + ("unit", vec![reference("leaf", 0, 1)]), + ("leaf", vec![shape]), + ]) + }; + let old = with_branch(2, square()); + let new = with_branch(3, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "CHIP → block → routing [affected]\n", + " fanout x1 [affected]\n", + " * left x1\n", + " target x2 -> x3\n", + " right x1 → unit → * leaf\n", + ) + ); + assert_eq!( + diff.hierarchy(false, 2), + "Hierarchy\nCHIP → block → routing [affected] [children omitted]\n" + ); + assert!(!diff.hierarchy(true, 8).contains('→')); +} + +#[test] +fn ascii_hierarchy_marks_repeated_subtrees_and_depth_limits() { + let bytes = layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("branch", 0, 2)]), + ("right", vec![reference("branch", 0, 3)]), + ("branch", vec![reference("leaf", 0, 1)]), + ("leaf", vec![]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2\n", + " \"leaf\" x1\n", + " \"right\" x1\n", + " \"branch\" x3 [subtree elided]\n", + ) + ); + assert_eq!( + doc.hierarchy(false, 2), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2 [children omitted]\n", + " \"right\" x1\n", + " \"branch\" x3 [children omitted]\n", + ) + ); } #[test] @@ -449,12 +623,13 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let old = Document::parse(&pair.old).unwrap(); let new = Document::parse(&pair.new).unwrap(); let diff = Comparison::between(&old, &new); - let tree = hierarchy(&old, &new, Some(&diff), false, 8); - assert!(tree.contains("[cell unchanged]"), "{tree}"); - assert!(tree.contains("[reference changed]"), "{tree}"); - if pair.name == "array count" { - assert!(tree.contains("x2 -> x3"), "{tree}"); - } + let tree = diff.hierarchy(false, 8); + let edge = if pair.name == "array count" { + " leaf x2 -> x3\n" + } else { + " leaf x1 [reference changed]\n" + }; + assert!(tree.ends_with(edge), "{tree}"); } let a = layout(&[ ("CHIP", vec![reference("leaf", 0, 1)]), @@ -463,14 +638,19 @@ fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { let b = layout(&[("CHIP", vec![])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); - let tree = hierarchy(&a, &b, Some(&Comparison::between(&a, &b)), false, 8); - assert!(tree.contains("\"leaf\" x1 -> x0"), "{tree}"); + for (old, new, edge) in [ + (&a, &b, " - leaf x1 -> x0\n"), + (&b, &a, " + leaf x0 -> x1\n"), + ] { + let tree = Comparison::between(old, new).hierarchy(false, 8); + assert!(tree.ends_with(edge), "{tree}"); + } } #[test] fn same_bounds_changes_and_truncated_reports_remain_differences() { let a = layout(&[("CHIP", vec![square()])]); - let b = layout(&[("CHIP", vec![triangle()])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new label")])]); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); @@ -478,8 +658,11 @@ fn same_bounds_changes_and_truncated_reports_remain_differences() { diff.cells[0].removed[0].bounds, diff.cells[0].added[0].bounds ); - assert!(diff.text(0).contains("0 of 2 object changes shown")); - assert_eq!(diff.exit_code(), 1); + let limited = diff.text(1); + assert!(limited.contains("Omitted · 2 object changes over the limit")); + assert!(limited.contains("+ 2/0: label")); + assert!(!limited.contains("1/0:")); + assert_eq!(diff.outcome().exit_code(), 1); assert!(diff.cells[0].removed[0].offset > 0); } @@ -494,19 +677,25 @@ fn unit_changes_report_values_without_claiming_geometry_equivalence() { .unwrap() .offset; let mut b = a.clone(); - b[units + 12..units + 20].copy_from_slice( - &klamath_rs_ext::basic::encode_real8(2e-9) - .unwrap() - .to_be_bytes(), - ); + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); - assert_eq!(diff.exit_code(), 1); - assert_eq!(diff.library_fields.len(), 1); - assert_eq!(diff.library_fields[0].field, "UNITS"); - assert!(diff.text(0).contains("UNITS changed")); + assert_eq!(diff.outcome().exit_code(), 1); + assert!(matches!( + diff.library_changes.as_slice(), + [pheon::review::LibraryChange::Record { + tag: RTAG_UNITS, + .. + }] + )); + let text = diff.text(0); + assert!(text.contains("library UNITS [0.001, 1e-9] -> [0.001, 2e-9]")); + assert!(text.contains("Coordinates use each file's database units")); + assert!(!text.contains("raw=")); assert!(diff.cells.is_empty()); + assert!(diff.hierarchy(false, 8).is_empty()); } #[test] @@ -515,17 +704,22 @@ fn dangling_references_cycles_and_duplicate_cell_names_are_reported() { let missing = Document::parse(&missing).unwrap(); assert!( missing - .issues + .diagnostics .iter() - .any(|s| s.contains("unresolved reference")) + .any(|d| d.code == "unresolved_reference") ); let cycle = layout(&[ ("a", vec![reference("b", 0, 1)]), ("b", vec![reference("a", 0, 1)]), ]); let cycle = Document::parse(&cycle).unwrap(); - assert!(cycle.issues.iter().any(|s| s.contains("cyclic"))); - assert!(hierarchy(&cycle, &cycle, None, false, 8).len() < 200); + assert!( + cycle + .diagnostics + .iter() + .any(|d| d.code == "hierarchy_cycle") + ); + assert!(cycle.hierarchy(false, 8).len() < 200); let duplicate = layout(&[("a", vec![]), ("a", vec![])]); assert!( Document::parse(&duplicate) @@ -547,6 +741,7 @@ fn absolute_transform_flags_are_not_discarded() { let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); assert_eq!( Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .outcome() .exit_code(), 1 ); @@ -568,7 +763,11 @@ fn polygon_normalization_handles_repeated_minimum_vertices() { rotated.push(rotated[0]); let candidate = layout(&[("CHIP", vec![polygon(&rotated)])]); let candidate = Document::parse(&candidate).unwrap(); - assert!(Comparison::between(&baseline, &candidate).is_equal()); + assert!( + Comparison::between(&baseline, &candidate) + .outcome() + .is_equal() + ); } open.reverse(); } @@ -609,7 +808,7 @@ fn malformed_lengths_fields_and_units_return_errors_without_panicking() { #[test] fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { - let directory = std::env::temp_dir().join(format!("klamath review {}", std::process::id())); + let directory = std::env::temp_dir().join(format!("pheon review {}", std::process::id())); fs::create_dir_all(&directory).unwrap(); struct Cleanup(PathBuf); impl Drop for Cleanup { @@ -622,9 +821,16 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { let new = directory.join("new file.gds"); fs::write(&old, shared(square())).unwrap(); fs::write(&new, shared(triangle())).unwrap(); - let command = || Command::new(env!("CARGO_BIN_EXE_klamath")); - let equal = command().arg("diff").arg(&old).arg(&old).output().unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let equal = command() + .arg("diff") + .arg(&old) + .arg(&old) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); assert_eq!(equal.status.code(), Some(0)); + assert_eq!(equal.stdout, b"Equal\n"); let diff = command() .arg("diff") .arg(&old) @@ -634,8 +840,9 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { .unwrap(); assert_eq!(diff.status.code(), Some(1)); let text = String::from_utf8(diff.stdout).unwrap(); - assert!(text.contains("0 of 2 object changes shown")); - assert!(text.contains("```mermaid\nflowchart TD")); + assert!(text.contains("Omitted · 2 object changes over the limit")); + assert!(text.contains("Hierarchy\n```mermaid\nflowchart TD")); + assert!(!text.contains("scope top=all")); let dump = command() .arg("dump") .arg(&old) @@ -656,6 +863,34 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { .unwrap() .starts_with("flowchart TD") ); + let with_reference = |x| layout(&[("CHIP", vec![reference("leaf", x, 1)]), ("leaf", vec![])]); + fs::write(&old, with_reference(0)).unwrap(); + fs::write(&new, with_reference(5)).unwrap(); + let moved = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); + assert_eq!(moved.status.code(), Some(1)); + assert_eq!( + moved.stdout, + b"Different\nCells:\n* CHIP\n ~ reference \"leaf\" moved by (5, 0) dbu\n" + ); + let scoped = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii", "--cell", "CHIP"]) + .output() + .unwrap(); + assert_eq!(scoped.status.code(), Some(1)); + assert!( + String::from_utf8(scoped.stdout) + .unwrap() + .ends_with("Hierarchy\n* CHIP\n leaf x1 [reference changed] [outside scope]\n") + ); fs::write(&new, b"invalid").unwrap(); assert_eq!( command() @@ -670,9 +905,279 @@ fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { ); } +#[test] +fn cli_color_preserves_plain_output_and_machine_formats() { + let directory = std::env::temp_dir().join(format!("pheon color {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let with_changes = |name, shape, positions: std::ops::Range| { + let objects = positions + .map(|x| reference("leaf", x, 1)) + .chain([shape]) + .collect(); + layout(&[ + ("top", vec![reference("route", 0, 1)]), + ("route", vec![reference("CHIP", 0, 1)]), + ("CHIP", objects), + ("leaf", vec![]), + (name, vec![]), + ]) + }; + fs::write(&old, with_changes("obsolete", square(), 0..2)).unwrap(); + fs::write(&new, with_changes("new", triangle(), 10..13)).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + let output = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + String::from_utf8(output.stdout).unwrap() + }; + let plain = diff(&["--hierarchy", "mermaid"]); + let colored = diff(&["--hierarchy", "mermaid", "--color=always"]); + for span in [ + "\x1b[31m- obsolete", + "\x1b[32m+ new", + "\x1b[33m* CHIP", + "\x1b[31m-2 refs", + "\x1b[32m+3 refs", + "\x1b[33m1 polygon changed", + ] { + assert!(colored.contains(span), "{span:?} in {colored:?}"); + } + let strip_ansi = |text: &str| { + let mut parts = text.split("\x1b["); + let mut plain = parts.next().unwrap().to_string(); + for part in parts { + let (code, text) = part.split_once('m').unwrap(); + assert!(code.bytes().all(|b| b.is_ascii_digit() || b == b';')); + plain.push_str(text); + } + plain + }; + assert_eq!(strip_ansi(&colored), plain); + assert_eq!(diff(&["--hierarchy", "mermaid", "--color", "never"]), plain); + assert_eq!( + colored.split_once("```mermaid\n").unwrap().1, + plain.split_once("```mermaid\n").unwrap().1 + ); + assert_eq!( + diff(&["--color", "always", "--format", "json"]), + diff(&["--format", "json"]) + ); + let ascii = diff(&["--hierarchy", "ascii", "--color=always"]); + let folded = ascii + .lines() + .find(|line| line.starts_with("top → route → ")) + .unwrap(); + assert!(folded.contains("\x1b[33m* CHIP")); + assert_eq!(strip_ansi(&ascii), diff(&["--hierarchy", "ascii"])); + + for (options, token) in [ + (&[][..], "\x1b[36mcell \"CHIP\""), + (&["--records"][..], "\x1b[36mBGNSTR"), + ] { + let dump = |choice| { + let output = command() + .arg("dump") + .arg(&old) + .args(options) + .args(["--color", choice]) + .output() + .unwrap(); + assert!(output.status.success()); + String::from_utf8(output.stdout).unwrap() + }; + let colored = dump("always"); + assert!(colored.contains(token), "{colored:?}"); + assert_eq!(strip_ansi(&colored), dump("auto")); + assert_eq!(dump("auto"), dump("never")); + } +} + +#[test] +fn cli_details_show_coordinates_and_properties_within_filters_and_limits() { + let directory = std::env::temp_dir().join(format!("pheon-details-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let common_ref = reference("leaf", 0, 1); + let additional_ref = reference("leaf", 5, 1); + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let shifted = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + fs::write( + &old, + layout(&[ + ( + "CHIP", + vec![with_property(square(), "old"), common_ref.clone()], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "before other")]), + ("removed", vec![label((2, 0), "payload in removed cell")]), + ]), + ) + .unwrap(); + fs::write( + &new, + layout(&[ + ( + "CHIP", + vec![ + with_property(shifted, "new"), + with_property(path(Some(8), &points), "path property"), + common_ref, + additional_ref.clone(), + additional_ref, + ], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "after other")]), + ("added", vec![label((2, 0), "payload in added cell")]), + ]), + ) + .unwrap(); + let diff = |options: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + }; + let detailed = diff(&["--show-cell", "CHIP", "--details", "--hierarchy", "ascii"]); + assert!(detailed.contains(concat!( + " ~ 1/0: rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10) property[17]=\"old\"\n", + " + 1/0: rectangle (2, -1) to (12, 9) property[17]=\"new\"\n", + ))); + assert!(detailed.contains(concat!( + " + 1/0: path width=[8] xy=[(0, 0), (4, 0), (4, -5), (13, -5),\n", + " (13, 9)] property[17]=\"path property\"\n", + ))); + assert!(detailed.contains(" + 2× reference \"leaf\" xy=(5, 0) · placements 1 -> 3\n")); + assert!(!detailed.contains("Hierarchy\n")); + assert!(!detailed.contains("after other")); + let limited = diff(&[ + "--show-cell", + "CHIP", + "--details", + "--max-changes", + "1", + "--hierarchy", + "ascii", + ]); + assert!(limited.contains("4 object changes over the limit")); + assert!(!limited.contains("~ 1/0: rectangle")); + assert!(!limited.contains("property[17]=\"old\"")); + assert!(limited.ends_with("Hierarchy\n* CHIP\n leaf x1 -> x3\n")); + + let all = diff(&["--details"]); + assert!(all.contains("Cells:\n- removed\n+ added\n* CHIP\n")); + assert!(!all.contains("payload in")); + assert_eq!( + diff(&["--details", "--format", "json"]), + diff(&["--format", "json"]) + ); +} + +#[test] +fn cli_decodes_gzip_by_header_and_checks_the_trailer() { + let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let plain = directory.join("plain.gds"); + let compressed = directory.join("compressed.gds"); + let data = shared(square()); + // Splitting a record between gzip members must preserve the decompressed layout. + let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat(); + fs::write(&plain, &data).unwrap(); + fs::write(&compressed, &encoded).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let dump = |path: &PathBuf| { + let output = command() + .arg("dump") + .arg(path) + .arg("--records") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout + }; + assert_eq!(dump(&plain), dump(&compressed)); + let compare = || { + let output = command() + .arg("diff") + .arg(&plain) + .arg(&compressed) + .args(["--format", "json"]) + .output() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + (output.status.code(), report) + }; + let (code, report) = compare(); + assert_eq!(code, Some(0)); + assert_eq!(report["result"], "equal"); + + // Corrupt the last CRC after all the GDSII bytes have already been decoded. + let crc = encoded.len() - 8; + encoded[crc] ^= 0xff; + fs::write(&compressed, encoded).unwrap(); + let (code, report) = compare(); + assert_eq!(code, Some(2)); + assert_eq!(report["status"], "failed"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "invalid_input"); + assert_eq!(diagnostic["side"], "new"); + assert!( + diagnostic["message"] + .as_str() + .unwrap() + .contains("gzip decompression failed") + ); +} + #[test] fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { - use klamath_rs_ext::review::write_layer_names; + use pheon::review::names::write_layer_names; use std::collections::BTreeMap; let mut bytes = shared(with_property(square(), "keep this")); bytes.extend_from_slice(&[0; 32]); @@ -720,7 +1225,7 @@ fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { #[test] fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { - use klamath_rs_ext::review::{parse_layer_name, write_layer_names}; + use pheon::review::names::{parse_layer_name, write_layer_names}; use std::collections::BTreeMap; let bytes = layout(&[("CHIP", vec![label((900, 7), "ordinary label")])]); assert!( @@ -733,7 +1238,7 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { .unwrap_err() .contains("does not exist") ); - let bytes = layout(&[("CHIP", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")])]); + let bytes = layout(&[("CHIP", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")])]); assert!( write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::new()) .unwrap_err() @@ -753,14 +1258,14 @@ fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { #[test] fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { - let directory = std::env::temp_dir().join(format!("klamath-metadata-{}", std::process::id())); + let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); - let source = directory.join("source.gds"); + let source = directory.join("source.gds.gz"); let output = directory.join("output.gds"); - let bytes = shared(square()); + let bytes = gzip(&shared(square())); fs::write(&source, &bytes).unwrap(); let _ = fs::remove_file(&output); - let mut command = Command::new(env!("CARGO_BIN_EXE_klamath")); + let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); command.arg("name-layers").arg(&source).arg(&output).args([ "--cell", "CHIP", @@ -788,7 +1293,7 @@ fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { #[test] fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { - use klamath_rs_ext::review::Scope; + use pheon::review::selection::Scope; let a = layout(&[ ("CHIP", vec![reference("leaf", 0, 1)]), ("leaf", vec![square()]), @@ -810,15 +1315,15 @@ fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { ["CHIP", "leaf"] ); assert_eq!(Comparison::between(&old, &new).cells.len(), 1); - assert!(old.canonical_dump().contains("scope top=\"CHIP\"")); + assert!(old.canonical_dump().contains("Scope top=\"CHIP\"\n")); let cell = Scope { cell: Some("CHIP".to_string()), ..Scope::default() }; let old = Document::parse(&a).unwrap().selected(&cell, false).unwrap(); let new = Document::parse(&b).unwrap().selected(&cell, false).unwrap(); - assert!(old.issues.is_empty()); - assert!(Comparison::between(&old, &new).is_equal()); + assert!(old.diagnostics.is_empty()); + assert!(Comparison::between(&old, &new).outcome().is_equal()); let missing = Scope { top: Some("missing".to_string()), ..Scope::default() @@ -843,7 +1348,7 @@ fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { #[test] fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { - use klamath_rs_ext::review::Scope; + use pheon::review::selection::Scope; let mut other_layer = square(); other_layer[8..10].copy_from_slice(&2_u16.to_be_bytes()); let a = layout(&[ @@ -875,14 +1380,17 @@ fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { .unwrap() .selected(&scope, false) .unwrap(); - assert_eq!(Comparison::between(&old, &new).exit_code(), expected); - assert!(old.issues.is_empty()); + assert_eq!( + Comparison::between(&old, &new).outcome().exit_code(), + expected + ); + assert!(old.diagnostics.is_empty()); } } #[test] fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { - use klamath_rs_ext::review::Scope; + use pheon::review::selection::Scope; let mut unknown = square(); unknown.truncate(unknown.len() - 4); record(&mut unknown, 0x7f00, &[]); @@ -897,7 +1405,7 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .is_empty() ); let scope = Scope { @@ -909,7 +1417,7 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .is_empty() ); let bytes = layout(&[ @@ -925,15 +1433,15 @@ fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { .unwrap() .selected(&scope, false) .unwrap() - .issues + .diagnostics .iter() - .any(|s| s.contains("cyclic")) + .any(|d| d.code == "hierarchy_cycle") ); } #[test] fn display_filters_never_change_equality_or_totals() { - use klamath_rs_ext::review::DisplayFilter; + use pheon::review::selection::DisplayFilter; let a = layout(&[("CHIP", vec![square()])]); let b = layout(&[("CHIP", vec![triangle()])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); @@ -944,15 +1452,298 @@ fn display_filters_never_change_equality_or_totals() { layer: Some((42, 0)), }, ); - assert!(text.contains("display-only")); - assert!(text.contains("0 of 2 object changes shown")); - assert_eq!(diff.exit_code(), 1); + assert!(text.starts_with("Different · 1 cell modified\n")); + assert!(text.contains("Display filter layer=42/0\n")); + assert!(text.contains("Omitted · 2 object changes filtered out")); + assert!(!text.contains("Cells:\n")); + assert_eq!(diff.outcome().exit_code(), 1); +} + +#[test] +fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { + use pheon::review::selection::DisplayFilter; + + let mut attributes = Vec::new(); + record(&mut attributes, RTAG_STRCLASS, &[0, 1]); + let a = layout(&[ + ("changed", vec![square()]), + ("other", vec![square()]), + ( + "gone", + vec![square(), triangle(), reference("target", 0, 1)], + ), + ("header", vec![]), + ("target", vec![]), + ]); + let b = layout(&[ + ("changed", vec![triangle()]), + ("other", vec![triangle()]), + ("new", vec![square(), reference("target", 0, 1)]), + ("empty", vec![]), + ("header", vec![attributes]), + ("target", vec![]), + ]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text(0); + assert!(text.starts_with("Different · 3 cells modified, 2 added, 1 removed\n")); + assert!(text.ends_with(concat!( + "Cells:\n", + "- gone\n", + "+ empty\n", + "+ new\n", + "* header (attributes changed)\n", + ))); + assert!(text.contains("Omitted · 4 object changes over the limit")); + + let text = diff.text_filtered( + 1, + &DisplayFilter { + cell: Some("changed".to_string()), + layer: Some((1, 0)), + }, + ); + assert!(text.contains(concat!( + "Display filter cell=\"changed\" layer=1/0\n", + "Omitted · 2 object changes filtered out · 2 object changes over the limit", + " · 4 cell-level changes filtered out\n", + ))); + assert!(!text.contains("Cells:\n")); +} + +#[test] +fn cell_list_quotes_names_with_whitespace() { + let old = layout(&[("old name", vec![]), ("same\nname", vec![square()])]); + let new = layout(&[("new_name", vec![]), ("same\nname", vec![triangle()])]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\n", + "Cells:\n", + "- \"old name\"\n", + "+ new_name\n", + "* \"same\\nname\"\n", + " 1/0: 1 polygon changed\n", + ) + ); + assert!(diff.hierarchy(false, 8).is_empty()); +} + +#[test] +fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps() { + let empty = layout(&[("leaf", vec![]), ("other", vec![])]); + let with_chip = layout(&[ + ( + "CHIP", + vec![ + square(), + label((2, 0), "chip label"), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "3:0:WG"), + reference("leaf", 0, 1), + reference("leaf", 5, 1), + reference("leaf", 2, 2), + reference("other", 10, 1), + ], + ), + ("leaf", vec![]), + ("other", vec![]), + ]); + let existing_chip = layout(&[("CHIP", vec![]), ("leaf", vec![]), ("other", vec![])]); + let empty = Document::parse(&empty).unwrap(); + let with_chip = Document::parse(&with_chip).unwrap(); + let existing_chip = Document::parse(&existing_chip).unwrap(); + + for (old, new, sign) in [(&empty, &with_chip, '+'), (&with_chip, &empty, '-')] { + let diff = Comparison::between(old, new); + let text_diff = TextComparison::between(old, new); + let text = diff.text(100); + assert!(text.ends_with(&format!("Cells:\n{sign} CHIP\n"))); + assert!(!text.contains("\n ")); + assert!(!text.contains("1/0")); + assert!(!text.contains("2/0")); + assert!(!text.contains("Omitted")); + assert_eq!(diff.text(0), text); + assert_eq!(text_diff.outcome(), diff.outcome()); + for mermaid in [false, true] { + assert_eq!(text_diff.hierarchy(mermaid, 8), diff.hierarchy(mermaid, 8)); + } + for show in [ + DisplayFilter::default(), + DisplayFilter { + cell: Some("other".into()), + layer: None, + }, + DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ] { + for max in [0, 2, 100] { + assert_eq!( + text_diff.render_text(max, &show, true), + diff.text_filtered(max, &show) + ); + } + } + } + + for (old, new, sign) in [ + (&existing_chip, &with_chip, "+"), + (&with_chip, &existing_chip, "-"), + ] { + let text = Comparison::between(old, new).text(100); + assert!(text.contains(&format!( + " {sign} 2 refs, 1 array ref to leaf (4 placements)\n" + ))); + assert!(!text.contains("\"leaf\" xy=")); + assert!(text.contains(&format!(" {sign} reference \"other\" xy=(10, 0)\n"))); + } + let dump = with_chip.canonical_dump(); + assert!(dump.contains(" reference \"leaf\" xy=(5, 0)\n")); + assert!(dump.contains(" array reference \"leaf\" (2 × 1) xy=[(2, 0), (102, 0), (2, 0)]\n")); +} + +#[test] +fn reference_summaries_combine_visible_changes_to_a_target() { + let make_layout = |positions: std::ops::Range, moving_x| { + let mut refs: Vec<_> = positions + .chain(20..27) + .map(|x| reference("leaf", x, 1)) + .collect(); + refs.push(reference("moving", moving_x, 1)); + refs.push(reference("moving", 99, 1)); + layout(&[("CHIP", refs), ("leaf", vec![]), ("moving", vec![])]) + }; + let old = make_layout(0..3, 0); + let new = make_layout(10..16, 5); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " ~ reference \"moving\" moved by (5, 0) dbu · 2 total placements\n", + " leaf · -3 refs, +6 refs · placements 10 -> 13\n", + ) + ); + let ascii = + |max, show: &DisplayFilter| diff.render_text_with_ascii_hierarchy(max, show, true, 8); + assert_eq!(ascii(100, &Default::default()), diff.text(100)); + assert!(diff.text(5).ends_with(" - 3 refs to leaf\n")); + let limited = diff.text(6); + assert!( + limited.starts_with( + "Different · 1 cell modified\nOmitted · 5 object changes over the limit\n" + ) + ); + assert!(limited.ends_with(" leaf · -3 refs, +1 ref\n")); + let limited_ascii = ascii(6, &Default::default()); + assert!(limited_ascii.starts_with(&limited)); + assert!(limited_ascii.ends_with("Hierarchy\n* CHIP\n leaf x10 -> x13\n")); + + let filtered = ascii( + 100, + &DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ); + assert!(filtered.contains("11 object changes filtered out")); + assert!(filtered.ends_with(" leaf x10 -> x13\n moving x2 [reference changed]\n")); +} + +#[test] +fn combined_ascii_hierarchy_preserves_paths_to_changed_and_shared_cells() { + let make_layout = |x, shape| { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ( + "left", + vec![ + reference("shared", x, 1), + reference("changed", x, 1), + reference("local", x, 1), + ], + ), + ("right", vec![reference("shared", x, 1)]), + ("shared", vec![]), + ("changed", vec![shape]), + ("local", vec![]), + ]) + }; + let old = make_layout(0, square()); + let new = make_layout(5, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let text = diff.render_text_with_ascii_hierarchy(100, &Default::default(), true, 8); + let (cells, hierarchy) = text.split_once("Hierarchy\n").unwrap(); + assert!(cells.contains("reference \"local\" moved by (5, 0) dbu")); + assert_eq!( + hierarchy, + concat!( + "CHIP [affected]\n", + " * left x1\n", + " * changed x1 [reference changed]\n", + " shared x1 [reference changed]\n", + " * right x1\n", + " shared x1 [reference changed]\n", + ) + ); +} + +#[test] +fn combined_reference_summaries_keep_array_counts_and_placements() { + let old = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 0, 1), + reference("leaf", 10, 2), + reference("leaf", 20, 3), + ], + ), + ("leaf", vec![]), + ]); + let new = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 50, 1), + reference("leaf", 60, 1), + reference("leaf", 70, 2), + reference("leaf", 80, 4), + ], + ), + ("leaf", vec![]), + ]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " leaf · -1 ref, -2 array refs (6 placements), +2 refs, +2 array refs (8 placements)\n", + ) + ); } #[test] fn git_external_driver_handles_changes_additions_deletions_and_errors() { let directory = std::env::temp_dir().join(format!( - "klamath-git-review-{}-{}", + "pheon-git-review-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -963,7 +1754,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let inherited_path = std::env::var_os("PATH").unwrap_or_default(); let git_path = std::env::join_paths( std::iter::once( - PathBuf::from(env!("CARGO_BIN_EXE_klamath")) + PathBuf::from(env!("CARGO_BIN_EXE_pheon")) .parent() .unwrap() .to_path_buf(), @@ -997,7 +1788,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { concat!(env!("CARGO_MANIFEST_DIR"), "/contrib/git/config"), ]); assert_eq!( - String::from_utf8(git(&["config", "--get", "diff.klamath.trustExitCode"]).stdout) + String::from_utf8(git(&["config", "--get", "diff.pheon.trustExitCode"]).stdout) .unwrap() .trim(), "false" @@ -1016,7 +1807,7 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { assert!( String::from_utf8(changed.stdout) .unwrap() - .contains("2 of 2 object changes shown") + .contains("Different\nCells:\n* CHIP\n") ); fs::write(&path, b"invalid").unwrap(); let invalid = Command::new("git") @@ -1027,24 +1818,37 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { .output() .unwrap(); assert!(!invalid.status.success()); - assert!(String::from_utf8_lossy(&invalid.stderr).contains("klamath:")); + assert!(String::from_utf8_lossy(&invalid.stderr).contains("pheon:")); fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + let mode_change = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["git-diff", "--hierarchy", "ascii", "--", "chip layout.gds"]) + .arg(&path) + .args(["old", "100644"]) + .arg(&path) + .args(["new", "100755"]) + .output() + .unwrap(); + assert!(mode_change.status.success()); + assert_eq!( + mode_change.stdout, + b"layout \"chip layout.gds\"\nmode \"100644\" -> \"100755\"\nEqual\n" + ); let added = directory.join("added layout.gds"); fs::write(&added, shared(square())).unwrap(); git(&["add", "added layout.gds"]); let result = git(&["diff", "--cached", "--", "added layout.gds"]); - assert!( - String::from_utf8(result.stdout) - .unwrap() - .contains("layout added") - ); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.starts_with("layout \"added layout.gds\" added\nDifferent · 4 cells added\n")); + assert!(text.contains("Cells:\n+ CHIP\n+ leaf\n+ left\n+ right\nHierarchy\n")); + assert!(!text.contains("\nmode ")); + assert!(!text.contains("\nlibrary ")); fs::remove_file(&path).unwrap(); git(&["add", "-u"]); let result = git(&["diff", "--cached", "--", "chip layout.gds"]); - assert!( - String::from_utf8(result.stdout) - .unwrap() - .contains("layout deleted") + let text = String::from_utf8(result.stdout).unwrap(); + assert_eq!( + text, + "layout \"chip layout.gds\" deleted\nDifferent\nCells:\n- CHIP\n" ); // Textconv is an alternative to the external driver, with a separate cache. git(&["commit", "-qm", "replace fixture"]); @@ -1055,19 +1859,15 @@ fn git_external_driver_handles_changes_additions_deletions_and_errors() { let result = git(&["diff", "--cached", "--find-renames"]); let text = String::from_utf8(result.stdout).unwrap(); assert!(text.contains("renamed layout.gds"), "{text}"); - assert!(text.contains("2 of 2 object changes shown"), "{text}"); + assert!(text.contains("Different\nCells:\n* leaf\n"), "{text}"); git(&["commit", "-qm", "rename fixture"]); - fs::write( - directory.join(".gitattributes"), - "*.gds diff=klamath-text\n", - ) - .unwrap(); + fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); fs::write(&renamed, shared(square())).unwrap(); let result = git(&["diff", "--", "renamed layout.gds"]); assert!( String::from_utf8(result.stdout) .unwrap() - .contains("- BOUNDARY") + .contains("- 1/0: polygon") ); fs::remove_dir_all(directory).unwrap(); } @@ -1087,76 +1887,475 @@ fn replacement_explanations_are_unique_and_keep_full_evidence() { let diff = Comparison::between(&old, &new); assert_eq!(diff.cells[0].replacements.len(), 1); let text = diff.text(100); - assert!(text.contains(" ~ ")); - assert!(text.contains(" - @")); - assert!(text.contains(" + @")); match pair.name { - "moved reference" => assert!(text.contains("delta=(5, 0) dbu")), - "array count" => assert!(text.contains("columns/rows 2 x 1 -> 3 x 1")), + "moved reference" => { + assert!(text.contains(" ~ reference \"leaf\" moved by (5, 0) dbu\n")) + } + "array count" => { + assert!(text.contains(" ~ array reference \"leaf\" columns/rows 2 x 1 -> 3 x 1\n")) + } _ => assert_eq!(text, include_str!("golden/review-diff.txt")), } + assert!(!text.contains(" - ")); + assert!(!text.contains(" + ")); + assert!(!text.contains("bbox=")); assert!(!diff.text(0).contains(" ~ ")); - assert!(diff.text(1).contains("1 of 2 object changes shown")); + assert_eq!( + diff.text(1), + "Different · 1 cell modified\nOmitted · 2 object changes over the limit\n" + ); } let a = layout(&[("CHIP", vec![square(), square()])]); let b = layout(&[("CHIP", vec![triangle(), triangle()])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); assert!(diff.cells[0].replacements.is_empty()); + let text = diff.text(100); + assert!(text.contains(" 1/0: 2 rectangles removed · 2 polygons added\n")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); +} + +#[test] +fn layer_only_shape_replacements_keep_evidence_and_respect_selection() { + let rect = [[10, 0], [0, 0], [0, 10], [10, 10], [10, 0]]; + let on_core = ring(RTAG_BOUNDARY, RTAG_DATATYPE, 2, &rect); + let named = |mut objects: Vec>, names: &[&str]| { + objects.push(label((900, 7), LAYER_SENTINEL)); + objects.extend(names.iter().map(|name| label((900, 7), name))); + layout(&[("CHIP", objects)]) + }; + let old = named(vec![square()], &["1:0:WG", "2:0:CORE"]); + let new = named(vec![on_core.clone()], &["1:0:WG", "2:0:CORE"]); + let old = Document::parse(&old).unwrap(); + let diff = Comparison::between(&old, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(2), + "Different\nCells:\n* CHIP\n ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n" + ); + assert!( + diff.render_text_with_options( + 2, + &Default::default(), + true, + TextReportOptions { + details: true, + ..Default::default() + }, + ) + .ends_with(concat!( + " ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10)\n", + " + 2/0: rectangle (0, 0) to (10, 10)\n", + )) + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(!limited.contains("Cells:")); + for (pair, suffix) in [ + ((1, 0), " 1/0 (WG): 1 rectangle removed\n"), + ((2, 0), " 2/0 (CORE): 1 rectangle added\n"), + ] { + let filtered = diff.text_filtered( + 1, + &DisplayFilter { + cell: None, + layer: Some(pair), + }, + ); + assert!(filtered.contains("1 object change filtered out")); + assert!(filtered.ends_with(suffix), "{filtered}"); + } + let json = diff.json_report(&Default::default(), &Default::default()); + let replacement = &json["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_layer_change"); + assert_eq!(replacement["field_changes"].as_array().unwrap().len(), 1); + assert_eq!(replacement["field_changes"][0]["tag"], RTAG_LAYER); + assert_eq!(json["summary"]["removed_objects"], "1"); + assert_eq!(json["summary"]["added_objects"], "1"); + + let old = named(vec![square()], &["1:0:WG"]); + let new = named(vec![on_core], &["2:0:CORE"]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!( + diff.text(2) + .ends_with(" ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n") + ); +} + +#[test] +fn layer_only_matching_requires_unique_shapes_and_identical_payloads() { + let core_square = ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ); + for (old, new) in [ + (vec![square(), square()], vec![core_square.clone()]), + ( + vec![square()], + vec![core_square.clone(), core_square.clone()], + ), + (vec![square()], vec![with_property(core_square, "new")]), + ( + vec![square()], + vec![ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + )], + ), + ] { + let old = layout(&[("CHIP", old)]); + let new = layout(&[("CHIP", new)]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!(diff.cells[0].replacements.is_empty()); + assert!(diff.text(100).contains(" removed\n")); + assert!(diff.text(100).contains(" added\n")); + } +} + +#[test] +fn datatype_only_path_changes_pair_on_the_same_layer() { + let old = layout(&[("CHIP", vec![path(Some(8), &[[0, 0], [10, 10]])])]); + let old_doc = Document::parse(&old).unwrap(); + let at = old_doc + .records + .iter() + .find(|record| record.tag == RTAG_DATATYPE) + .unwrap() + .offset + + 4; + let mut new = old.clone(); + new[at..at + 2].copy_from_slice(&3_u16.to_be_bytes()); + let diff = Comparison::between(&old_doc, &Document::parse(&new).unwrap()); + assert!(diff.text(2).ends_with(" ~ 1/0 → 1/3 · path\n")); + assert_eq!( + diff.cells[0].replacements[0].match_rule, + "unique_layer_change" + ); +} + +#[test] +fn polygon_diff_reports_single_displacements_and_groups_bulk_moves() { + let before = layout(&[("CHIP", vec![with_property(square(), "old")])]); + let before = Document::parse(&before).unwrap(); + for (shape, label) in [ + ( + polygon(&[[-5, 2], [5, 2], [5, 12], [-5, 12], [-5, 2]]), + "1 rectangle changed", + ), + ( + polygon(&[[0, 0], [20, 0], [20, 10], [0, 10], [0, 0]]), + "1 rectangle changed", + ), + ( + polygon(&[[5, 0], [10, 5], [5, 10], [0, 5], [5, 0]]), + "1 polygon changed", + ), + (square(), "1 rectangle changed"), + ] { + let after = layout(&[("CHIP", vec![with_property(shape, "new")])]); + let after = Document::parse(&after).unwrap(); + let diff = Comparison::between(&before, &after); + let text = diff.text(100); + assert!(text.contains(&format!(" 1/0: {label}\n")), "{text}"); + assert!(!text.contains("dbu")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); + } + + let moved_triangle = polygon(&[[2, -1], [12, -1], [2, 9], [2, -1]]); + let moved_square = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + let named_layout = |mut objects: Vec>| { + objects.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", objects)]) + }; + for (shape, shifted, name) in [ + (triangle(), moved_triangle, "polygon"), + (square(), moved_square.clone(), "rectangle"), + ( + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ), + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]], + ), + "box", + ), + ] { + let before = named_layout(vec![shape]); + let after = named_layout(vec![shifted]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!( + diff.text(100) + .contains(&format!(" 1/0 (WG): {name} moved by (2, -1) dbu\n")) + ); + } + + let before = named_layout(vec![square()]); + let after = named_layout(vec![ + moved_square, + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[20, 0], [30, 0], [30, 10], [20, 10], [20, 0]], + ), + ]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!( + diff.text(100) + .contains(" 1/0 (WG): 1 rectangle moved · 1 box added\n") + ); + assert!( + diff.text(2) + .contains(" 1/0 (WG): rectangle moved by (2, -1) dbu\n") + ); +} + +#[test] +fn polygon_diff_groups_shapes_by_layer_across_dimensions() { + let shapes = [ + square(), + polygon(&[[30, 0], [50, 0], [50, 5], [30, 5], [30, 0]]), + polygon(&[[70, 0], [80, 0], [71, 9], [70, 0]]), + ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + ), + ]; + let layout_with_property = |value| { + let objects = shapes + .iter() + .map(|shape| with_property(shape.clone(), value)) + .collect(); + layout(&[("CHIP", objects)]) + }; + let before = layout_with_property("old"); + let after = layout_with_property("new"); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\n", + "Cells:\n* CHIP\n", + " 1/0: 2 rectangles, 1 polygon changed\n", + " 2/0: 1 polygon changed\n", + ) + ); +} + +#[test] +fn polygon_diff_distinguishes_boxes_from_rectangular_boundaries() { + let before = layout(&[("CHIP", vec![square()])]); + let points = [[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]; + let after = layout(&[("CHIP", vec![ring(RTAG_BOX, RTAG_BOXTYPE, 1, &points)])]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + let text = diff.text(100); + assert!(text.contains(" 1/0: 1 rectangle removed · 1 box added\n")); } #[test] fn path_width_explanation_preserves_absent_and_zero() { - let path = |width: Option| { - let mut bytes = Vec::new(); - record(&mut bytes, RTAG_PATH, &[]); - record(&mut bytes, RTAG_LAYER, &[0, 1]); - record(&mut bytes, RTAG_DATATYPE, &[0, 0]); - if let Some(n) = width { - record(&mut bytes, RTAG_WIDTH, &n.to_be_bytes()); - } - xy(&mut bytes, &[[0, 0], [10, 10]]); - record(&mut bytes, RTAG_ENDEL, &[]); - layout(&[("CHIP", vec![bytes])]) - }; - let a = path(None); - let b = path(Some(0)); + let points = [[0, 0], [10, 10]]; + let a = layout(&[("CHIP", vec![path(None, &points)])]); + let b = layout(&[("CHIP", vec![path(Some(0), &points)])]); let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); - assert!(diff.text(100).contains("width absent -> 0 dbu")); + assert!( + diff.text(100) + .contains(" ~ 1/0: path width absent -> 0 dbu\n") + ); +} + +#[test] +fn unmatched_paths_summarize_coordinates_that_would_wrap() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let empty = layout(&[("CHIP", vec![])]); + let paths = layout(&[( + "CHIP", + vec![ + path(None, &points), + path(Some(-8), &points), + path(Some(0), &points[..4]), + ], + )]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let text = Comparison::between(old, new).text(100); + for width in ["absent", "-8 dbu"] { + assert!(text.contains(&format!( + " {sign} 1/0: path · width {width} · 5 vertices · bounds (0, -5) to (13, 9) dbu\n" + ))); + } + assert_eq!( + text.lines().filter(|line| line.starts_with(" ")).count(), + 3, + "{text}" + ); + assert!(text.contains("xy=[(0, 0), (4, 0), (4, -5), (13, -5)]")); + } + assert!(paths.canonical_dump().contains("\n (13, 9)]")); + let report = + Comparison::between(&empty, &paths).json_report(&Default::default(), &Default::default()); + assert!( + report["cell_changes"][0]["added"] + .as_array() + .unwrap() + .iter() + .flat_map(|object| object["fields"].as_array().unwrap()) + .any(|field| field["tag"] == RTAG_XY && field["value"] == serde_json::json!(points)) + ); +} + +#[test] +fn repeated_unmatched_rows_group_only_identical_objects_within_the_limit() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let mut other_points = points; + other_points[1][0] = 5; + let named_paths = |mut paths: Vec>| { + paths.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", paths)]) + }; + let empty = named_paths(vec![]); + let paths = named_paths(vec![ + path(None, &points), + path(None, &other_points), + path(None, &points), + ]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let diff = Comparison::between(old, new); + let details = "path · width absent · 5 vertices · bounds (0, -5) to (13, 9) dbu"; + let full = diff.text(3); + assert!(full.contains("1/0: name \"WG\"\nCells:\n")); + assert!(full.ends_with(&format!( + " {sign} 2× 1/0: {details}\n {sign} 1/0: {details}\n" + ))); + assert!( + diff.text(2) + .ends_with(&format!(" {sign} 2× 1/0 (WG): {details}\n")) + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(limited.ends_with(&format!(" {sign} 1/0 (WG): {details}\n"))); + } +} + +#[test] +fn label_replacements_show_only_changed_fields() { + let a = layout(&[("CHIP", vec![label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert_eq!( + diff.text(100), + "Different\nCells:\n* CHIP\n ~ 2/0: label text \"old\" -> \"new\"\n" + ); } #[test] fn unchanged_layer_names_explain_geometry_changes_once() { - let named = |geometry| { - layout(&[( - "CHIP", - vec![ - geometry, - label((900, 7), LAYER_SENTINEL), - label((900, 7), "1:0:WG"), - ], - )]) + let named = |geometry, text| { + layout(&[ + ( + "CHIP", + vec![ + geometry, + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + ), + ("leaf", vec![label((1, 0), text)]), + ]) }; - let a = named(square()); - let b = named(triangle()); - let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); - let text = diff.text(100); - assert_eq!(text.matches("name=\"WG\"").count(), 1); + let old = named(square(), "old"); + let a = Document::parse(&old).unwrap(); + let new = named(triangle(), "old"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(100), + "Different\nCells:\n* CHIP\n 1/0 (WG): 1 polygon changed\n" + ); assert!(diff.layer_names.is_empty()); + + let new = named(triangle(), "new"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + let text = diff.text(100); + assert!(text.contains("1/0: name \"WG\"\nCells:\n")); + assert!(!text.contains("(WG)")); + assert!(diff.text(2).ends_with(" 1/0 (WG): 1 polygon changed\n")); + let show = DisplayFilter { + cell: Some("leaf".into()), + layer: None, + }; + assert!( + diff.text_filtered(100, &show) + .ends_with(" ~ 1/0 (WG): label text \"old\" -> \"new\"\n") + ); + + let names = corpus() + .into_iter() + .find(|case| case.name == "layer name only") + .unwrap(); + let old = Document::parse(&names.old).unwrap(); + let new = Document::parse(&names.new).unwrap(); + assert!( + Comparison::between(&old, &new) + .text(0) + .contains("1/0: name \"WG\" -> \"CORE:WG\"") + ); } #[test] -fn an_added_empty_document_is_still_a_presence_change() { +fn library_presence_changes_distinguish_empty_and_absent_revisions() { let bytes = layout(&[]); let doc = Document::parse(&bytes).unwrap(); - let diff = Comparison::between(&doc.empty_like(), &doc); - assert_eq!(diff.exit_code(), 1); - assert!(diff.text(0).contains("library presence:")); - assert!(!diff.text(0).contains("equal under")); + let absent = doc.empty_like(); + let added = Comparison::between(&absent, &doc); + assert_eq!(added.outcome().exit_code(), 1); + assert!(added.text(0).contains("library added\n")); + assert!(added.text(0).starts_with("Different · 1 library change\n")); + assert_eq!( + added.render_text(0, &Default::default(), false), + "Different\n" + ); + let removed = Comparison::between(&doc, &absent); + assert_eq!(removed.outcome().exit_code(), 1); + assert!(removed.text(0).contains("library removed\n")); } #[test] fn cli_scope_and_display_filters_have_distinct_exit_behavior() { - let directory = std::env::temp_dir().join(format!("klamath-scope-{}", std::process::id())); + let directory = std::env::temp_dir().join(format!("pheon-scope-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); let old = directory.join("old.gds"); let new = directory.join("new.gds"); @@ -1170,7 +2369,7 @@ fn cli_scope_and_display_filters_have_distinct_exit_behavior() { (vec!["--show-layer", "99/0"], 1), (vec!["--top", "missing"], 2), ] { - let output = Command::new(env!("CARGO_BIN_EXE_klamath")) + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) .arg("diff") .args(&options) .arg(&old) @@ -1185,7 +2384,7 @@ fn cli_scope_and_display_filters_have_distinct_exit_behavior() { ); } fs::write(&old, layout(&[])).unwrap(); - let output = Command::new(env!("CARGO_BIN_EXE_klamath")) + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) .args(["diff", "--top", "CHIP"]) .arg(&old) .arg(&new) @@ -1195,6 +2394,85 @@ fn cli_scope_and_display_filters_have_distinct_exit_behavior() { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn cli_resolves_layer_names_before_applying_scope_and_display_filters() { + let directory = std::env::temp_dir().join(format!("pheon-named-layer-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let fixture = |shape, text, name| { + layout(&[ + ("CHIP", vec![shape, label((2, 0), text)]), + ( + "legend", + vec![label((900, 7), LAYER_SENTINEL), label((900, 7), name)], + ), + ]) + }; + fs::write(&old, fixture(square(), "before", "1:0:WG")).unwrap(); + fs::write(&new, fixture(triangle(), "after", "1:0:CORE")).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap() + }; + for (option, name) in [("--layer", "WG"), ("--show-layer", "CORE")] { + for format in ["text", "json"] { + let named = diff(&[option, name, "--cell", "CHIP", "--format", format]); + let numeric = diff(&[option, "1/0", "--cell", "CHIP", "--format", format]); + assert_eq!( + named.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&named.stderr) + ); + assert_eq!(named.stdout, numeric.stdout); + } + } + + let dump = |layer| { + let output = command() + .arg("dump") + .arg(&old) + .args(["--layer", layer]) + .output() + .unwrap(); + assert!(output.status.success()); + output.stdout + }; + assert_eq!(dump("WG"), dump("1/0")); + + let unknown = diff(&["--show-layer", "wg", "--format", "json"]); + assert_eq!(unknown.status.code(), Some(2)); + let report: serde_json::Value = serde_json::from_slice(&unknown.stdout).unwrap(); + assert_eq!(report["diagnostics"][0]["code"], "command_error"); + assert_eq!( + report["diagnostics"][0]["message"], + "--show-layer unknown layer name \"wg\"" + ); + + fs::write(&new, fixture(triangle(), "after", "2:0:WG")).unwrap(); + let ambiguous = diff(&["--layer", "WG"]); + assert_eq!(ambiguous.status.code(), Some(2)); + assert_eq!( + String::from_utf8(ambiguous.stderr).unwrap().trim_end(), + "pheon: --layer ambiguous layer name \"WG\" (choose 1/0, 2/0)" + ); + assert_eq!(diff(&["--layer", "1/0"]).status.code(), Some(1)); +} + #[test] fn large_polygons_and_billion_instance_arrays_stay_compact() { let mut points = vec![[0, 0], [0, 1]]; @@ -1213,8 +2491,8 @@ fn large_polygons_and_billion_instance_arrays_stay_compact() { 32767 * 32767 ); assert_eq!(doc.cells["CHIP"].objects.len(), 1); - assert!(hierarchy(&doc, &doc, None, false, 8).len() < 200); - assert!(Comparison::between(&doc, &doc).is_equal()); + assert!(doc.hierarchy(false, 8).len() < 200); + assert!(Comparison::between(&doc, &doc).outcome().is_equal()); } #[test] @@ -1241,7 +2519,7 @@ fn deep_hierarchy_comparison_and_rendering_are_iterative() { let a = Document::parse(&a).unwrap(); let b = Document::parse(&b).unwrap(); let diff = Comparison::between(&a, &b); - let tree = hierarchy(&a, &b, Some(&diff), false, 5); + let tree = diff.hierarchy(false, 5); assert_eq!(diff.cells.len(), 1); assert!(tree.contains("children omitted")); assert!(tree.len() < 500); diff --git a/tests/review/reports.rs b/tests/review/reports.rs index c428818..71f06cd 100644 --- a/tests/review/reports.rs +++ b/tests/review/reports.rs @@ -1,7 +1,8 @@ use super::*; -use klamath_rs_ext::review::{ - DisplayFilter, Scope, +use pheon::review::{ + Diagnostic, Outcome, RevisionPair, Side, report::{ReportInputs, ReportOptions, failure_report}, + selection::{DisplayFilter, Scope}, }; use serde_json::{Value, json}; @@ -15,7 +16,7 @@ fn inputs() -> ReportInputs { fn report(old: &[u8], new: &[u8], options: &ReportOptions) -> Value { let a = Document::parse(old).unwrap(); let b = Document::parse(new).unwrap(); - Comparison::between(&a, &b).json_report(&a, &b, &inputs(), options) + Comparison::between(&a, &b).json_report(&inputs(), options) } fn unknown(geometry: Vec) -> Vec { @@ -26,12 +27,186 @@ fn unknown(geometry: Vec) -> Vec { layout(&[("CHIP", vec![object])]) } +#[test] +fn adaptive_pair_rebases_diagnostics_and_keeps_metadata_and_graph_state() { + let mut stable = vec![square(); 512]; + stable.extend([ + reference("leaf", 0, 2), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "PHEON_LAYER_NAMES_V2"), + ]); + let mut unsupported = square(); + unsupported.truncate(unsupported.len() - 4); + record(&mut unsupported, 0x7f02, &[0, 42]); + record(&mut unsupported, RTAG_ENDEL, &[]); + let old = layout(&[ + ("leaf", vec![]), + ("changed", vec![square()]), + ("stable", stable.clone()), + ("unsupported", vec![unsupported.clone()]), + ]); + let mut new = layout(&[ + ("unsupported", vec![unsupported]), + ("stable", stable), + ("changed", vec![triangle()]), + ]); + let stable_offset = Document::parse(&new).unwrap().cells["stable"].offset; + new[stable_offset + 4..stable_offset + 6].copy_from_slice(&2026_u16.to_be_bytes()); + + let full = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let adaptive = RevisionPair::parse(&old, &new).unwrap().compare(); + let options = ReportOptions { + hierarchy: true, + ..ReportOptions::default() + }; + let actual = adaptive.json_report(&inputs(), &options); + assert_eq!(actual, full.json_report(&inputs(), &options)); + assert_eq!(actual["layer_legend"][0]["name"], "WG"); + assert_eq!(actual["status"], "incomplete"); + assert!( + actual["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|issue| { issue["code"] == "unresolved_reference" && issue["side"] == "new" }) + ); + let text = RevisionPair::parse(&old, &new).unwrap().compare_text(); + assert_eq!( + text.render_text(100, &DisplayFilter::default(), true), + full.text(100) + ); + assert_eq!(text.hierarchy(false, 8), full.hierarchy(false, 8)); + assert_eq!(text.hierarchy(true, 8), full.hierarchy(true, 8)); +} + +#[test] +fn adaptive_pair_prioritizes_old_structure_errors_over_new_frame_errors() { + let mut old = layout(&[("CHIP", vec![square()])]); + let name_offset = Document::parse(&old) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_LIBNAME) + .unwrap() + .offset; + old[name_offset + 2] = 0x7f; + let error = RevisionPair::parse(&old, b"bad").unwrap_err(); + assert_eq!(error.side, Some(Side::Old)); + assert!(error.message.contains("LIBNAME and UNITS")); +} + +#[test] +fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { + let comparison = { + let mut old_bytes = shared(square()); + let mut new_bytes = shared(triangle()); + let old = Document::parse(&old_bytes).unwrap(); + let new = Document::parse(&new_bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + drop(old); + drop(new); + old_bytes.fill(0); + new_bytes.fill(0); + comparison + }; + assert_eq!(comparison.outcome(), Outcome::Different); + assert!(comparison.old.present && comparison.new.present); + assert_eq!(comparison.old.units.as_ref().unwrap().len(), 16); + let mut actual = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + ..Default::default() + }, + ); + actual["producer"]["version"] = json!("test"); + let expected: Value = + serde_json::from_str(include_str!("../golden/report-geometry.json")).unwrap(); + assert_eq!(actual, expected); + assert_eq!( + comparison.hierarchy(false, 8), + include_str!("../golden/review-hierarchy.txt") + ); + assert_eq!( + comparison.hierarchy(true, 8), + include_str!("../golden/review-hierarchy.mmd") + ); + // Display choices affect evidence, not the captured outcome or full graph. + let limited = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + max_objects: Some(0), + ..Default::default() + }, + ); + assert_eq!(limited["hierarchy"], actual["hierarchy"]); + assert_eq!(limited["summary"], actual["summary"]); + assert_eq!(limited["exit_code"], 1); + assert!( + comparison + .text(0) + .contains("Omitted · 2 object changes over the limit") + ); +} + +#[test] +fn comparison_diagnostics_capture_both_sides_and_scope_mismatch_once() { + let comparison = { + let bytes = unknown(square()); + let old = Document::parse(&bytes) + .unwrap() + .selected( + &Scope { + cell: Some("CHIP".to_string()), + ..Default::default() + }, + false, + ) + .unwrap(); + let mut new = Document::parse(&bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + // Changes to source documents cannot alter an existing result. + new.scope = old.scope.clone(); + new.diagnostics.clear(); + comparison + }; + assert_eq!( + comparison.outcome(), + Outcome::Incomplete { different: false } + ); + assert_eq!(comparison.diagnostics.len(), 3); + assert_eq!(comparison.diagnostics[0].side, Some(Side::Old)); + assert_eq!(comparison.diagnostics[1].side, Some(Side::New)); + assert_eq!(comparison.diagnostics[2].code, "scope_mismatch"); + assert_eq!(comparison.diagnostics[2].side, None); + let report = comparison.json_report(&inputs(), &ReportOptions::default()); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["result"], "unknown"); + assert_eq!(report["exit_code"], 2); + assert_eq!(report["scope"]["cell"], "CHIP"); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 3); + assert_eq!(report["diagnostics"][0]["side"], "old"); + assert_eq!(report["diagnostics"][1]["side"], "new"); + assert_eq!(report["diagnostics"][2]["code"], "scope_mismatch"); + let text = comparison.text(0); + assert!(text.starts_with("Incomplete\nScope cell=\"CHIP\"\nDiagnostics\n")); + let scope = text.find("\n comparison scopes differ").unwrap(); + let old = text.find("\n old:").unwrap(); + let new = text.find("\n new:").unwrap(); + assert!(scope < old && old < new); +} + #[test] fn reports_preserve_fixture_results_totals_and_replacement_links() { for pair in corpus() { let r = report(&pair.old, &pair.new, &ReportOptions::default()); assert_eq!(r["schema_version"], 1); - assert_eq!(r["comparison_profile"], "klamath-object-v2"); + assert_eq!(r["comparison_profile"], "pheon-structural-v1"); assert_eq!(r["producer"]["version"], env!("CARGO_PKG_VERSION")); assert_eq!(r["status"], "complete", "{}", pair.name); assert_eq!( @@ -110,12 +285,12 @@ fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { let bytes = if location == "CHIP" { layout(&[( "CHIP", - vec![square(), label((900, 7), "KLAMATH_LAYER_NAMES_V2")], + vec![square(), label((900, 7), "PHEON_LAYER_NAMES_V2")], )]) } else { layout(&[ ("CHIP", vec![square()]), - ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), ]) }; let source = Document::parse(&bytes).unwrap().diagnostics[0].offset; @@ -159,10 +334,10 @@ fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { assert_eq!(diagnostic.layer, Some((900, 7))); assert_eq!(diagnostic.offset, source); let diff = Comparison::between(&doc, &doc); - assert_eq!(diff.exit_code(), 2); - assert!(!diff.is_equal()); - assert!(diff.text(0).contains("INCOMPLETE")); - let json = diff.json_report(&doc, &doc, &inputs(), &ReportOptions::default()); + assert_eq!(diff.outcome().exit_code(), 2); + assert!(!diff.outcome().is_equal()); + assert!(diff.text(0).starts_with("Incomplete\n")); + let json = diff.json_report(&inputs(), &ReportOptions::default()); assert_eq!(json["status"], "incomplete"); assert_eq!(json["result"], "unknown"); assert_eq!(json["exit_code"], 2); @@ -224,6 +399,33 @@ fn json_is_unbounded_by_default_and_explicit_filters_do_not_change_totals() { assert_eq!(filtered["cell_changes"][0]["added"], json!([])); } +#[test] +fn streamed_report_keeps_object_ids_and_replacements_across_layer_filter_gaps() { + let a = layout(&[("CHIP", vec![square(), label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let options = ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + max_objects: Some(2), + ..Default::default() + }; + let mut bytes = Vec::new(); + diff.write_json_report(&mut bytes, &inputs(), &options) + .unwrap(); + let actual: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(actual, diff.json_report(&inputs(), &options)); + assert_eq!(actual["report_selection"]["filtered_objects"], "2"); + let cell = &actual["cell_changes"][0]; + assert_eq!(cell["added"].as_array().unwrap().len(), 1); + assert_eq!(cell["removed"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"][0]["removed"], "c0:old:1"); + assert_eq!(cell["replacements"][0]["added"], "c0:new:1"); +} + #[test] fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { let mut a = reference("leaf", 0, 1); @@ -250,6 +452,21 @@ fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { b[within..within + 8].copy_from_slice(&0x4201000000000000_u64.to_be_bytes()); let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + let before = Document::parse(&a).unwrap(); + let after = Document::parse(&b).unwrap(); + assert!(before.canonical_dump().contains("mag=1.0 property")); + assert!( + after + .canonical_dump() + .contains("mag=1.0 raw=0x4201000000000000") + ); + let text = Comparison::between(&before, &after).text(100); + assert!(text.contains("mag 1.0 raw=0x4110000000000000 -> 1.0 raw=0x4201000000000000")); + let omitted = layout(&[("CHIP", vec![reference("leaf", 0, 1)]), ("leaf", vec![])]); + let omitted = Document::parse(&omitted).unwrap(); + let text = Comparison::between(&before, &omitted).text(100); + assert!(text.contains("mag 1.0 -> absent")); + assert!(!text.contains("raw=")); let r = report(&a, &b, &ReportOptions::default()); let replacement = &r["cell_changes"][0]["replacements"][0]; assert_eq!(replacement["match_rule"], "unique_kind_target"); @@ -339,7 +556,7 @@ fn hierarchy_reports_shared_ancestors_and_symbolic_arrays() { .unwrap() .selected(&scope, false) .unwrap(); - let r = Comparison::between(&doc, &doc).json_report(&doc, &doc, &inputs(), &options); + let r = Comparison::between(&doc, &doc).json_report(&inputs(), &options); assert_eq!(r["scope"]["cell"], "CHIP"); assert_eq!( r["hierarchy"]["nodes"] @@ -369,6 +586,14 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { let r = report(&a, &b, &ReportOptions::default()); assert_eq!(r["cell_changes"][0]["before"], Value::Null); assert_eq!(r["cell_changes"][0]["after"]["attributes"], json!([])); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); + let removed = report(&b, &a, &ReportOptions::default()); + assert_eq!( + removed["cell_changes"][0]["before"]["attributes"], + json!([]) + ); + assert_eq!(removed["cell_changes"][0]["after"], Value::Null); + assert_eq!(removed["cell_changes"][0]["attributes_changed"], true); let mut b = b; let end = Document::parse(&b).unwrap().cells["empty"].end_offset; let mut attribute = Vec::new(); @@ -379,15 +604,11 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { r["cell_changes"][0]["after"]["attributes"][0]["value"], json!([1]) ); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); assert_eq!(r["inputs"]["old"]["units"]["encoding"], "real8"); let doc = Document::parse(&a).unwrap(); let absent = doc.empty_like(); - let r = Comparison::between(&absent, &doc).json_report( - &absent, - &doc, - &inputs(), - &ReportOptions::default(), - ); + let r = Comparison::between(&absent, &doc).json_report(&inputs(), &ReportOptions::default()); assert_eq!(r["inputs"]["old"]["present"], false); assert_eq!(r["inputs"]["old"]["units"], Value::Null); assert_eq!( @@ -401,11 +622,8 @@ fn library_attributes_empty_cells_units_and_absence_are_explicit() { .find(|r| r.tag == RTAG_UNITS) .unwrap() .offset; - b[units + 12..units + 20].copy_from_slice( - &klamath_rs_ext::basic::encode_real8(2e-9) - .unwrap() - .to_be_bytes(), - ); + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); let r = report(&a, &b, &ReportOptions::default()); assert_eq!(r["library_changes"][0]["tag"], RTAG_UNITS); assert_ne!(r["inputs"]["old"]["units"], r["inputs"]["new"]["units"]); @@ -450,7 +668,7 @@ fn reports_keep_extreme_coordinates_unsigned_layers_and_timestamp_equality() { #[test] fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { use std::os::unix::ffi::OsStringExt; - let directory = std::env::temp_dir().join(format!("klamath json paths {}", std::process::id())); + let directory = std::env::temp_dir().join(format!("pheon json paths {}", std::process::id())); fs::create_dir_all(&directory).unwrap(); struct Cleanup(PathBuf); impl Drop for Cleanup { @@ -461,7 +679,7 @@ fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { let _cleanup = Cleanup(directory.clone()); let path = directory.join(std::ffi::OsString::from_vec(b"file\xff.gds".to_vec())); fs::write(&path, layout(&[])).unwrap(); - let output = Command::new(env!("CARGO_BIN_EXE_klamath")) + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) .arg("diff") .args(["--format", "json", "--"]) .arg(&path) @@ -482,17 +700,13 @@ fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { for name in ["--format", "json"] { fs::write(directory.join(name), layout(&[])).unwrap(); } - let output = Command::new(env!("CARGO_BIN_EXE_klamath")) + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) .current_dir(&directory) .args(["diff", "--", "--format", "json"]) .output() .unwrap(); assert_eq!(output.status.code(), Some(0)); - assert!( - String::from_utf8(output.stdout) - .unwrap() - .starts_with("profile ") - ); + assert_eq!(output.stdout, b"Equal\n"); } #[test] @@ -530,9 +744,7 @@ fn machine_reports_match_versioned_goldens() { &inputs(), &Scope::default(), &ReportOptions::default(), - "invalid_input", - "truncated record", - Some("new"), + &Diagnostic::new("invalid_input", "truncated record", Some(Side::New)), ), ), ]; @@ -540,7 +752,7 @@ fn machine_reports_match_versioned_goldens() { actual["producer"]["version"] = json!("test"); let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join(format!("tests/golden/report-{name}.json")); - if std::env::var_os("KLAMATH_UPDATE_REPORT_GOLDENS").is_some() { + if std::env::var_os("PHEON_UPDATE_REPORT_GOLDENS").is_some() { fs::write(&path, serde_json::to_string_pretty(&actual).unwrap() + "\n").unwrap(); } let expected: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); @@ -550,7 +762,7 @@ fn machine_reports_match_versioned_goldens() { #[test] fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { - let directory = std::env::temp_dir().join(format!("klamath json {}", std::process::id())); + let directory = std::env::temp_dir().join(format!("pheon json {}", std::process::id())); fs::create_dir_all(&directory).unwrap(); struct Cleanup(PathBuf); impl Drop for Cleanup { @@ -564,7 +776,7 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { fs::write(&a, shared(square())).unwrap(); fs::write(&b, shared(triangle())).unwrap(); let command = |extra: &[&str]| { - let output = Command::new(env!("CARGO_BIN_EXE_klamath")) + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) .arg("diff") .arg(&a) .arg(&b) @@ -580,9 +792,43 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { ); report }; + let check_text_failure = |report: &Value, extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(extra) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + format!( + "pheon: {}\n", + report["diagnostics"][0]["message"].as_str().unwrap() + ) + ); + }; let full = command(&["--hierarchy", "graph"]); assert_eq!(full["result"], "different"); assert!(full["hierarchy"].is_object()); + let large = |mut bytes: Vec| { + bytes.resize(3 * 1024 * 1024, 0); + bytes + }; + let mut large_old = large(shared(square())); + let mut large_new = large(shared(triangle())); + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&["--hierarchy", "graph"]), full); + *large_old.last_mut().unwrap() = 1; + *large_new.last_mut().unwrap() = 1; + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&[])["diagnostics"][0]["side"], "old"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); let limited = command(&["--max-changes", "0"]); assert_eq!(limited["summary"], full["summary"]); assert_eq!(limited["report_selection"]["emitted_objects"], "0"); @@ -597,14 +843,35 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { assert_eq!(failure["status"], "failed"); assert_eq!(failure["result"], "unknown"); assert_eq!(failure["summary"], Value::Null); + if extra.first() == Some(&"--top") { + assert_eq!(failure["diagnostics"][0]["code"], "invalid_scope"); + assert_eq!(failure["diagnostics"][0]["side"], Value::Null); + check_text_failure(&failure, extra); + } } fs::write(&b, b"invalid").unwrap(); let invalid = command(&[]); assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); assert_eq!(invalid["diagnostics"][0]["side"], "new"); assert_eq!(invalid["inputs"]["new"]["path"], b.to_str().unwrap()); + check_text_failure(&invalid, &[]); + fs::write(&a, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "old"); + check_text_failure(&invalid, &[]); + fs::write(&a, shared(square())).unwrap(); fs::remove_file(&b).unwrap(); - assert_eq!(command(&[])["diagnostics"][0]["code"], "io_error"); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "new"); + check_text_failure(&missing, &[]); + fs::remove_file(&a).unwrap(); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "old"); + check_text_failure(&missing, &[]); + fs::write(&a, shared(square())).unwrap(); fs::write(&b, fs::read(&a).unwrap()).unwrap(); assert_eq!(command(&[])["result"], "equal"); fs::write(&b, unknown(square())).unwrap(); @@ -613,7 +880,7 @@ fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { &b, layout(&[ ("CHIP", vec![square()]), - ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), ]), ) .unwrap(); diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs index 35abc2e..cd06348 100644 --- a/tests/selective_reads.rs +++ b/tests/selective_reads.rs @@ -1,12 +1,12 @@ mod common; -use arrow::array::{ - Array, BooleanArray, Float64Array, Int32Array, ListArray, StringArray, StructArray, +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; +use arrow_array::{ + Array, BooleanArray, Float64Array, Int16Array, Int32Array, ListArray, StringArray, StructArray, UInt32Array, UInt64Array, }; -use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; use common::{library_header, record}; -use klamath_rs_ext::{ +use pheon::{ basic::encode_real8, last_error_message, library::{read_library, read_selected_library, scan_library}, @@ -122,11 +122,41 @@ fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { #[test] fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { + let check_array_error = |bytes: &[u8], message: &str| { + let ranges = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for result in [ + read_library(bytes), + scan_library(bytes), + read_selected_library(bytes, &ranges), + ] { + let (_, error) = result.unwrap_err(); + assert!(matches!(error, pheon::basic::ErrType::Failed(reason) + if reason.contains(message))); + } + assert!( + pheon::review::Document::parse(bytes) + .unwrap_err() + .contains(message) + ); + }; for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); - assert!(read_library(&bytes).is_err()); - assert!(scan_library(&bytes).is_err()); + check_array_error(&bytes, "AREF column and row counts must be positive"); } + + let mut bytes = aref_fixture([0, 0], [10, 0], [0, 10], [1, 2]); + let xy_offset = pheon::review::Document::parse(&bytes) + .unwrap() + .records + .iter() + .find(|record| record.tag == records::RTAG_XY) + .unwrap() + .offset; + let mut replacement = Vec::new(); + record(&mut replacement, records::RTAG_XY, &[0; 16]); + bytes.splice(xy_offset..xy_offset + 28, replacement); + check_array_error(&bytes, "AREF needs exactly three XY coordinate pairs"); + let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); assert!(read_library(&bytes).is_err()); assert!(scan_library(&bytes).is_err()); @@ -173,10 +203,8 @@ fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { }, -1 ); - let path = std::env::temp_dir().join(format!( - "klamath-fractional-aref-{}.gds", - std::process::id() - )); + let path = + std::env::temp_dir().join(format!("pheon-fractional-aref-{}.gds", std::process::id())); struct Cleanup(std::path::PathBuf); impl Drop for Cleanup { fn drop(&mut self) { @@ -203,9 +231,9 @@ fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { record(&mut bytes, records::RTAG_STRNAME, b"leaf"); record(&mut bytes, records::RTAG_ENDSTR, &[]); record(&mut bytes, records::RTAG_ENDLIB, &[]); - let doc = klamath_rs_ext::review::Document::parse(&bytes).unwrap(); + let doc = pheon::review::Document::parse(&bytes).unwrap(); assert_eq!( - klamath_rs_ext::review::Comparison::between(&doc, &doc).exit_code(), + pheon::review::Comparison::between(&doc, &doc).outcome().exit_code(), 0 ); } @@ -471,6 +499,173 @@ fn selections_support_subsets_reordering_and_empty_results() { assert_eq!(list_struct(&empty, "cells", 0).len(), 0); } +#[test] +fn id_tables_preserve_encounter_order_and_selection_local_ids() { + let mut bytes = library_header(); + for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha\0", 9, 4)] { + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); + if name == "zeta" { + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"alpha\0"); + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &datatype.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + } + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, full) = read_library(&bytes).unwrap(); + let cell_ranges = ranges(&scan); + + let check = |library: &StructArray, names: &[&str], layers: &[u32], ids: &[u32]| { + let values = |name| { + library + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + }; + let name_values = values("cell_names"); + assert_eq!( + name_values + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(Option::unwrap) + .collect::>(), + names + ); + let cells = list_struct(library, "cells", 0); + assert_eq!( + cells + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + ids + ); + if library.column_by_name("layers").is_some() { + let layer_values = values("layers"); + assert_eq!( + layer_values + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + layers + ); + } + }; + let layer_a = (9 << 16) | 4; + let layer_b = (2 << 16) | 8; + check( + &full, + &["zeta", "alpha", "middle"], + &[layer_a, layer_b], + &[0, 2, 1], + ); + check(&scan, &["zeta", "alpha", "middle"], &[], &[0, 2, 1]); + let (_, all_selected) = read_selected_library(&bytes, &cell_ranges).unwrap(); + assert_eq!(all_selected, full); + let (_, subset) = read_selected_library(&bytes, &cell_ranges[1..]).unwrap(); + check(&subset, &["middle", "alpha"], &[layer_b, layer_a], &[0, 1]); + let (_, reversed) = + read_selected_library(&bytes, &[cell_ranges[2], cell_ranges[1], cell_ranges[0]]).unwrap(); + check( + &reversed, + &["alpha", "middle", "zeta"], + &[layer_a, layer_b], + &[0, 1, 2], + ); +} + +#[test] +fn streamed_and_buffered_properties_preserve_duplicates_order_and_empty_lists() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for (kind, properties) in [ + (records::RTAG_BOUNDARY, true), + (records::RTAG_PATH, true), + (records::RTAG_PATH, false), + ] { + record(&mut bytes, kind, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + let vertices = if kind == records::RTAG_BOUNDARY { + xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]) + } else { + xy(&[0, 0, 10, 10]) + }; + record(&mut bytes, records::RTAG_XY, &vertices); + if properties { + for (key, value) in [(7_i16, b"first\0".as_slice()), (7, b"second"), (9, b"")] { + record(&mut bytes, records::RTAG_PROPATTR, &key.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, value); + } + } + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + + let (remaining, full) = read_library(&bytes).unwrap(); + assert!(remaining.is_empty()); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(selected, full); + let cells = list_struct(&full, "cells", 0); + let boundaries = list_struct(&cells, "boundary_props", 0); + let paths = list_struct(&cells, "paths", 0); + assert_eq!(boundaries.len(), 1); + assert_eq!(paths.len(), 2); + for elements in [&boundaries, &paths] { + let properties = list_struct(elements, "properties", 0); + let keys = properties + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(keys.values().as_ref(), &[7, 7, 9]); + let values = properties + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + values.iter().collect::>(), + [Some("first"), Some("second"), Some("")] + ); + } + let empty_properties = paths + .column_by_name("properties") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!empty_properties.is_null(1)); + assert_eq!(empty_properties.value_length(1), 0); +} + #[test] fn selected_ranges_must_cover_exactly_one_complete_cell() { let bytes = fixture(); diff --git a/tests/writing.rs b/tests/writing.rs index 0af060f..dc07994 100644 --- a/tests/writing.rs +++ b/tests/writing.rs @@ -1,7 +1,8 @@ -use arrow::array::Array; -use klamath_rs_ext::{ +use arrow_array::Array; +use pheon::{ + basic::{pack_bitarray, pack_int2, pack_int4, pack_real8}, library::{FileHeader, read_library}, - record::{Record, RecordHeader}, + record::{Int2Array, Int4Array, Record, RecordData, RecordHeader}, records, }; use std::io::{self, Write}; @@ -18,7 +19,7 @@ fn record_headers_include_their_own_size() { data_size: u16::MAX - 4, tag: 0x1003, }; - assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]); + assert_eq!(maximum.pack_into().unwrap(), [0xff, 0xff, 0x10, 0x03]); } #[test] @@ -37,6 +38,52 @@ fn integer_array_records_round_trip() { assert_eq!(parsed.collect::>(), values); } +#[test] +fn scalar_packers_write_only_their_prefix_and_check_short_buffers() { + type Case = (fn(&mut [u8]), &'static [u8]); + let cases: &[Case] = &[ + (|bytes| pack_bitarray(bytes, &[true; 16]), &[0xff, 0xff]), + (|bytes| pack_int2(bytes, -2), &[0xff, 0xfe]), + (|bytes| pack_int4(bytes, -2), &[0xff, 0xff, 0xff, 0xfe]), + ( + |bytes| pack_real8(bytes, 1.0).unwrap(), + &[0x41, 0x10, 0, 0, 0, 0, 0, 0], + ), + ]; + for (pack, expected) in cases { + let mut bytes = [0xa5; 16]; + pack(&mut bytes); + assert_eq!(&bytes[..expected.len()], *expected); + assert!(bytes[expected.len()..].iter().all(|&byte| byte == 0xa5)); + let mut short = vec![0xa5; expected.len() - 1]; + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| pack(&mut short))).is_err() + ); + assert!(short.iter().all(|&byte| byte == 0xa5)); + } + // Invalid REAL8 values are rejected before accessing the output buffer. + assert!(pack_real8(&mut [], f64::INFINITY).is_err()); +} + +#[test] +fn array_packers_reject_nonexact_buffers_before_writing() { + for width in [2, 4] { + let required = 2 * width; + for size in [required - 1, required + 1, required + width] { + let mut bytes = vec![0xa5; size]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if width == 2 { + Int2Array::pack_into(&mut bytes, &[-1, 1]); + } else { + Int4Array::pack_into(&mut bytes, &[-1, 1]); + } + })); + assert!(result.is_err()); + assert!(bytes.iter().all(|&byte| byte == 0xa5)); + } + } +} + #[test] fn string_records_pad_only_when_needed_and_support_empty_strings() { for value in [b"".as_slice(), b"odd", b"even"] { @@ -88,4 +135,58 @@ fn oversized_records_fail_before_writing() { let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); assert!(bytes.is_empty()); + + // Header packing and direct header writes reject every overflowing size too. + for data_size in u16::MAX - 3..=u16::MAX { + let header = RecordHeader { + tag: records::RTAG_XY, + data_size, + }; + assert_eq!( + header.pack_into().unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + header.write(&mut bytes).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(bytes.is_empty()); + } +} + +#[test] +fn invalid_record_sizes_preserve_the_output() { + let mut bytes = vec![0xa5]; + for count in [0, 1, 3] { + let error = records::COLROW::write(&mut bytes, &vec![2; count]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } + let error = records::COLROW::write_header(&mut bytes, 2).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + + macro_rules! check_metadata { + ($record:ident) => { + for count in [43, 44, 45] { + let payload = vec![b'x'; count]; + let mut bytes = vec![0xa5]; + let result = records::$record::write(&mut bytes, &payload); + if count == 45 { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } else { + // Validation uses the padded record size, not the input length. + assert_eq!(result.unwrap(), 48); + assert_eq!(records::$record::read(&bytes[1..]).unwrap().1, payload); + } + } + let error = records::$record::write_header(&mut bytes, 46).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + }; + } + check_metadata!(REFLIBS); + check_metadata!(FONTS); + check_metadata!(ATTRTABLE); } diff --git a/tools/build_manylinux.py b/tools/build_manylinux.py new file mode 100644 index 0000000..c90c7da --- /dev/null +++ b/tools/build_manylinux.py @@ -0,0 +1,104 @@ +"""Build Linux release wheels using Podman storage in this checkout.""" + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +CACHE = ROOT / ".cache/wheels" + + +def local_environment() -> dict[str, str]: + podman = CACHE / "podman" + scratch = CACHE / "tmp" + podman.mkdir(parents=True, exist_ok=True) + scratch.mkdir(exist_ok=True) + storage = podman / "storage.conf" + graphroot = json.dumps(str(podman / "storage"), ensure_ascii=False) + runroot = json.dumps(str(podman / "run"), ensure_ascii=False) + config = f"[storage]\ngraphroot = {graphroot}\nrunroot = {runroot}\n" + # Replace the config atomically so simultaneous Podman commands can read it. + if not storage.exists() or storage.read_text(encoding="utf-8") != config: + temporary = podman / f"storage.{os.getpid()}.conf" + temporary.write_text(config, encoding="utf-8") + temporary.replace(storage) + + env = dict(os.environ, CONTAINERS_STORAGE_CONF=str(storage), TMPDIR=str(scratch)) + for key in ("CONTAINER_HOST", "CONTAINER_CONNECTION"): + env.pop(key, None) + return env + + +def build(sdist: Path) -> None: + env = local_environment() + image = (ROOT / "contrib/wheels/manylinux-image.txt").read_text(encoding="utf-8").strip() + + # Use the same Podman invocation as cibuildwheel to check the active store. + graphroot = subprocess.check_output( + ["podman", "info", "--format", "{{.Store.GraphRoot}}"], env=env, text=True, + ).strip() + if Path(graphroot).resolve() != (CACHE / "podman/storage").resolve(): + raise SystemExit(f"Expected the checkout's Podman store, found {graphroot}") + + subprocess.run( + ["cargo", "fetch", "--locked", "--target", "x86_64-unknown-linux-gnu"], + cwd=ROOT, env=env, check=True, + ) + toolchain = subprocess.check_output( + ["rustc", "--print", "sysroot"], cwd=ROOT, env=env, text=True, + ).strip() + registry = Path(env.get("CARGO_HOME", Path.home() / ".cargo")).resolve() / "registry" + for directory in ("cargo/registry", "pip", "target", "cibuildwheel"): + (CACHE / directory).mkdir(parents=True, exist_ok=True) + mounts = [ + f"{toolchain}:/rust-toolchain:ro", + f"{CACHE / 'cargo'}:/pheon-cache/cargo", + f"{registry}:/pheon-cache/cargo/registry:ro", + f"{CACHE / 'pip'}:/pheon-cache/pip", + f"{CACHE / 'target'}:/pheon-cache/target", + ] + create_args = shlex.join([arg for mount in mounts for arg in ("--volume", mount)]) + env.update( + CIBW_CONTAINER_ENGINE=f"podman; disable_host_mount: true; create_args: {create_args}", + CIBW_MANYLINUX_X86_64_IMAGE=image, + CIBW_ENVIRONMENT_LINUX=( + "PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo " + "CARGO_TARGET_DIR=/pheon-cache/target CARGO_NET_OFFLINE=true " + "PIP_CACHE_DIR=/pheon-cache/pip" + ), + CIBW_CACHE_PATH=str(CACHE / "cibuildwheel"), + CIBW_BUILD="cp31{1,2,3,4}-manylinux_x86_64", + CIBW_TEST_COMMAND="python -I {project}/tools/check_installed.py", + ) + env.pop("PHEON_LIBRARY", None) + subprocess.run( + [sys.executable, "-m", "cibuildwheel", "--platform", "linux", "--archs", "x86_64", + "--output-dir", str(sdist.parent), str(sdist)], + cwd=ROOT, env=env, check=True, + ) + + +def main() -> int: + if sys.argv[1:2] == ["podman"]: + return subprocess.run( + ["podman", "--remote=false", *sys.argv[2:]], env=local_environment(), + ).returncode + + parser = argparse.ArgumentParser( + description=__doc__, epilog="Use 'podman ARGS' to inspect the local store.", + ) + parser.add_argument("sdist", type=Path, help="Pheon source archive") + args = parser.parse_args() + if not args.sdist.is_file(): + parser.error(f"source archive does not exist: {args.sdist}") + build(args.sdist.resolve()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/check_gds_corpus.py b/tools/check_gds_corpus.py new file mode 100644 index 0000000..8331145 --- /dev/null +++ b/tools/check_gds_corpus.py @@ -0,0 +1,167 @@ +"""Check a pinned open-source layout corpus against Pheon's installed readers.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tomllib +from urllib.parse import quote + +from pheon import arrow + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_input(path, url, digest, fetch): + if path.is_file(): + data = path.read_bytes() + elif fetch: + data = subprocess.check_output( + ["curl", "-fLsS", "--retry", "2", "--max-time", "60", url] + ) + else: + raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.") + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError(f"SHA-256 mismatch for {path}") + if not path.is_file(): + path.write_bytes(data) + return data + + +def inspect_layout(cli, path, data): + results = {} + reads = { + "full": lambda: arrow.read_path(path), + "scan": lambda: arrow.scan_bytes(data), + } + layouts = {} + for name, read in reads.items(): + try: + layout = read() + layouts[name] = layout + results[name] = { + "status": "ok", + "cells": len(layout.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results[name] = {"status": "failed", "message": str(error)} + + if "scan" in layouts: + cells = layouts["scan"].field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + strict=True, + ) + ) + try: + selected = arrow.read_cells_bytes(data, ranges) + if "full" in layouts and not selected.equals(layouts["full"]): + raise ValueError( + "Reading all scanned cell ranges differs from the full read" + ) + results["selected"] = { + "status": "ok", + "cells": len(selected.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results["selected"] = {"status": "failed", "message": str(error)} + else: + results["selected"] = {"status": "skipped"} + + try: + run = subprocess.run( + [cli, "diff", path, path, "--format", "json"], + capture_output=True, + timeout=60, + ) + except (subprocess.TimeoutExpired, OSError) as error: + results["cli"] = {"status": "failed", "message": str(error)} + return results + try: + report = json.loads(run.stdout) + results["cli"] = { + "status": report["status"], + "result": report["result"], + "exit_code": run.returncode, + "diagnostics": report["diagnostics"], + } + if (report["status"], report["result"], run.returncode) not in { + ("complete", "equal", 0), + ("incomplete", "unknown", 2), + ("failed", "unknown", 2), + }: + results["cli"]["status"] = "failed" + except (ValueError, KeyError, TypeError) as error: + results["cli"] = { + "status": "failed", + "message": str(error), + "exit_code": run.returncode, + "stderr": run.stderr[-4096:].decode("utf-8", errors="replace"), + } + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fetch", action="store_true", help="download missing inputs using curl" + ) + binary = "pheon.exe" if os.name == "nt" else "pheon" + parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary) + parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus") + args = parser.parse_args() + cli = args.cli.resolve() + if not cli.is_file(): + parser.error(f"{cli} is missing. Build the CLI or pass --cli.") + + for name in ("files", "licenses"): + (args.cache / name).mkdir(parents=True, exist_ok=True) + sources = tomllib.loads( + (ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8") + ) + all_results = {} + for key, source in sources.items(): + base = ( + f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}" + ) + read_input( + args.cache / "licenses" / f"{key}-LICENSE", + base + "/LICENSE", + source["license_sha256"], + args.fetch, + ) + for upstream, digest in source["layouts"].items(): + name = f"{key}-{Path(upstream).name}" + path = args.cache / "files" / name + raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch) + data = gzip.decompress(raw) if path.suffix == ".gz" else raw + results = inspect_layout(cli, path, data) + all_results[name] = results + print( + name, + *( + f"{operation}={result['status']}" + for operation, result in results.items() + ), + ) + + report = args.cache / "report.json" + report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8") + failures = sum( + any(result["status"] == "failed" for result in case.values()) + for case in all_results.values() + ) + print( + f"{len(all_results)} layouts checked, {failures} with failures. See {report}." + ) + return int(failures != 0) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_installed.py b/tools/check_installed.py new file mode 100644 index 0000000..437aadb --- /dev/null +++ b/tools/check_installed.py @@ -0,0 +1,26 @@ +"""Run outside the checkout, without PHEON_LIBRARY, after installing a wheel.""" +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +# A separate interpreter checks that PyArrow stays unloaded during packing. +subprocess.run( + [sys.executable, "-I", "-c", """ +import sys +from importlib.metadata import version +import pheon +from pheon._native import library +assert pheon.__version__ == version('pheon') +assert library.parent == __import__('pathlib').Path(pheon.__file__).parent +assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') +assert 'pyarrow' not in sys.modules +"""], + check=True, +) +suite = unittest.defaultTestLoader.discover(str(ROOT / "tests/python")) +if not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(): + raise SystemExit(1)