diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2d14a76 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: Native checks + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + rust: + runs-on: ubuntu-latest + strategy: + matrix: + rust: ['1.88.0', stable] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: rustup toolchain install ${{ matrix.rust }} --profile minimal + - run: cargo +${{ matrix.rust }} test --locked + - run: cargo +${{ matrix.rust }} test --locked --release diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..3a0c0bc --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,123 @@ +name: Package artifacts + +on: + pull_request: + workflow_dispatch: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - run: python -m pip install build + - run: python -m build --sdist + - uses: actions/upload-artifact@v4 + with: + name: source + path: dist/*.tar.gz + + wheels: + needs: sdist + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: source + path: source-dist + - name: Unpack the source release + shell: bash + run: | + mkdir source-build + tar -xzf source-dist/*.tar.gz -C source-build --strip-components=1 + - name: Build and repair Python-independent wheels from the sdist + uses: pypa/cibuildwheel@v4.3.5 + with: + package-dir: source-build + env: + CIBW_BUILD: cp311-* + CIBW_SKIP: '*-musllinux_* *-win32' + CIBW_ARCHS: native + CIBW_ENVIRONMENT_LINUX: 'CARGO_HOME=/tmp/pheon-cargo RUSTUP_HOME=/tmp/pheon-rustup PATH=/tmp/pheon-cargo/bin:$PATH' + CIBW_BEFORE_ALL_LINUX: 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable' + CIBW_TEST_EXTRAS: arrow + CIBW_TEST_COMMAND: 'python -I {project}/tools/check_installed.py' + - name: Check wheel ABI tags + shell: python + run: | + from pathlib import Path + wheels = list(Path('wheelhouse').glob('*.whl')) + assert wheels + assert all('-py3-none-' in p.name for p in wheels), wheels + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse/*.whl + + same-wheel: + needs: wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + python: ['3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse + - name: Install the same artifact on another Python version + shell: bash + run: python -m pip install wheelhouse/*.whl pyarrow + - run: python -I tools/check_installed.py + + cli: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: cargo test --release --locked + - run: cargo build --release --locked --bin pheon + - name: Assemble standalone CLI artifact + shell: bash + run: | + mkdir cli-dist + if [ "$RUNNER_OS" = Windows ]; then + cp target/release/pheon.exe cli-dist/ + else + cp target/release/pheon cli-dist/ + fi + cp LICENSE.md README.md cli-dist/ + cp -R contrib cli-dist/ + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.os }} + path: cli-dist/ diff --git a/.gitignore b/.gitignore index 5d80173..e67c112 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ /target/ **/*.rs.bk -Cargo.lock *.swp *.swo + +__pycache__/ +/build/ +/dist/ +/*.egg-info/ diff --git a/ARROW_SCHEMA.md b/ARROW_SCHEMA.md new file mode 100644 index 0000000..14aa848 --- /dev/null +++ b/ARROW_SCHEMA.md @@ -0,0 +1,154 @@ +# Arrow Schema Documentation + +Pheon's native `pheon_arrow_schema_version()` returns 1 for the full-read, +scan and selected-read layouts documented here. This version is independent +of the C API version, package release and CLI JSON report schema. The stable +Arrow C Data Interface transports these application-specific layouts; consumers +must check both the native API and application schema before reading data. + +This project uses Apache Arrow to represent GDSII (Graphic Data System II) library data. The schema is optimized for memory efficiency and fast access, employing several packed representations for common data types. + +## Top-Level Structure + +The top-level object is a `StructArray` representing the entire GDSII library. + +| Field | Type | Description | +|---|---|---| +| `meters_per_db_unit` | `Float64` | Database unit size in meters. | +| `user_units_per_db_unit` | `Float64` | Number of user units in one database unit. | +| `lib_name` | `Utf8` | The name of the GDSII library. | +| `mod_time` | `FixedSizeList(Int16, 6)` | Last modification time [year, month, day, hour, minute, second]. | +| `acc_time` | `FixedSizeList(Int16, 6)` | Last access time [year, month, day, hour, minute, second]. | +| `cells` | `List(Cell)` | A list of cells (structures) in the library. | +| `cell_names` | `List(Utf8)` | Names of all cells, indexed by their ID. | +| `layers` | `List(UInt32)` | List of packed layer/datatype pairs, indexed by their ID. | + +## Cell Structure + +Each cell in the `cells` list is a `StructArray`. + +| Field | Type | Description | +|---|---|---| +| `id` | `UInt32` | Unique identifier for the cell. | +| `file_offset` | `UInt64` | Byte offset immediately after the cell's `STRNAME` record (the first element, or `ENDSTR` for an empty cell). Identical for full and selected reads. | +| `srefs` | `List(SRef)` | Simple structure references (no properties). | +| `arefs` | `List(ARef)` | Array references (no properties). | +| `sref_props` | `List(SRefProp)` | Structure references with properties. | +| `aref_props` | `List(ARefProp)` | Array references with properties. | +| `rect_batches` | `List(RectBatch)` | Batches of rectangular boundaries (optimized). | +| `boundary_batches` | `List(BoundaryBatch)` | Batches of generic boundaries. | +| `boundary_props` | `List(BoundaryProp)` | Individual boundaries with properties. | +| `paths` | `List(Path)` | Path elements. | +| `nodes` | `List(BoxNode)` | Node elements. | +| `boxes` | `List(BoxNode)` | Box elements. | +| `texts` | `List(Text)` | Text labels. | + +## Hierarchy Scan Schema + +`scan_library` and `scan_bytes` return the same library metadata and `cell_names` +as a full read, with the following `cells` schema and no `layers` field: + +| Field | Type | Description | +|---|---|---| +| `id` | `UInt32` | Cell ID indexing this result's `cell_names`. | +| `struct_start_offset` | `UInt64` | Inclusive byte offset of the `BGNSTR` record. | +| `struct_end_offset` | `UInt64` | Exclusive byte offset immediately after `ENDSTR`. | +| `refs` | `List(ScanRef)` | SREFs and AREFs, including references with properties, sorted by target ID. | + +Each scan reference has `target`, `invert_y`, `scale`, `angle_rad`, `xy`, `xy0`, +`xy1`, and `counts`, using the ARef types and packed encodings described below. +SREFs use zero basis vectors and counts `(1, 1)`. Properties are omitted from the +scan output; geometry and text are skipped without decoding. + +Pass the half-open scan ranges to `read_selected_library` or `read_cells_bytes` +to obtain the full library schema for just those cells. Each range must contain +exactly one complete cell, from `BGNSTR` through `ENDSTR`. Cells appear in the +requested order. Cell and layer IDs are assigned within each result, so use +`cell_names` and `layers` from that result when resolving them; IDs can differ +from a full read or a scan. An empty selection returns metadata and empty lists. +The Rust random-access reader returns the original input slice as its remainder. + +## Element Types + +### Common Packed Encodings + +- **Layer ID (`UInt32`)**: A packed representation of `(layer, datatype)`. + - `layer = (layer_id >> 16) as i16` + - `datatype = (layer_id & 0xFFFF) as i16` +- **XY Coordinate (`UInt64`)**: A packed representation of a single point `(x, y)`. + - `x = (xy >> 32) as i32` + - `y = (xy & 0xFFFFFFFF) as i32` +- **Array Counts (`UInt32`)**: A packed representation of `(columns, rows)`. + - `columns = (counts >> 16) as i16` + - `rows = (counts & 0xFFFF) as i16` + +### Reference Elements (SRef / ARef) + +References point to other cells in the library. + +**SRef (Simple Reference)**: +- `target`: `UInt32` (Cell ID) +- `invert_y`: `Boolean` (Vertical flip) +- `scale`: `Float64` (Magnification) +- `angle_rad`: `Float64` (Rotation in radians) +- `xy`: `UInt64` (Packed origin point) + +**ARef (Array Reference)**: +- Same as SRef, plus: +- `xy0`: `UInt64` (Packed column basis vector increment) +- `xy1`: `UInt64` (Packed row basis vector increment) +- `counts`: `UInt32` (Packed columns and rows) + +Array increments must be exact signed 32-bit integer database-unit values. Full +reads, hierarchy scans, selected reads, and their FFI entry points reject arrays +whose endpoint/count division is fractional or outside that range. They never +round or truncate placements. The review CLI retains original endpoints and can +compare fractional arrays independently of this Arrow representation. + +**SRefProp / ARefProp**: +- Same as above, plus a `properties` field. + +### Boundary Elements + +Boundaries are grouped into batches for efficiency when they share properties or are simple rectangles. + +**RectBatch**: +- `layer`: `UInt32` (Packed Layer ID) +- `rects`: `List(Int32)` (Flattened list of `[min_x, min_y, max_x, max_y]` for each rectangle) + +**BoundaryBatch**: +- `layer`: `UInt32` (Packed Layer ID) +- `vertices`: `List(Int32)` (Flattened list of all vertices for all boundaries in the batch) +- `vertex_offsets`: `List(UInt32)` (Start index for each boundary in the `vertices` list) + +**BoundaryProp**: +- Individual boundary with `properties`. + +### Other Elements + +**Path**: +- `layer`: `UInt32` +- `path_type`: `Int16` (Optional) +- `extension_start`: `Int32` (Optional) +- `extension_end`: `Int32` (Optional) +- `width`: `Int32` (Optional; null when `WIDTH` is absent) +- `xy`: `List(Int32)` (centerline coordinates) +- `properties`: `List(Property)` + +**Text**: +- `layer`: `UInt32` +- `presentation_horiz`: `UInt8` +- `presentation_vert`: `UInt8` +- `presentation_font`: `UInt8` +- `path_type`: `Int16` +- `width`: `Int32` +- `invert_y`: `Boolean` +- `mag`: `Float64` +- `angle_deg`: `Float64` +- `xy`: `UInt64` (Packed position) +- `string`: `Utf8` +- `properties`: `List(Property)` + +**Property**: +- `key`: `Int16` +- `value`: `Utf8` diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..3a525b0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,587 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pheon" +version = "0.3.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..49e7ad5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "pheon" +version = "0.3.0" +authors = ["jan "] +edition = "2024" +rust-version = "1.88" +description = "GDSII parsing, Arrow interchange, and structural layout review" +license = "GPL-3.0-only" +readme = "README.md" +repository = "https://mpxd.net/code/jan/pheon" + + +[lib] +name = "pheon" +crate-type = ["cdylib", "rlib"] + + +[dependencies] +arrow-array = {version = "^54", features = ["ffi"]} +arrow-schema = "^54" +serde_json = "1" + + +[profile.release] +lto = true diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..175443c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,595 @@ +GNU General Public License +========================== + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +* **a)** The work must carry prominent notices stating that you modified it, and giving a +relevant date. +* **b)** The work must carry prominent notices stating that it is released under this +License and any conditions added under section 7. This requirement modifies the +requirement in section 4 to “keep intact all notices”. +* **c)** You must license the entire work, as a whole, under this License to anyone who +comes into possession of a copy. This License will therefore apply, along with any +applicable section 7 additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no permission to license the +work in any other way, but it does not invalidate such permission if you have +separately received it. +* **d)** If the work has interactive user interfaces, each must display Appropriate Legal +Notices; however, if the Program has interactive interfaces that do not display +Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +* **a)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by the Corresponding Source fixed on a +durable physical medium customarily used for software interchange. +* **b)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by a written offer, valid for at least +three years and valid for as long as you offer spare parts or customer support for +that product model, to give anyone who possesses the object code either **(1)** a copy of +the Corresponding Source for all the software in the product that is covered by this +License, on a durable physical medium customarily used for software interchange, for +a price no more than your reasonable cost of physically performing this conveying of +source, or **(2)** access to copy the Corresponding Source from a network server at no +charge. +* **c)** Convey individual copies of the object code with a copy of the written offer to +provide the Corresponding Source. This alternative is allowed only occasionally and +noncommercially, and only if you received the object code with such an offer, in +accord with subsection 6b. +* **d)** Convey the object code by offering access from a designated place (gratis or for +a charge), and offer equivalent access to the Corresponding Source in the same way +through the same place at no further charge. You need not require recipients to copy +the Corresponding Source along with the object code. If the place to copy the object +code is a network server, the Corresponding Source may be on a different server +(operated by you or a third party) that supports equivalent copying facilities, +provided you maintain clear directions next to the object code saying where to find +the Corresponding Source. Regardless of what server hosts the Corresponding Source, +you remain obligated to ensure that it is available for as long as needed to satisfy +these requirements. +* **e)** Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are being +offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +* **a)** Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or +* **b)** Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed by works +containing it; or +* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that +modified versions of such material be marked in reasonable ways as different from the +original version; or +* **d)** Limiting the use for publicity purposes of names of licensors or authors of the +material; or +* **e)** Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or +* **f)** Requiring indemnification of licensors and authors of that material by anyone +who conveys the material (or modified versions of it) with contractual assumptions of +liability to the recipient, for any liability that these contractual assumptions +directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a552f5f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include Cargo.toml Cargo.lock rust-toolchain.toml README.md ARROW_SCHEMA.md LICENSE.md +recursive-include src *.rs +recursive-include pheon *.py *.h py.typed +recursive-include tests *.rs *.py *.json *.txt *.mmd +recursive-include docs *.md *.json +recursive-include contrib * +recursive-include tools *.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..885fc01 --- /dev/null +++ b/README.md @@ -0,0 +1,337 @@ +# Pheon + +A fast GDSII parser and writer in Rust, with Apache Arrow integration for efficient data handling. + +Pheon provides native GDSII codecs and a standalone layout review CLI. The Rust +crate, Python package and executable are all named `pheon`. Python Klamath and +Masque remain separate consumers. The former `klamath_rs_ext` import and +`klamath` executable are not provided as aliases. + +## Overview + +`pheon` is designed to handle large GDSII files with high performance. It uses Apache Arrow as its primary in-memory data representation, allowing for efficient memory usage and easy integration with other tools in the Arrow ecosystem (like pandas, polars, or DuckDB via Python bindings). + +## Documentation + +- [Arrow Schema Documentation](ARROW_SCHEMA.md): Detailed information about the Arrow schema used for GDSII data. + +## Features + +- Fast parsing of GDSII files. +- Efficient memory representation using Apache Arrow. +- Support for common GDSII elements: SREF, AREF, Boundary, Path, Text, etc. +- Optimized "batch" representations for rectangles and boundaries sharing the same layer. +- C-compatible FFI for integration with other languages (e.g., Python). + +REAL8 writing uses normalized GDS encodings and represents every in-range `f64` +exactly. Finite magnitudes below `2^-260` underflow to canonical zero; nonfinite +values and magnitudes at or above `2^252` are rejected by the encoder. + +## Usage + +`read_path` exports a library through the Arrow C Data Interface. Allocate +initialized `ArrowArray` and `ArrowSchema` output structures, then check the +returned status: `0` means success and `-1` means failure. Only import the Arrow +outputs after success; the caller owns their release callbacks. Failed calls +leave the outputs unchanged. `pheon/_ffi.h` declares the C API; `test.py` +demonstrates the supported Python reader. + +`last_error_message()` returns a UTF-8, null-terminated error string, or null +after a successful parser call. Errors belong to the calling thread. Copy the +message before making another parser call on that thread, and before the thread +exits; calls on other threads do not invalidate it. Parser panics that unwind +are converted into errors. + +For partial reads, call `scan_bytes` with a borrowed GDSII byte buffer to obtain +cell ranges and references without decoding geometry. Pass the desired ranges +to `read_cells_bytes` to load their full contents. Both calls use the same Arrow +outputs and status/error handling as `read_path`: + +```c +int scan_bytes(const uint8_t *data, size_t size, + struct ArrowArray *array, struct ArrowSchema *schema); +int read_cells_bytes(const uint8_t *data, size_t size, + const uint64_t *ranges, size_t range_count, + struct ArrowArray *array, struct ArrowSchema *schema); +``` + +`ranges` contains alternating inclusive start and exclusive end offsets, with +two native-endian `uint64_t` values per cell. Each range must cover one complete +cell. A null `ranges` pointer is accepted when `range_count` is zero. Likewise, +null `data` is accepted for zero bytes and produces an ordinary parse error for +the empty input. Nonempty buffers must be readable for the duration of the call; +range values must be aligned. Exported Arrow data owns its storage independently +of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan +schema, offset definitions, and ID assignment. + +## Python package and validation + +Python 3.11 or newer is required. Source builds require Rust 1.88 or newer; +installing a prebuilt wheel requires no Rust compiler. The shared library uses +a C ABI rather than Python's or Rust's native ABI, so one platform wheel serves +multiple Python minor versions. CFFI, NumPy and optional PyArrow must themselves +support the selected interpreter. A Rust application uses the crate normally +and compiles it with its own compatible toolchain. The checkout selects the +stable toolchain locally through `rust-toolchain.toml`, without changing your +global default. CI checks both Rust 1.88 and current stable. + +```sh +python3 -m pip install 'pheon[arrow]' +``` + +```python +from pheon import arrow + +library = arrow.read_path('chip.gds') # one-row StructArray; also accepts gzip +scan = arrow.scan_bytes(gds_bytes) +cells = scan.field('cells')[0].values +ranges = list(zip(cells.field('struct_start_offset').to_pylist(), + cells.field('struct_end_offset').to_pylist())) +selected = arrow.read_cells_bytes(gds_bytes, ranges[:2]) +``` + +`read_bytes` reads a complete borrowed buffer. Full, scanned and selected Arrow +results own their storage independently of that buffer. `PheonError` reports +native parse failures; malformed range shapes and types raise Python value/type +errors before calling native code. Integer packing needs no PyArrow import. + +### Discovery and companion packages + +Klamath's `native` extra installs `pheon`; Masque's `arrow` extra installs +`pheon[arrow]`. Klamath imports Pheon's packing helpers, while Masque calls +`pheon.arrow` and translates parse failures to its existing error type. + +The Python loader selects the library bundled with the imported `pheon` package, +or the exact file selected by `PHEON_LIBRARY`. It checks C ABI version 1 and +Arrow layout schema version 1 before exposing native functions. Incompatible +or unversioned libraries fail with an import error. Those versions are independent +of package releases and the CLI's JSON report schema. + +`cargo install --path . --locked --bin pheon` installs the CLI, not the Python +package or a discoverable shared library. Klamath does not search PATH or Cargo +directories for it. To reuse an external native build, install the Python +components too and set `PHEON_LIBRARY`; an editable source install is useful +during development. The old library override variables are not recognized. + +The Python packing helpers require NumPy and CFFI, both declared as runtime +dependencies. Setuptools invokes `cargo build --release --lib` and bundles the +C ABI shared library in a platform-specific wheel. Its version comes from +`Cargo.toml`. The C declarations are in `pheon/_ffi.h`; the loader uses +CFFI ABI mode, so no Python extension compilation or Maturin is required. + +```sh +python3 -m pip install . +``` + +To build a source archive and a wheel with the `build` frontend and installed +Setuptools/Wheel tooling: + +```sh +python3 -m build --no-isolation +``` + +Source archives include a Cargo lockfile, and wheel builds use `--locked` so +building from an archive retains the resolved Rust dependency versions. Wheels +are tagged for the build platform and use the Python-independent `py3-none` ABI. +The CLI continues to build directly with Cargo and has no Python dependency. + +For source-tree development, point the CFFI loader at a Cargo-built library: + +```sh +cargo build --release --lib +PHEON_LIBRARY="$PWD/target/release/libpheon.so" \ + python3 -m unittest discover -s tests/python -v +``` + +Use `libpheon.dylib` on macOS or `pheon.dll` on Windows. Without +the explicit override, the loader uses the library bundled inside the package. +After installing a wheel, run its tests outside the source checkout to exercise +the installed package: + +```sh +python3 -I -m unittest discover -s /absolute/path/to/pheon/tests/python -v +``` + +The Python tests cover multidimensional and noncontiguous packing, input +preservation, range errors, and the C declarations for Arrow exports and release +callbacks, input lifetimes and ABI mismatches. Run `cargo test` and +`cargo test --release` for native validation. See [release validation](docs/releases.md) +for wheel builds, platform repair and testing the same artifact on multiple +Python versions. + +## Layout review CLI + +The review CLI is a standalone Rust executable: + +```sh +cargo build --bin pheon +target/debug/pheon dump chip.gds +target/debug/pheon dump chip.gds --records +target/debug/pheon diff old.gds new.gds --hierarchy ascii +target/debug/pheon diff old.gds new.gds --hierarchy mermaid --max-changes 20 +target/debug/pheon hierarchy chip.gds --format mermaid --depth 4 +target/debug/pheon diff old.gds new.gds --top CHIP --layer 1/0 +target/debug/pheon diff old.gds new.gds --show-layer 1/0 +target/debug/pheon diff old.gds new.gds --format json +target/debug/pheon diff old.gds new.gds --format json --hierarchy graph +target/debug/pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 --name 1:0:WG +``` + +The object dump is deterministic and omits source offsets. Record dumps preserve +record order and show offsets, tags, sizes, and values, including records the +object reviewer does not support. They can inspect an invalid metadata declaration +without invoking object-level validation. Zero padding after ENDLIB is accepted. + +The fixed `klamath-object-v2` comparison profile: + +- Matches cells by name and layers by numeric pair. The default scope is the + entire library; explicit selectors are recorded in each report. +- Ignores library/cell timestamps, cell order, element order, and property order. +- Normalizes boundary/box starting vertex and winding, retaining other vertices. +- Preserves duplicate objects and duplicate properties as multisets. +- Compares paths, labels, references, array endpoints/counts, properties, and all + retained flags exactly. Explicit defaults and their omission remain distinct; + SREFs and single-element AREFs remain distinct. REAL8 encodings are retained. +- Reports library fields, including UNITS, separately. It does not rescale files + into a shared grid, infer cell renames, or perform geometric Boolean operations. +- Uses bounding boxes as coordinate envelopes, not evidence of equality. Path + envelopes do not include stroke width; text/reference envelopes are coordinates, + not rendered or recursively transformed bounds. +- Returns an incomplete result for unsupported records, future metadata versions, + unresolved references, or cycles. These cannot produce an equality claim. + +`diff` exits with **0** for equality under this profile, **1** for differences, +and **2** for invalid input or incomplete comparison. `--max-changes` limits the +number of object details displayed, not the comparison. Source offsets are +included in changed-object reports for tracing changes back to record dumps. + +`diff --format json` emits a versioned report with typed before/after evidence, +structured diagnostics, and independent comparison status and completeness. +JSON includes all changes by default. Explicit display filters and `--max-changes` +record omitted evidence without changing totals or exit status. `--hierarchy graph` +adds the complete scoped reference graph, with symbolic array multiplicities; +`--depth` applies to text output only. Timestamp-only changes still return 0. +Input and argument errors also produce JSON when requested. See the +[report contract](docs/diff-reports.md) and +[version 1 JSON Schema](docs/diff-report-v1.schema.json). + +Unmatched objects are paired for explanation only when candidates are unique by +type/layer/bounding box, then by type/layer among the remaining objects. +References additionally match by target. Ambiguous groups stay additions and +removals. Reports explain reference movement, array-count changes, path widths, +and polygons changed within the same bounds. Full before/after records remain +visible, and duplicate counts remain significant. Unchanged layer names appear +once as a legend for changed objects on those layers. + +### Comparison scope and display filters + +`--top NAME` compares that cell and its reachable subtree independently in each +revision. `--cell NAME` compares the exact cell, including its reference records, +without comparing descendant contents. Combine the two to select a cell within +a subtree. A selector absent from both revisions is an error; absence on one +side is an addition/deletion. + +`--layer L/T` selects objects on a numeric layer/type pair, plus references whose +targets use that layer transitively. Cells with no relevant content are omitted. +Unresolved references, cycles, and unsupported records with unknown layer effect +remain incomplete results. Unsupported objects on another known layer can be +excluded. Library fields, including units, remain compared. Unsupported layer-name +version diagnostics are always retained, since those declarations can affect any +layer or cell in the library. Scoped dumps and +hierarchy views accept the same selectors. Scope selection parses the complete +input: malformed record grammar and conflicting metadata are still errors. + +`--show-cell NAME` and `--show-layer L/T` filter object details in a diff report +without changing comparison status, totals, metadata changes, or hierarchy. +References have no numeric layer and are hidden by `--show-layer`. The comparison +scope and any display-only filter are printed explicitly. The Rust API accepts +one combined `Scope`; applying another scope to an already scoped document is +rejected rather than losing reference coverage information. + +Hierarchy output distinguishes directly changed cells, affected ancestors, and +unchanged targets of modified references. It includes edges from both revisions, +so removed branches remain visible. Counts are direct edge multiplicities, +including arrays, rather than expanded placements. Shared definitions are marked +as already shown in text and appear once in Mermaid. `--depth` limits displayed +descendants only. The CLI emits Mermaid source, requiring no diagram renderer. + +Layer names are read from labels in an existing cell. A label with the exact +string `KLAMATH_LAYER_NAMES_V1` identifies an annotation carrier `(layer, texttype)` +pair within that cell. All other labels on that same pair in that same cell must +be `layer:datatype:name` declarations, for example `1:0:WG`. The carrier pair is +not fixed; choose a suitable annotation layer in the design's technology. +Declarations split at the first two colons, allowing colons in names. Numeric +fields are unsigned 16-bit decimal integers. Names are nonempty UTF-8 strings. +Repeated identical declarations are accepted; conflicting declarations fail. +Names are collected across the library. Recognized metadata labels are omitted +from object comparison; mappings are compared separately and printed once per +numeric pair, so renaming a layer does not rewrite every shape description. + +`name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ...` writes +this convention. It consolidates existing recognized declarations into one legend +in the chosen existing cell, retains existing names not explicitly updated, and +overrides the named pairs. The chosen carrier must contain no ordinary design +objects in that cell. Unrelated records, timestamps, unknown records and trailing +padding are copied byte-for-byte. No structures or references are added. Output +must be a new file: the CLI refuses to overwrite an existing destination, including +the input. Invalid declarations, conflicting arguments, oversized labels and +future metadata versions fail before output creation. + +The record-preserving review adapter retains information normalized or omitted +by the Arrow geometry reader; existing Arrow APIs are unchanged. Structured Rust +change records feed the text and hierarchy renderers. LayoutEditor's `LayerNames` +extension, layout extraction/remapping, and geometric Boolean comparison remain +outside the current CLI. + +### Git integration + +Install the executable on PATH with `cargo install --path . --bin pheon`. +The templates in [contrib/git/config](contrib/git/config) and +[contrib/git/attributes](contrib/git/attributes) provide two alternative drivers. +For structural summaries and affected hierarchy, add this attribute: + +```gitattributes +*.gds diff=pheon +``` + +Configure the native external driver in Git config: + +```gitconfig +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false +``` + +The adapter consumes Git's old/new temporary files directly, handles additions, +deletions and renames, and quotes display paths. It returns success after rendering +either equality or differences, as expected by the external-driver protocol. +Invalid or incomplete comparisons retain a nonzero exit and diagnostics. Scope +and display options can precede the final `--` in the configured command. Existing +repositories using `diff=layout` can put the same command under `[diff "layout"]`. + +For ordinary Git line diffs, use canonical textconv instead: + +```gitattributes +*.gds diff=pheon-text +``` + +Configure the driver in Git config: + +```gitconfig +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true +``` + +This is a human review representation, not an applicable GDS patch. Invalidate +Git's textconv cache when changing the dump profile/tool version. For automated +equality checks, use `pheon diff` and its exit status directly. + +`tests/review.rs` contains ten initial fixture pairs, checked-in canonical/text/ +Mermaid goldens, metadata byte-preservation checks, scope and report regressions, +and actual Git integration tests. It also checks near-limit polygon records, +2048-cell hierarchies and arrays representing over a billion placements without +expanding them. Run `cargo test` and `cargo test --release` for debug and optimized +validation. No KLayout installation or Python package is required. diff --git a/contrib/git/attributes b/contrib/git/attributes new file mode 100644 index 0000000..ed817cc --- /dev/null +++ b/contrib/git/attributes @@ -0,0 +1,5 @@ +# Add these lines to the layout repository's .gitattributes. +*.gds diff=pheon + +# For canonical textconv instead, replace the line above with: +# *.gds diff=pheon-text diff --git a/contrib/git/config b/contrib/git/config new file mode 100644 index 0000000..ca84adf --- /dev/null +++ b/contrib/git/config @@ -0,0 +1,10 @@ +# Include this file from Git config after installing pheon on PATH. +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false + +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true diff --git a/docs/diff-report-v1.schema.json b/docs/diff-report-v1.schema.json new file mode 100644 index 0000000..fc85eaa --- /dev/null +++ b/docs/diff-report-v1.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Pheon diff report version 1", + "description": "Object-profile comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", + "type": "object", + "required": ["schema_version", "kind", "producer", "comparison_profile", "inputs", "scope", "coordinates", "status", "result", "exit_code", "summary", "report_selection", "layer_legend", "library_changes", "layer_name_changes", "cell_changes", "hierarchy", "diagnostics"], + "properties": { + "schema_version": {"const": 1}, + "kind": {"const": "diff"}, + "producer": { + "type": "object", "required": ["name", "version"], + "properties": {"name": {"const": "pheon"}, "version": {"type": "string"}} + }, + "comparison_profile": {"type": "string"}, + "inputs": { + "type": "object", "required": ["old", "new"], + "properties": {"old": {"$ref": "#/$defs/input"}, "new": {"$ref": "#/$defs/input"}} + }, + "scope": { + "type": "object", "required": ["top", "cell", "layer"], + "properties": {"top": {"type": ["string", "null"]}, "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}} + }, + "coordinates": { + "type": "object", "required": ["unit", "frame", "bounds"], + "properties": {"unit": {"const": "database_unit"}, "frame": {"const": "cell_local"}, "bounds": {"const": "coordinate_envelope"}} + }, + "status": {"enum": ["complete", "incomplete", "failed"]}, + "result": {"enum": ["equal", "different", "unknown"]}, + "exit_code": {"enum": [0, 1, 2]}, + "summary": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/summary"}]}, + "report_selection": {"$ref": "#/$defs/selection"}, + "layer_legend": { + "type": "array", "items": { + "type": "object", "required": ["layer", "name"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "name": {"type": "string"}} + } + }, + "library_changes": {"type": "array", "items": {"$ref": "#/$defs/libraryChange"}}, + "layer_name_changes": { + "type": "array", "items": { + "type": "object", "required": ["layer", "before", "after"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "before": {"type": ["string", "null"]}, "after": {"type": ["string", "null"]}} + } + }, + "cell_changes": {"type": "array", "items": {"$ref": "#/$defs/cellChange"}}, + "hierarchy": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/graph"}]}, + "diagnostics": {"type": "array", "items": {"$ref": "#/$defs/diagnostic"}} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "complete"}}}, + "then": { + "properties": {"result": {"enum": ["equal", "different"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "exact"}}}, "diagnostics": {"maxItems": 0}}, + "if": {"properties": {"result": {"const": "equal"}}}, + "then": {"properties": {"exit_code": {"const": 0}}}, + "else": {"properties": {"exit_code": {"const": 1}}} + }, + "else": {"properties": {"exit_code": {"const": 2}, "diagnostics": {"minItems": 1}}} + }, + { + "if": {"properties": {"status": {"const": "incomplete"}}}, + "then": {"properties": {"result": {"enum": ["different", "unknown"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "observed"}}}}} + }, + { + "if": {"properties": {"status": {"const": "failed"}}}, + "then": {"properties": {"result": {"const": "unknown"}, "summary": {"type": "null"}, "library_changes": {"maxItems": 0}, "layer_name_changes": {"maxItems": 0}, "cell_changes": {"maxItems": 0}, "hierarchy": {"type": "null"}}} + } + ], + "$defs": { + "decimal": {"type": "string", "pattern": "^(0|[1-9][0-9]*)$"}, + "hex": {"type": "string", "pattern": "^0x([0-9a-f]{2})*$"}, + "uint16": {"type": "integer", "minimum": 0, "maximum": 65535}, + "int32": {"type": "integer", "minimum": -2147483648, "maximum": 2147483647}, + "layer": { + "type": "object", "required": ["number", "purpose"], + "properties": {"number": {"$ref": "#/$defs/uint16"}, "purpose": {"$ref": "#/$defs/uint16"}} + }, + "nullableLayer": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/layer"}]}, + "point": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"$ref": "#/$defs/int32"}}, + "fields": {"type": "array", "items": {"$ref": "#/$defs/field"}}, + "field": { + "type": "object", "required": ["tag", "name", "encoding", "value", "payload_hex"], + "properties": { + "tag": {"$ref": "#/$defs/uint16"}, "name": {"type": "string"}, + "encoding": {"enum": ["xy", "uint16", "int16", "int32", "bits", "real8", "ascii", "none", "bytes"]}, + "payload_hex": {"$ref": "#/$defs/hex"}, + "value": {"type": ["array", "string", "null"]} + }, + "allOf": [ + {"if": {"properties": {"encoding": {"const": "xy"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/point"}}}}}, + {"if": {"properties": {"encoding": {"enum": ["uint16", "bits"]}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/uint16"}}}}}, + {"if": {"properties": {"encoding": {"const": "int16"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": "integer", "minimum": -32768, "maximum": 32767}}}}}, + {"if": {"properties": {"encoding": {"const": "int32"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/int32"}}}}}, + {"if": {"properties": {"encoding": {"const": "real8"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": ["number", "null"]}}}}}, + {"if": {"properties": {"encoding": {"const": "ascii"}}}, "then": {"properties": {"value": {"type": ["string", "null"]}}}}, + {"if": {"properties": {"encoding": {"enum": ["none", "bytes"]}}}, "then": {"properties": {"value": {"type": "null"}}}} + ] + }, + "input": { + "type": "object", "required": ["path", "path_encoded_hex", "present", "units"], + "properties": { + "path": {"type": ["string", "null"]}, + "path_encoded_hex": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/hex"}]}, + "present": {"type": ["boolean", "null"]}, + "units": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]} + } + }, + "summary": { + "type": "object", "required": ["counts", "changed_cells", "removed_objects", "added_objects", "layer_name_changes", "library_field_changes"], + "properties": { + "counts": {"enum": ["exact", "observed"]}, + "changed_cells": {"$ref": "#/$defs/decimal"}, "removed_objects": {"$ref": "#/$defs/decimal"}, "added_objects": {"$ref": "#/$defs/decimal"}, + "layer_name_changes": {"$ref": "#/$defs/decimal"}, "library_field_changes": {"$ref": "#/$defs/decimal"} + } + }, + "selection": { + "type": "object", "required": ["cell", "layer", "max_objects", "truncated", "filtered_cells", "filtered_objects", "limited_objects", "emitted_objects"], + "properties": { + "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}, + "max_objects": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "truncated": {"type": "boolean"}, + "filtered_cells": {"$ref": "#/$defs/decimal"}, "filtered_objects": {"$ref": "#/$defs/decimal"}, + "limited_objects": {"$ref": "#/$defs/decimal"}, "emitted_objects": {"$ref": "#/$defs/decimal"} + } + }, + "libraryChange": { + "oneOf": [ + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"const": "presence"}, "tag": {"type": "null"}, "before": {"type": "boolean"}, "after": {"type": "boolean"}} + }, + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "before": {"$ref": "#/$defs/fields"}, "after": {"$ref": "#/$defs/fields"}} + } + ] + }, + "source": { + "type": "object", "required": ["side", "byte_offset", "end_offset"], + "properties": {"side": {"enum": ["old", "new"]}, "byte_offset": {"$ref": "#/$defs/decimal"}, "end_offset": {"$ref": "#/$defs/decimal"}} + }, + "property": { + "type": "object", "required": ["attribute", "value", "payload_hex"], + "properties": {"attribute": {"$ref": "#/$defs/uint16"}, "value": {"type": ["string", "null"]}, "payload_hex": {"$ref": "#/$defs/hex"}} + }, + "object": { + "type": "object", "required": ["id", "kind", "tag", "source", "layer", "bounds", "target", "origin", "width", "counts", "fields", "properties"], + "properties": { + "id": {"type": "string"}, "kind": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "source": {"$ref": "#/$defs/source"}, + "layer": {"$ref": "#/$defs/nullableLayer"}, + "bounds": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 4, "maxItems": 4, "items": {"$ref": "#/$defs/int32"}}]}, + "target": {"type": ["string", "null"]}, "origin": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/point"}]}, + "width": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/int32"}]}, + "counts": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "integer", "minimum": 1, "maximum": 32767}}]}, + "fields": {"$ref": "#/$defs/fields"}, "properties": {"type": "array", "items": {"$ref": "#/$defs/property"}} + } + }, + "replacement": { + "type": "object", "required": ["removed", "added", "match_rule", "explanation", "field_changes", "properties_changed"], + "properties": { + "removed": {"type": "string"}, "added": {"type": "string"}, "match_rule": {"type": "string"}, "explanation": {"type": "string"}, "properties_changed": {"type": "boolean"}, + "field_changes": {"type": "array", "items": { + "type": "object", "required": ["tag", "before", "after"], + "properties": {"tag": {"$ref": "#/$defs/uint16"}, "before": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}, "after": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}} + }} + } + }, + "cellSide": { + "oneOf": [{"type": "null"}, { + "type": "object", "required": ["byte_offset", "attributes"], + "properties": {"byte_offset": {"$ref": "#/$defs/decimal"}, "attributes": {"$ref": "#/$defs/fields"}} + }] + }, + "cellChange": { + "type": "object", "required": ["name", "before", "after", "attributes_changed", "removed", "added", "replacements"], + "properties": { + "name": {"type": "string"}, "before": {"$ref": "#/$defs/cellSide"}, "after": {"$ref": "#/$defs/cellSide"}, "attributes_changed": {"type": "boolean"}, + "removed": {"type": "array", "items": {"$ref": "#/$defs/object"}}, "added": {"type": "array", "items": {"$ref": "#/$defs/object"}}, + "replacements": {"type": "array", "items": {"$ref": "#/$defs/replacement"}} + } + }, + "diagnostic": { + "type": "object", "required": ["code", "severity", "message", "side", "byte_offset", "cell", "layer", "target"], + "properties": { + "code": {"type": "string"}, "severity": {"const": "error"}, "message": {"type": "string"}, "side": {"enum": ["old", "new", null]}, + "byte_offset": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "cell": {"type": ["string", "null"]}, + "layer": {"$ref": "#/$defs/nullableLayer"}, "target": {"type": ["string", "null"]} + } + }, + "graph": { + "type": "object", "required": ["nodes", "edges"], + "properties": { + "nodes": {"type": "array", "items": { + "type": "object", "required": ["cell", "before", "after", "change"], + "properties": { + "cell": {"type": "string"}, "before": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, "after": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, + "change": {"enum": ["direct", "affected", "unchanged"]} + } + }}, + "edges": {"type": "array", "items": { + "type": "object", "required": ["parent", "target", "before_count", "after_count", "reference_changed"], + "properties": {"parent": {"type": "string"}, "target": {"type": "string"}, "before_count": {"$ref": "#/$defs/decimal"}, "after_count": {"$ref": "#/$defs/decimal"}, "reference_changed": {"type": "boolean"}} + }} + } + } + } +} diff --git a/docs/diff-reports.md b/docs/diff-reports.md new file mode 100644 index 0000000..24dcbf3 --- /dev/null +++ b/docs/diff-reports.md @@ -0,0 +1,189 @@ +# Machine-readable diff reports + +```sh +pheon diff before.gds after.gds --format json > report.json +pheon diff before.gds after.gds --format json --hierarchy graph +pheon diff before.gds after.gds --format json --top CHIP --layer 1/0 +``` + +The command writes one UTF-8 JSON document followed by a newline to stdout. +Comparison diagnostics belong to that document. Output write failures are +reported on stderr and return 2; interrupted output is not a valid report. +JSON output currently belongs to `diff`; `dump`, `hierarchy`, and the Git +external driver retain their existing text interfaces. No external process is +used to produce a report. + +The [version 1 JSON Schema](diff-report-v1.schema.json) defines the structure. +[Golden reports](../tests/golden/report-geometry.json) illustrate complete +evidence; other `report-*.json` goldens cover equality, metadata, incomplete +comparison, and failure. Their producer version is replaced with `test` so +package version changes do not alter the fixtures. + +## Versions and determinism + +Pheon emits schema version 1 with producer name `pheon`. This schema has not +had a public release. The comparison profile remains `klamath-object-v2`, and +the `KLAMATH_LAYER_NAMES_V1` GDS marker retains its existing meaning. + +`schema_version` describes the report structure. `comparison_profile` names the +equality rules, currently `klamath-object-v2`. `producer.version` identifies the +implementation. These versions are independent. Consumers interpreting equality +must understand the comparison profile as well as the schema. + +Compatible additions may introduce optional fields. Consumers should ignore +unfamiliar additional fields and handle unfamiliar diagnostic codes and matching +rules without treating them as equality. Removed fields or changed meanings +require a new schema version. The schema intentionally permits additional fields. + +For the same inputs, paths, options, and producer version, serialization is +deterministic. Cells, layer pairs, fields, properties, and graph edges are sorted. +Duplicate objects and properties remain separate occurrences. Reports contain +no generation timestamp or required content fingerprint. + +## Result and exit status + +| `status` | `result` | Exit code | Interpretation | +| --- | --- | --- | --- | +| `complete` | `equal` | 0 | Equal under the recorded profile and scope. | +| `complete` | `different` | 1 | Differences found. | +| `incomplete` | `different` | 2 | Definite differences found, with comparison limitations. | +| `incomplete` | `unknown` | 2 | Equality cannot be established. | +| `failed` | `unknown` | 2 | No comparison result is available. | + +Unsupported records, future metadata versions, unresolved references, and +hierarchy cycles make the comparison incomplete. Malformed input, read errors, +invalid scopes, and invalid command arguments produce failed reports. CI must +check completeness as well as differences. An exit code of 1 is a normal +comparison outcome, including when stdout is redirected to a report file. + +Library and cell timestamps are excluded from the comparison profile. A valid, +complete timestamp-only comparison returns 0. Raw record dumps remain available +for examining those timestamps. + +`summary` counts the entire scoped comparison, before display selection. Its +`counts` is `exact` for a complete comparison and `observed` for an incomplete +one. Failed reports have a null summary and empty change arrays. Replacement +suggestions and affected ancestors never add to change counts. + +## Scope, selection, and numbers + +`scope` records the actual `top`, `cell`, and numeric `layer` selectors. Null +means unrestricted. The complete file is parsed before applying scope, so an +invalid record outside the selected subtree still fails the command. + +`report_selection` records display-only `cell`, `layer`, and `max_objects`. +JSON has no default object limit. Explicit `--max-changes N` limits the total +number of emitted object occurrences across both sides. Removals precede +additions within each cell. `filtered_cells`, `filtered_objects`, +`limited_objects`, and `emitted_objects` explain evidence selection. `truncated` +is true if filtering or a limit omits any changed cell or object. Library and +layer-name changes, diagnostics, summaries, and the optional graph are retained. +A replacement is emitted only when both linked objects are included. + +All aggregate counts, limits, and byte offsets are unsigned decimal **strings**. +This avoids loss of precision in JSON consumers using binary64 numbers. +Coordinates, numeric tags, layer/type pairs, widths, and array dimensions are +bounded integers represented as JSON numbers. Null means absent or unavailable; +zero and empty arrays do not imply absence. + +`inputs.old` and `inputs.new` contain input paths, presence, and UNITS fields. +Presence is distinct from an empty library. The Rust API also supports absent +Git revisions using `Document::empty_like`; no units are invented for absent +inputs. Failed reports may have null presence and units. Argument failures can +lack input/selection context because the command could not be parsed. + +Paths are UTF-8 when representable. Otherwise `path` is null and +`path_encoded_hex` preserves Rust's platform-native encoded path bytes, prefixed +with `0x`. This is platform-dependent evidence rather than a portable path name. + +## Change evidence + +`library_changes` records changed header fields as typed before/after arrays. +Library presence changes use the special field `presence`, a null tag, and +boolean before/after values. Units are not rescaled to a common grid. + +`layer_name_changes` records numeric pairs with before/after names. A layer +pair is `{ "number": 1, "purpose": 0 }`, where purpose is DATATYPE, TEXTTYPE, +BOXTYPE, or NODETYPE according to the object kind. `layer_legend` contains +unchanged names available in the scope. Embedded annotation labels are excluded +from ordinary object changes. A metadata rename appears once. + +Each `cell_changes` entry has its name, nullable before/after cell records, +attribute evidence, removed and added objects, and replacement suggestions. +A null cell side means the cell is absent in that scope. An existing empty +cell has a byte offset and an empty attribute array. + +Each object retains: + +- Its kind and numeric tag, numeric layer pair, target cell, and coordinate envelope. +- Its original source side and byte range `[byte_offset, end_offset)`. +- Typed field records and the sorted property multiset, including duplicates. +- Convenience fields for text/reference origin, path/text width, and reference + array dimensions. SREF dimensions are `[1, 1]`; the kind and retained fields + distinguish an SREF from a one-element AREF and explicit fields from omission. + +Coordinates are cell-local integer database units. `bounds` contains +`[xmin, ymin, xmax, ymax]` of stored coordinates. It excludes path stroke width, +rendered text dimensions, and recursively transformed target geometry. +Equal envelopes do not establish geometric equality. Unit changes must be +considered before deriving physical movements or dimensions. + +A retained field has a numeric `tag`, convenience `name`, `encoding`, decoded +`value`, and `payload_hex`. The numeric tag is authoritative; names are display +aids. XY values are arrays of `[x, y]` pairs. Other numeric fields are arrays +of their declared type. Unsupported encodings retain bytes with a null value. +String and property values are null if their bytes are not valid UTF-8; +`payload_hex` always preserves them without replacement characters. + +Payloads follow the comparison profile: boundary/box rings use canonical start +and winding, string padding is removed, and fields/properties are sorted. They +are comparison evidence, not a verbatim serialization of the source records. +REAL8 payload encodings are retained exactly alongside their decoded values. +Explicit defaults and omitted fields remain distinct. Source ranges allow raw +record inspection when original encoding or order is needed. + +Replacement suggestions link report-local removed/added IDs, with a `match_rule`, +typed `field_changes`, and `properties_changed`. Their explanation is optional +for consumers to display and must not be parsed. Current matching rules are +`unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. +These are explanatory matches, not persistent identity or shape equivalence. +Ambiguous candidates stay independent removals and additions. IDs are unique +within a report and have no meaning across reports. + +## Diagnostics and hierarchy + +Diagnostics have `code`, `severity`, `message`, input `side`, original byte +offset, cell, layer, and target context. Unknown context is null. Messages are +for display; codes and context support automation. Current limitation codes are +`unsupported_record`, `unsupported_metadata_version`, `unresolved_reference`, +`hierarchy_cycle`, and `scope_mismatch`. Failure codes are `invalid_input`, +`io_error`, `invalid_scope`, and `command_error`. Fatal parser errors currently +retain their location in the human message; their structured byte offset is +null. Consumers must not depend on message wording. + +`--hierarchy graph` includes the full scoped union graph of both revisions. +Otherwise `hierarchy` is null. Nodes are keyed by cell name, with per-side +states `included`, `outside_scope`, `unresolved`, or `absent`. Their `change` +is `direct`, `affected` through a descendant, or `unchanged`. + +Edges record parent, target, before/after direct placement counts, and whether +the underlying reference records changed. AREF columns times rows contribute +symbolically; arrays are never expanded. Counts are direct edge multiplicities, +not recursively accumulated placements. Shared cells occur once. Removed edges +remain visible, and ancestors are computed over the union of both revisions. +Graph content is independent of display filters and object limits. Text depth +limits and ASCII/Mermaid choices do not apply to JSON graphs. + +## Development + +`cargo test --test review` checks report semantics, CLI behavior, and versioned +goldens. To intentionally regenerate report goldens after reviewing a contract +change, run: + +```sh +PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens +``` + +Schema validators can validate the goldens or actual CLI output against +`diff-report-v1.schema.json`. Schema validation does not replace semantic checks +such as replacement links, summary totals, or status/exit-code agreement. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..5836f98 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,51 @@ +# Building and validating a release + +The source archive, Python wheels and standalone CLI are separate artifacts. +Python wheels contain `pheon` and its shared library; Cargo installation contains +the CLI. Neither installation searches for the other. + +Build the source archive and then the wheel from that archive: + +```sh +python -m pip install build +python -m build +``` + +The default `build` invocation builds its wheel from the source archive. Both +carry the Cargo version, license and locked dependency resolution. A source build +requires Rust 1.88 or newer. The checkout selects current stable through +`rust-toolchain.toml`; `RUSTUP_TOOLCHAIN=1.88.0` selects the minimum compiler +explicitly for source-package builds. An installed wheel has no Rust runtime +requirement. + +The package workflow builds the archive once, then uses cibuildwheel to build +from it and repair wheels for Linux x86-64/ARM64, macOS x86-64/ARM64 and Windows +x86-64. Repair uses the platform tools supplied by cibuildwheel. Wheels must keep +the `py3-none` interpreter/ABI tags and a platform tag, including an appropriate +manylinux tag for Linux releases. Local unrepaired `linux_*` wheels are development +artifacts, not portable Linux releases. + +Each repaired wheel is tested on Python 3.11 through 3.14 without rebuilding it. +Tests run in isolated mode against the installed package and bundled library, +including an import that verifies integer packing does not import PyArrow. To +repeat that check locally after installing a wheel and PyArrow: + +```sh +python -I /path/to/pheon/tools/check_installed.py +``` + +Unset `PHEON_LIBRARY` for artifact tests. It exists to select an explicit external +library during development, and version checks apply equally to that library. +The minimum Rust compiler and current stable compiler run native tests in both +debug and release modes. CLI artifacts contain the executable, license, README +and Git integration templates. They are tied to their build OS and architecture. + +Workflows upload build artifacts only. They do not publish to PyPI, crates.io, +or a hosted release. The repository URL and checkout directory can be renamed +to `pheon` separately; no remote repository settings are changed by this work. + +The project rename provides no Python import shim or old CLI alias. Companion +releases must migrate their imports and dependencies together. The Python C API +version and Arrow layout version are currently 1. CLI reports use schema version +1 and retain the `klamath-object-v2` comparison profile and existing GDS layer-name +metadata marker. diff --git a/pheon/__init__.py b/pheon/__init__.py new file mode 100644 index 0000000..09bb1d6 --- /dev/null +++ b/pheon/__init__.py @@ -0,0 +1,8 @@ +"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers.""" + +from ._native import PheonError as PheonError +from .basic import pack_int2 as pack_int2 +from .basic import pack_int4 as pack_int4 + + +__version__ = "0.3.0" diff --git a/pheon/_ffi.h b/pheon/_ffi.h new file mode 100644 index 0000000..6bca31f --- /dev/null +++ b/pheon/_ffi.h @@ -0,0 +1,53 @@ +#ifndef PHEON_H +#define PHEON_H +#include +#include + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +uint32_t pheon_abi_version(void); +uint32_t pheon_arrow_schema_version(void); +int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema); +const char *last_error_message(void); + +double f64_to_i16(double *data, size_t size); +double f64_to_i32(double *data, size_t size); +float f32_to_i16(float *data, size_t size); +float f32_to_i32(float *data, size_t size); +int64_t i64_to_i16(int64_t *data, size_t size); +int64_t i64_to_i32(int64_t *data, size_t size); +int32_t i32_to_i16(int32_t *data, size_t size); +int32_t i32_to_i32(int32_t *data, size_t size); +int16_t i16_to_i16(int16_t *data, size_t size); +uint64_t u64_to_i16(uint64_t *data, size_t size); +uint64_t u64_to_i32(uint64_t *data, size_t size); +uint32_t u32_to_i16(uint32_t *data, size_t size); +uint32_t u32_to_i32(uint32_t *data, size_t size); +uint16_t u16_to_i16(uint16_t *data, size_t size); +#endif diff --git a/pheon/_native.py b/pheon/_native.py new file mode 100644 index 0000000..fc9dbd7 --- /dev/null +++ b/pheon/_native.py @@ -0,0 +1,55 @@ +"""Load a bundled or explicitly selected library through its stable C ABI.""" + +import os +from pathlib import Path +import sys + +from cffi import FFI + + +ffi = FFI() +header = Path(__file__).with_name("_ffi.h").read_text() +ffi.cdef( + "\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")) +) + +if sys.platform == "win32": + filename = "pheon.dll" +elif sys.platform == "darwin": + filename = "libpheon.dylib" +else: + filename = "libpheon.so" + +# An explicit override supports source-tree development without copying binaries +# into the package. Installed wheels load only their bundled native library. +library = ( + Path(os.environ["PHEON_LIBRARY"]).expanduser() + if "PHEON_LIBRARY" in os.environ + else Path(__file__).with_name(filename) +) +try: + lib = ffi.dlopen(str(library.resolve())) +except OSError as error: + raise ImportError( + f"Could not load Pheon library {library}: {error}. " + "Install a Pheon wheel or set PHEON_LIBRARY to a Cargo-built shared library." + ) from error + +ABI_VERSION = 1 +ARROW_SCHEMA_VERSION = 1 +try: + native_abi = lib.pheon_abi_version() + native_schema = lib.pheon_arrow_schema_version() +except AttributeError as error: + raise ImportError( + "The selected library does not expose the Pheon version API" + ) from error +if (native_abi, native_schema) != (ABI_VERSION, ARROW_SCHEMA_VERSION): + raise ImportError( + f"Incompatible Pheon library: C ABI {native_abi}, Arrow schema {native_schema}; " + f"expected C ABI {ABI_VERSION}, Arrow schema {ARROW_SCHEMA_VERSION}" + ) + + +class PheonError(ValueError): + """A native GDSII parse or selection error.""" diff --git a/pheon/arrow.py b/pheon/arrow.py new file mode 100644 index 0000000..5d90631 --- /dev/null +++ b/pheon/arrow.py @@ -0,0 +1,105 @@ +"""GDSII readers returning a one-row Arrow StructArray for the library. + +The returned arrays own their buffers independently of the input. This module +requires the optional ``arrow`` extra; integer packing does not import PyArrow. +""" + +from __future__ import annotations + +from collections.abc import Sequence +import gzip +import os +from typing import Any, Callable + +import numpy +import pyarrow + +from ._native import PheonError, ffi, lib + + +def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray: + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + if call(array, schema) != 0: + error = lib.last_error_message() + message = ( + ffi.string(error).decode("utf-8", errors="replace") + if error != ffi.NULL + else "GDSII read failed" + ) + raise PheonError(message) + # PyArrow consumes the structures, setting their release callbacks to + # null. The finally block also covers failure during import. + return pyarrow.Array._import_from_c( + int(ffi.cast("uintptr_t", array)), int(ffi.cast("uintptr_t", schema)) + ) + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + +def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: + """Read a GDSII file, recognizing gzip by its magic bytes.""" + filename = os.path.abspath(os.path.expanduser(os.fspath(path))) + with open(filename, "rb") as stream: + compressed = stream.read(2) == b"\x1f\x8b" + if compressed: + stream.seek(0) + with gzip.GzipFile(fileobj=stream) as source: + return read_bytes(source.read()) + # The native Windows path API accepts UTF-8; Unix accepts filesystem bytes. + encoded = filename.encode("utf-8") if os.name == "nt" else os.fsencode(filename) + return _export(lambda array, schema: lib.read_path(encoded, array, schema)) + + +def _read_buffer( + buffer: Any, call: Callable[[Any, int, Any, Any], int] +) -> pyarrow.StructArray: + with memoryview(buffer) as view: + if not view.c_contiguous: + raise ValueError("GDSII input must be a contiguous buffer") + with ffi.from_buffer("const uint8_t[]", view) as data: + return _export(lambda array, schema: call(data, view.nbytes, array, schema)) + + +def read_bytes(buffer: Any) -> pyarrow.StructArray: + """Read a complete GDSII library from a contiguous borrowed byte buffer.""" + return _read_buffer(buffer, lib.read_bytes) + + +def scan_bytes(buffer: Any) -> pyarrow.StructArray: + """Scan metadata, cell byte ranges and references without decoding geometry.""" + return _read_buffer(buffer, lib.scan_bytes) + + +def read_cells_bytes( + buffer: Any, ranges: Sequence[Sequence[int]] | numpy.ndarray +) -> pyarrow.StructArray: + """Read complete cells in the requested order from half-open byte ranges. + + ``ranges`` has shape (N, 2) and unsigned 64-bit integer offsets. Signed + integer inputs are accepted after checking that every offset is nonnegative. + An empty sequence requests metadata with no cells. + """ + values = numpy.asarray(ranges) + if values.shape == (0,): + values = numpy.empty((0, 2), dtype=numpy.uint64) + if values.ndim != 2 or values.shape[1] != 2: + raise ValueError("Cell ranges must have shape (N, 2)") + if values.dtype.kind not in "iu": + raise TypeError("Cell offsets must be integers") + if values.dtype.kind == "i" and numpy.any(values < 0): + raise ValueError("Cell offsets must be nonnegative") + values = numpy.require( + values, dtype=numpy.uint64, requirements=("C_CONTIGUOUS", "ALIGNED") + ) + with ffi.from_buffer("const uint64_t[]", values) as offsets: + return _read_buffer( + buffer, + lambda data, size, array, schema: lib.read_cells_bytes( + data, size, offsets, len(values), array, schema + ), + ) diff --git a/pheon/basic.py b/pheon/basic.py new file mode 100644 index 0000000..9ca4745 --- /dev/null +++ b/pheon/basic.py @@ -0,0 +1,72 @@ +from collections.abc import Sequence +from typing import Any + +import numpy +from numpy.typing import NDArray + +from ._native import lib, ffi + + +CONV_TABLE_i16 = { + numpy.float64: lib.f64_to_i16, + numpy.float32: lib.f32_to_i16, + numpy.int64: lib.i64_to_i16, + numpy.int32: lib.i32_to_i16, + numpy.int16: lib.i16_to_i16, + numpy.uint64: lib.u64_to_i16, + numpy.uint32: lib.u32_to_i16, + numpy.uint16: lib.u16_to_i16, +} + +CONV_TABLE_i32 = { + numpy.float64: lib.f64_to_i32, + numpy.float32: lib.f32_to_i32, + numpy.int64: lib.i64_to_i32, + numpy.int32: lib.i32_to_i32, + numpy.uint64: lib.u64_to_i32, + numpy.uint32: lib.u32_to_i32, +} + + +def _pack_int( + data: NDArray[numpy.integer] | Sequence[int] | int, + width: int, + converters: dict[type[numpy.generic], Any], +) -> bytes: + arr = numpy.atleast_1d(data) + target_dtype = numpy.dtype(f">i{width}") + + for dtype, convert in converters.items(): + if arr.dtype != dtype: + continue + + arr = numpy.require( + arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") + ) + if arr is data: + arr = numpy.array(arr, copy=True) + + with ffi.from_buffer( + ffi.typeof(convert).args[0], arr, require_writable=True + ) as buf: + result = convert(buf, arr.size) + if result != 0: + raise ValueError(f"Invalid value for conversion to Int{width}: {result}") + + return arr.reshape(-1).view(target_dtype)[:: arr.itemsize // width].tobytes() + + if arr.dtype == target_dtype: + return arr.tobytes() + + limit = 1 << (width * 8 - 1) + if (arr > limit - 1).any() or (arr < -limit).any(): + raise Exception(f"int{width} data out of range: {arr}") + return arr.astype(target_dtype).tobytes() + + +def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 2, CONV_TABLE_i16) + + +def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 4, CONV_TABLE_i32) diff --git a/pheon/py.typed b/pheon/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..835880a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pheon" +description = "Native GDSII parsing, Arrow interchange, and structural layout review" +readme = "README.md" +license = "GPL-3.0-only" +license-files = ["LICENSE.md"] +authors = [ + { name="Jan Petykiewicz", email="jan@mpxd.net" }, + ] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", +# "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Manufacturing", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", + ] +requires-python = ">=3.11" +dynamic = ["version"] +dependencies = [ + "cffi", + "numpy", + ] + +[project.optional-dependencies] +arrow = ["pyarrow>=14"] + +[project.urls] +Homepage = "https://mpxd.net/code/jan/pheon" +Repository = "https://mpxd.net/code/jan/pheon" + +[tool.setuptools] +packages = ["pheon"] + +[tool.setuptools.package-data] +pheon = ["py.typed", "_ffi.h"] + diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..292fe49 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ca4fd70 --- /dev/null +++ b/setup.py @@ -0,0 +1,76 @@ +"""Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tomllib + +from setuptools import Distribution, setup +from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist +from wheel.bdist_wheel import bdist_wheel + + +ROOT = Path(__file__).resolve().parent +VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] + + +def ensure_lockfile(): + if not (ROOT / "Cargo.lock").is_file(): + subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True) + + +class CargoBuild(build_py): + def run(self): + ensure_lockfile() + super().run() + result = subprocess.run( + ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], + cwd=ROOT, capture_output=True, text=True, + ) + sys.stderr.write(result.stderr) + library = None + for line in result.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + print(line, file=sys.stderr) + continue + if message.get("reason") == "compiler-message": + sys.stderr.write(message["message"].get("rendered") or "") + if (message.get("reason") == "compiler-artifact" + and message["target"]["name"] == "pheon" + and "cdylib" in message["target"]["crate_types"]): + for filename in message["filenames"]: + if Path(filename).suffix in (".so", ".dylib", ".dll"): + library = Path(filename) + result.check_returncode() + if library is None: + raise RuntimeError("Cargo did not produce the pheon shared library") + shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) + + +class CargoSdist(sdist): + def run(self): + ensure_lockfile() + super().run() + + +class NativeDistribution(Distribution): + def has_ext_modules(self): + return True + + +class NativeWheel(bdist_wheel): + def get_tag(self): + # CFFI ABI mode needs the host platform, but no CPython extension ABI. + _, _, platform = super().get_tag() + return "py3", "none", platform + + +setup( + version=VERSION, + distclass=NativeDistribution, + cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, +) diff --git a/src/basic.rs b/src/basic.rs new file mode 100644 index 0000000..f089ff3 --- /dev/null +++ b/src/basic.rs @@ -0,0 +1,353 @@ +/// +/// Functionality for parsing and writing basic data types +/// +use std::io; +use std::fmt; + +pub type OResult = Result; + +pub type IResult<'a, O> = Result<(&'a [u8], O), (&'a [u8], ErrType)>; + +#[derive(Debug)] +pub enum ErrType { + Incomplete(Option), + Failed(String), +} + +pub fn fail(input: &[u8], msg: String) -> IResult<'_, O> { + Err((input, ErrType::Failed(msg))) +} + +pub fn incomplete(input: &[u8], size: Option) -> IResult<'_, O> { + Err((input, ErrType::Incomplete(size))) +} + +pub fn take_bytes>(input: &[u8], count: CC) -> IResult<'_, &[u8]> { + let cc = count.into(); + if input.len() < cc { + incomplete(input, Some(cc)) + } else { + let (taken, input) = input.split_at(cc); + Ok((input, taken)) + } +} + + +/* + * Parse functions + */ +fn take_array(input: &[u8]) -> IResult<'_, [u8; N]> { + match input.split_first_chunk::() { + Some((bytes, remaining)) => Ok((remaining, *bytes)), + None => incomplete(input, Some(N)), + } +} + +pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> { + let (input, bytes) = take_array(input)?; + Ok((input, u16::from_be_bytes(bytes))) +} + +pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> { + let (input, bytes) = take_array(input)?; + Ok((input, i16::from_be_bytes(bytes))) +} + +pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> { + let (input, bytes) = take_array(input)?; + Ok((input, i32::from_be_bytes(bytes))) +} + +/// Convert GDS REAL8 to IEEE float64 +pub fn decode_real8(int: u64) -> f64 { + let neg = int & 0x8000_0000_0000_0000; + let exp = (int >> 56) & 0x7f; + let mut mant = (int & 0x00ff_ffff_ffff_ffff) as f64; + if neg != 0 { + mant *= -1.0 + } + let exp2 = 4 * (exp as i32 - 64) - 56; + mant * 2_f64.powi(exp2) +} + +pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { + let (input, bytes) = take_array(input)?; + let data = u64::from_be_bytes(bytes); + Ok((input, decode_real8(data))) +} + + +pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { + let mut buf = [0_i16; 6]; + let mut input = input; + for bb in &mut buf { + (input, *bb) = parse_int2(input)?; + } + buf[0] += 1900; // Year is from 1900 + Ok((input, buf)) +} + + +pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { + let mut bits = [false; 16]; + let (input, val) = parse_int2(input)?; + for (ii, bit) in bits.iter_mut().enumerate() { + *bit = ((val >> (16 - 1 - ii)) & 0x01) == 1; + } + Ok((input, bits)) +} + + +pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec> { + let length = length as usize; + let (input, data) = take_bytes(input, length)?; + let true_length = length - usize::from(data.last() == Some(&0)); + let vec = data[..true_length].to_vec(); + Ok((input, vec)) +} + + +/* + * Pack functions + */ + +pub fn bitarray2int(bits: &[bool; 16]) -> u16 { + let mut int: u16 = 0; + for (ii, bit) in bits.iter().enumerate() { + int |= (*bit as u16) << (16 - 1 - ii); + } + int +} + +pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { + buf[..2].copy_from_slice(&bitarray2int(bits).to_be_bytes()); +} + + +pub fn pack_int2(buf: &mut [u8], int: i16) { + buf[..2].copy_from_slice(&int.to_be_bytes()); +} + +pub fn pack_int4(buf: &mut [u8], int: i32) { + buf[..4].copy_from_slice(&int.to_be_bytes()); +} + +pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { + let bytes = encode_real8(fnum)?.to_be_bytes(); + buf[..8].copy_from_slice(&bytes); + Ok(()) +} + +pub fn pack_ascii(buf: &mut [u8], data: &[u8]) -> usize { + let len = data.len(); + buf[..len].copy_from_slice(data); + if len % 2 == 1 { + buf[len] = 0; + len + 1 + } else { + len + } +} + + +pub fn pack_datetime(buf: &mut [u8], date: &[i16; 6]) { + assert!(buf.len() >= 6 * 2); + let year = date[0] - 1900; + pack_int2(buf, year); + for ii in 1..6 { + pack_int2(&mut buf[(2 * ii)..], date[ii]); + } +} + + +#[derive(Debug, Clone)] +pub struct FloatTooBigError { + float_value: f64, +} +impl fmt::Display for FloatTooBigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Float {0} is too large for Real8", self.float_value) + } +} + +/// Convert float64 to a normalized GDS REAL8 representation. +/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0. +/// Nonfinite values and magnitudes at or above 2^252 return an error. +pub fn encode_real8(fnum: f64) -> Result { + let ieee = fnum.to_bits(); + let ieee_exp = ((ieee >> 52) & 0x7ff) as i32; + if ieee_exp == 0x7ff { + return Err(FloatTooBigError { float_value: fnum }); + } + + let exp2 = ieee_exp - 1023; + if exp2 < -260 { + // This also handles both signed zeros and every IEEE subnormal. + return Ok(0); + } + + let gds_exp = exp2.div_euclid(4) + 65; + if gds_exp > 0x7f { + return Err(FloatTooBigError { float_value: fnum }); + } + + // The 53-bit IEEE significand needs only 0..3 left shifts to become + // a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8. + let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52); + let mantissa = significand << exp2.rem_euclid(4); + let sign = ieee & 0x8000_0000_0000_0000; + Ok(sign | ((gds_exp as u64) << 56) | mantissa) +} + + +#[cfg(test)] +mod tests { + #[test] + fn test_parse_bitarray() { + use crate::basic::parse_bitarray; + + //assert!(parse_bitarray(b"59") == 13625); + assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); + assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); + + let arr_0001 = parse_bitarray(b"\x00\x01").unwrap().1; + for (ii, &vv) in arr_0001.iter().enumerate() { + assert_eq!(ii == 15, vv); + } + + let arr_8000 = parse_bitarray(b"\x80\x00").unwrap().1; + for (ii, &vv) in arr_8000.iter().enumerate() { + assert_eq!(ii == 0, vv); + } + } + + #[test] + fn test_parse_int2() { + use crate::basic::parse_int2; + assert_eq!(parse_int2(b"59").unwrap().1, 13625); + assert_eq!(parse_int2(b"\0\0").unwrap().1, 0); + assert_eq!(parse_int2(b"\xff\xff").unwrap().1, -1); + } + + #[test] + fn test_parse_int4() { + use crate::basic::parse_int4; + assert_eq!(parse_int4(b"4321").unwrap().1, 875770417); + } + + #[test] + fn test_decode_real8() { + use crate::basic::decode_real8; + + // zeroes + assert_eq!(decode_real8(0x0), 0.0); + assert_eq!(decode_real8(1<<63), 0.0); // negative + assert_eq!(decode_real8(0xff << 56), 0.0); // denormalized + + assert_eq!(decode_real8(0x4110 << 48), 1.0); + assert_eq!(decode_real8(0xC120 << 48), -2.0); + } + + #[test] + #[should_panic] + fn test_encode_real8_panic() { + use crate::basic::encode_real8; + encode_real8(1e80).unwrap(); + } + + + #[test] + fn test_parse_real8() { + use crate::basic:: parse_real8; + + assert_eq!(0.0, parse_real8(&[0; 8]).unwrap().1); + assert_eq!(1.0, parse_real8(&[0x41, 0x10, 0, 0, 0, 0, 0, 0]).unwrap().1); + assert_eq!(-2.0, parse_real8(&[0xC1, 0x20, 0, 0, 0, 0, 0, 0]).unwrap().1); + } + + #[test] + fn test_parse_ascii() { + use crate::basic::parse_ascii; + + assert_eq!(parse_ascii(b"12345", 5).unwrap().1, b"12345"); + assert_eq!(parse_ascii(b"12345\0", 6).unwrap().1, b"12345"); // strips trailing null byte + assert_eq!(parse_ascii(b"123456", 6).unwrap().1, b"123456"); + } + + #[test] + fn test_pack_bitarray() { + use crate::basic::pack_bitarray; + let mut buf = [10; 3]; + let mut bools = [false; 16]; + bools[1] = true; + bools[2] = true; + bools[11] = true; + + pack_bitarray(&mut buf, &bools); + assert_eq!(buf[0], 0b0110_0000); + assert_eq!(buf[1], 0b0001_0000); + assert_eq!(buf[2], 10); + } + + #[test] + fn test_pack_int2() { + use crate::basic::pack_int2; + let mut buf = [10; 3 * 2]; + pack_int2(&mut buf, -3); + pack_int2(&mut buf[2..], 2); + pack_int2(&mut buf[4..], -1); + assert_eq!(buf[0..2], [0xFF, 0xFD]); + assert_eq!(buf[2..4], [0x00, 0x02]); + assert_eq!(buf[4..6], [0xFF, 0xFF]); + } + + #[test] + fn test_pack_int4() { + use crate::basic::pack_int4; + let mut buf = [10; 3 * 4]; + pack_int4(&mut buf, -3); + pack_int4(&mut buf[4..], 2); + pack_int4(&mut buf[8..], -1); + assert_eq!(buf[0..4], [0xFF, 0xFF, 0xFF, 0xFD]); + assert_eq!(buf[4..8], [0x00, 0x00, 0x00, 0x02]); + assert_eq!(buf[8..12], [0xFF, 0xFF, 0xFF, 0xFF]); + } + + #[test] + fn test_encode_real8() { + use crate::basic::{encode_real8, decode_real8}; + const REALS: [f64; 5] = [1.0, -2.0, 1e-9, 1e-3, 1e-12]; + for vv in REALS { + print!("{vv}\n"); + assert!((decode_real8(encode_real8(vv).unwrap()) - vv).abs() < f64::EPSILON); + } + } + + #[test] + fn test_pack_real8() { + use crate::basic::{pack_real8, parse_real8}; + const COUNT: usize = 7; + const REALS: [f64; COUNT] = [0.0, 1.0, -1.0, 0.5, 1e-9, 1e-3, 1e-12]; + let mut buf = [10; 8 * COUNT]; + for (ii, &vv) in REALS.iter().enumerate() { + pack_real8(&mut buf[ii * 8..], vv).unwrap(); + } + for (ii, &vv) in REALS.iter().enumerate() { + print!("{vv}\n"); + let parsed_val = parse_real8(&buf[ii * 8..]).unwrap().1; + assert!((parsed_val - vv).abs() < f64::EPSILON); + } + } + + #[test] + fn test_pack_ascii() { + use crate::basic::pack_ascii; + let mut buf = [10; 12]; + pack_ascii(&mut buf[0..], "4321".as_bytes()); + pack_ascii(&mut buf[6..], "321".as_bytes()); + assert_eq!(&buf[0..4], "4321".as_bytes()); + assert_eq!(&buf[4..6], [10, 10]); + assert_eq!(&buf[6..9], "321".as_bytes()); + assert_eq!(&buf[9..], [0, 10, 10]); + } +} diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs new file mode 100644 index 0000000..646ec6e --- /dev/null +++ b/src/bin/pheon.rs @@ -0,0 +1,379 @@ +use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; +use pheon::review::{ + Comparison, DisplayFilter, Document, Scope, dump_records, hierarchy, parse_layer, + parse_layer_name, write_layer_names, +}; +use std::{ + collections::BTreeMap, + ffi::OsString, + fs, + io::{self, Write}, + path::PathBuf, + process::ExitCode, +}; + +const HELP: &str = "Usage: + pheon dump FILE [--records] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T]\n [--show-cell NAME] [--show-layer L/T] [--hierarchy ascii|mermaid|none|graph]\n [--max-changes N] [--depth N] [--format text|json] + pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE + pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... + +Object profile: ignore timestamps and element order, normalize polygon start/winding, +preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. +"; + +fn value(args: &[OsString], index: &mut usize) -> Result { + *index += 1; + args.get(*index) + .and_then(|s| s.to_str()) + .map(str::to_owned) + .ok_or_else(|| "missing UTF-8 option value".to_string()) +} + +fn name_layers(args: &[OsString]) -> Result { + if args.len() < 2 { + return Err(HELP.to_string()); + } + let (mut cell, mut carrier) = (None, None); + let mut names = BTreeMap::new(); + let mut i = 2; + while i < args.len() { + match args[i].to_str() { + Some("--cell") => cell = Some(value(args, &mut i)?), + Some("--carrier") => carrier = Some(parse_layer(&value(args, &mut i)?)?), + Some("--name") => { + let (pair, name) = parse_layer_name(&value(args, &mut i)?)?; + if names.get(&pair).is_some_and(|old| old != &name) { + return Err("conflicting --name arguments".to_string()); + } + names.insert(pair, name); + } + _ => { + return Err( + "name-layers requires --cell, --carrier and --name arguments".to_string(), + ); + } + } + i += 1; + } + if names.is_empty() { + return Err("provide at least one --name".to_string()); + } + let input = fs::read(&args[0]).map_err(|e| format!("{:?}: {e}", args[0]))?; + let output = write_layer_names( + &input, + &cell.ok_or("provide --cell")?, + carrier.ok_or("provide --carrier")?, + &names, + )?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[1]) + .map_err(|e| format!("{:?}: {e}", args[1]))?; + if let Err(e) = file.write_all(&output).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(&args[1]); + return Err(e.to_string()); + } + Ok(0) +} + +fn stdout(output: &str) -> Result<(), String> { + io::stdout() + .lock() + .write_all(output.as_bytes()) + .map_err(|e| e.to_string()) +} + +fn json_stdout(report: &serde_json::Value) -> Result<(), String> { + let mut output = io::stdout().lock(); + serde_json::to_writer_pretty(&mut output, report).map_err(|e| e.to_string())?; + output.write_all(b"\n").map_err(|e| e.to_string()) +} + +fn json_diff(paths: &[OsString], scope: &Scope, options: &ReportOptions) -> Result { + let inputs = ReportInputs { + old: Some(PathBuf::from(&paths[0])), + new: Some(PathBuf::from(&paths[1])), + }; + let comparison = || { + let read = |index, side| { + fs::read(&paths[index]) + .map_err(|e| ("io_error", format!("{:?}: {e}", paths[index]), Some(side))) + }; + let old_bytes = read(0, "old")?; + let new_bytes = read(1, "new")?; + let old = Document::parse(&old_bytes).map_err(|e| ("invalid_input", e, Some("old")))?; + let new = Document::parse(&new_bytes).map_err(|e| ("invalid_input", e, Some("new")))?; + scope + .validate_pair(&old, &new) + .map_err(|e| ("invalid_scope", e, None))?; + let old = old + .selected(scope, true) + .map_err(|e| ("invalid_scope", e, Some("old")))?; + let new = new + .selected(scope, true) + .map_err(|e| ("invalid_scope", e, Some("new")))?; + let diff = Comparison::between(&old, &new); + Ok::<_, (&str, String, Option<&str>)>(( + diff.json_report(&old, &new, &inputs, options), + diff.exit_code(), + )) + }; + let (report, status) = match comparison() { + Ok(result) => result, + Err((code, message, side)) => ( + failure_report(&inputs, scope, options, code, &message, side), + 2, + ), + }; + if let Err(error) = json_stdout(&report) { + // Once output starts, never append a second report on a write failure. + eprintln!("pheon: {error}"); + return Ok(2); + } + Ok(status) +} + +fn run(args: &[OsString]) -> Result { + if args.len() == 1 && matches!(args[0].to_str(), Some("--version" | "-V")) { + stdout(&format!("pheon {}\n", env!("CARGO_PKG_VERSION")))?; + return Ok(0); + } + if args.is_empty() || args[0] == "--help" || args[0] == "-h" { + stdout(HELP)?; + return Ok(0); + } + let command = args[0].to_str().ok_or("invalid command")?; + if command == "name-layers" { + return name_layers(&args[1..]); + } + let expected = match command { + "dump" | "hierarchy" => 1, + "diff" => 2, + "git-diff" => 7, + _ => return Err(HELP.to_string()), + }; + let compare = command == "diff" || command == "git-diff"; + let mut scope = Scope::default(); + let mut show = DisplayFilter::default(); + let mut raw = false; + let mut format = if compare { + "none".to_string() + } else { + "ascii".to_string() + }; + let mut max_changes = 100; + let mut explicit_max = None; + let mut output_format = "text".to_string(); + let mut depth = 8; + let mut explicit_depth = false; + let mut paths = Vec::new(); + let mut literal = false; + let mut i = 1; + while i < args.len() { + if literal { + paths.push(args[i].clone()); + i += 1; + continue; + } + match args[i].to_str() { + Some("--") => literal = true, + Some("--help" | "-h") => { + stdout(HELP)?; + return Ok(0); + } + Some("--records") if command == "dump" => raw = true, + Some("--top") => scope.top = Some(value(args, &mut i)?), + Some("--cell") => scope.cell = Some(value(args, &mut i)?), + Some("--layer") => scope.layer = Some(parse_layer(&value(args, &mut i)?)?), + Some("--show-cell") if compare => show.cell = Some(value(args, &mut i)?), + Some("--show-layer") if compare => { + show.layer = Some(parse_layer(&value(args, &mut i)?)?) + } + Some("--hierarchy") if compare => format = value(args, &mut i)?, + Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, + Some("--format") if command == "diff" => output_format = value(args, &mut i)?, + Some("--max-changes") if compare => { + max_changes = value(args, &mut i)? + .parse() + .map_err(|_| "invalid change limit")?; + explicit_max = Some(max_changes); + } + Some("--depth") if command != "dump" => { + depth = value(args, &mut i)?.parse().map_err(|_| "invalid depth")?; + explicit_depth = true; + } + Some(s) if s.starts_with('-') => return Err(format!("unknown option {s:?}")), + _ => paths.push(args[i].clone()), + } + i += 1; + } + if paths.len() != expected && !(command == "git-diff" && paths.len() == 9) { + return Err(HELP.to_string()); + } + if !matches!(output_format.as_str(), "text" | "json") { + return Err("diff format must be text or json".to_string()); + } + if output_format == "json" { + if !matches!(format.as_str(), "none" | "graph") || explicit_depth { + return Err( + "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), + ); + } + return json_diff( + &paths, + &scope, + &ReportOptions { + filter: show, + max_objects: explicit_max, + hierarchy: format == "graph", + }, + ); + } + if !matches!(format.as_str(), "ascii" | "mermaid" | "none") + || (command == "hierarchy" && format == "none") + { + return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); + } + let read = |path: &OsString| fs::read(path).map_err(|e| format!("{path:?}: {e}")); + if raw { + if scope != Scope::default() { + return Err("record dumps do not accept object scope selectors".to_string()); + } + stdout(&dump_records(&read(&paths[0])?)?)?; + return Ok(0); + } + if !compare { + let bytes = read(&paths[0])?; + let doc = Document::parse(&bytes)?.selected(&scope, false)?; + let output = if command == "dump" { + doc.canonical_dump() + } else { + hierarchy(&doc, &doc, None, format == "mermaid", depth) + }; + stdout(&output)?; + if command == "hierarchy" { + for issue in &doc.issues { + eprintln!("INCOMPLETE: {issue}"); + } + } + return Ok(if doc.issues.is_empty() { 0 } else { 2 }); + } + let git = command == "git-diff"; + let old_missing = git && (paths[3] == "000000" || paths[3] == "."); + let new_missing = git && (paths[6] == "000000" || paths[6] == "."); + if old_missing && new_missing { + return Err("both Git revisions are absent".to_string()); + } + let old_bytes = if old_missing { + None + } else { + Some(read(&paths[if git { 1 } else { 0 }])?) + }; + let new_bytes = if new_missing { + None + } else { + Some(read(&paths[if git { 4 } else { 1 }])?) + }; + let old = old_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; + let new = new_bytes.as_ref().map(|b| Document::parse(b)).transpose()?; + let old = old.unwrap_or_else(|| new.as_ref().unwrap().empty_like()); + let new = new.unwrap_or_else(|| old.empty_like()); + scope.validate_pair(&old, &new)?; + let old = old.selected(&scope, true)?; + let new = new.selected(&scope, true)?; + let diff = Comparison::between(&old, &new); + let mut output = String::new(); + if git { + if paths.len() == 9 { + output.push_str(&format!("layout {:?} -> {:?}\n", paths[0], paths[7])); + } else { + output.push_str(&format!("layout {:?}\n", paths[0])); + } + if old_missing { + output.push_str("layout added\n"); + } else if new_missing { + output.push_str("layout deleted\n"); + } + if paths[3] != paths[6] { + output.push_str(&format!("mode {:?} -> {:?}\n", paths[3], paths[6])); + } + } + output.push_str(&diff.text_filtered(max_changes, &show)); + if format != "none" { + if format == "mermaid" { + output.push_str("```mermaid\n"); + } + output.push_str(&hierarchy( + &old, + &new, + Some(&diff), + format == "mermaid", + depth, + )); + if format == "mermaid" { + output.push_str("```\n"); + } + } + stdout(&output)?; + // Git's external driver expects successful rendering for either equality + // or differences. Actual comparison/input failures remain nonzero. + Ok(if git && diff.exit_code() == 1 { + 0 + } else { + diff.exit_code() + }) +} + +fn wants_json(args: &[OsString]) -> bool { + if args.first().is_none_or(|arg| arg != "diff") { + return false; + } + let mut format = None; + let mut i = 1; + while i < args.len() { + match args[i].to_str() { + Some("--") => break, + Some("--format") => { + format = args.get(i + 1).and_then(|v| v.to_str()); + i += 1; + } + Some( + "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" + | "--max-changes" | "--depth", + ) => i += 1, + _ => (), + } + i += 1; + } + format == Some("json") +} + +fn main() -> ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let json = wants_json(&args); + match run(&args) { + Ok(status) => ExitCode::from(status), + Err(message) => { + if json { + let report = failure_report( + &ReportInputs::default(), + &Scope::default(), + &ReportOptions::default(), + "command_error", + &message, + None, + ); + if let Err(error) = json_stdout(&report) { + eprintln!("pheon: {error}"); + } + } else { + eprintln!("pheon: {message}"); + } + ExitCode::from(2) + } + } +} diff --git a/src/elements.rs b/src/elements.rs new file mode 100644 index 0000000..f043ccc --- /dev/null +++ b/src/elements.rs @@ -0,0 +1,1004 @@ +/// +/// Functionality for reading/writing elements (geometry, text labels, +/// structure references) and associated properties. +/// + +use crate::records::{ + //BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, + DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, + ENDEL, BGNEXTN, ENDEXTN, SNAME, + }; + +use crate::records; +use crate::record::{RecordHeader, Record}; +use crate::basic::{IResult, fail, take_bytes}; //OResult + +use std::string::String; +use std::collections::HashMap; +//use std::io::Write; + +use arrow_array::builder::{ + StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, + Int32Builder, Int16Builder, UInt64Builder, UInt32Builder, UInt8Builder, Float64Builder, + }; + + +pub type DListBuilder = ListBuilder>; +pub type FListBuilder = FixedSizeListBuilder>; + +struct PropertyRecord { + key: i16, + value: String, +} + +struct BoundaryRecord { + layer_id: u32, + vertices: Vec, + properties: Vec, +} + +struct BoundaryBatchRecord { + layer_id: u32, + vertices: Vec, + vertex_offsets: Vec, +} + +struct RectBatchRecord { + layer_id: u32, + rects: Vec, +} + +struct SRefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + properties: Vec, +} + +struct ARefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + xy0: u64, + xy1: u64, + counts: u32, + properties: Vec, +} + +#[derive(Clone)] +pub struct ScanRefRecord { + pub target_id: u32, + pub invert_y: bool, + pub scale: f64, + pub angle_rad: f64, + pub xy: u64, + pub xy0: u64, + pub xy1: u64, + pub counts: u32, +} + +enum RefRecord { + SRef(SRefRecord), + ARef(ARefRecord), +} + + +const UNIT_COUNTS: u32 = (1_u32 << 16) | 1_u32; +const ZERO_XY: u64 = 0; + + +fn layer_id_for( + layer: i16, + dtype: i16, + layers: &mut HashMap, + ) -> u32 { + let layer32 = ((layer as u16 as u32) << 16) | (dtype as u16 as u32); + let next_id = layers.len(); + *layers.entry(layer32).or_insert(next_id.try_into().unwrap()) +} + + +fn insert_layer( + layer: i16, + dtype: i16, + layers: &mut HashMap, + struct_builder: &mut StructBuilder, + field_index: usize, + ) { + let id = layer_id_for(layer, dtype, layers); + let layer_builder = struct_builder.field_builder::(field_index).unwrap(); + layer_builder.append_value(id); +} + + +pub fn read_elements<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + names: &mut HashMap, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let mut input = input; + let mut boundaries = Vec::::new(); + let mut srefs = Vec::::new(); + let mut arefs = Vec::::new(); + let (_, mut header) = RecordHeader::read(input)?; // don't consume tag + while header.tag != records::RTAG_ENDSTR { + (input, _) = match header.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, header.tag, names)?; + match ref_record { + RefRecord::SRef(sref) => srefs.push(sref), + RefRecord::ARef(aref) => arefs.push(aref), + } + (next_input, ()) + }, + records::RTAG_BOUNDARY => { + let (next_input, boundary) = read_boundary(input, layers)?; + boundaries.push(boundary); + (next_input, ()) + }, + records::RTAG_PATH => read_path(input, cell_builder, layers)?, + records::RTAG_NODE => read_boxnode(input, cell_builder, header.tag, layers)?, + records::RTAG_BOX => read_boxnode(input, cell_builder, header.tag, layers)?, + records::RTAG_TEXT => read_text(input, cell_builder, layers)?, + _ => { + // don't care, skip + let result = take_bytes(input, header.data_size + 4)?; + (result.0, ()) + } + }; + (_, header) = RecordHeader::read(input)?; // don't consume tag + } + (input, _) = take_bytes(input, 4_usize)?; // consume endstr tag + + let mut sref_plain = Vec::::with_capacity(srefs.len() + arefs.len() / 4); + let mut sref_props = Vec::::with_capacity(srefs.len() / 16 + arefs.len() / 64); + for sref in srefs { + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } + + let mut aref_plain = Vec::::with_capacity(arefs.len()); + let mut aref_props = Vec::::with_capacity(arefs.len() / 16); + for aref in arefs { + if aref.counts == UNIT_COUNTS { + let sref = aref_to_sref(aref); + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } else if aref.properties.is_empty() { + aref_plain.push(aref); + } else { + aref_props.push(aref); + } + } + sref_plain.sort_by_key(|sref| sref.target_id); + aref_plain.sort_by_key(|aref| aref.target_id); + + let mut boundary_plain = Vec::::with_capacity(boundaries.len()); + let mut boundary_props = Vec::::with_capacity(boundaries.len() / 16); + for boundary in boundaries { + if boundary.properties.is_empty() { + boundary_plain.push(boundary); + } else { + boundary_props.push(boundary); + } + } + boundary_plain.sort_by_key(|boundary| boundary.layer_id); + + let mut rect_batches = Vec::::new(); + let mut boundary_batches = Vec::::new(); + let mut ii = 0; + while ii < boundary_plain.len() { + let start = ii; + let layer_id = boundary_plain[ii].layer_id; + while ii < boundary_plain.len() && boundary_plain[ii].layer_id == layer_id { + ii += 1; + } + + let group = &boundary_plain[start..ii]; + let rect_capacity = group.len() * 4; + let vertex_capacity = group.iter().map(|boundary| boundary.vertices.len()).sum(); + let mut rect_batch = RectBatchRecord { + layer_id, + rects: Vec::with_capacity(rect_capacity), + }; + let mut boundary_batch = BoundaryBatchRecord { + layer_id, + vertices: Vec::with_capacity(vertex_capacity), + vertex_offsets: Vec::with_capacity(group.len()), + }; + + let mut next_offset: u32 = 0; + for boundary in group { + if let Some(rect) = boundary_to_rect(&boundary.vertices) { + rect_batch.rects.extend_from_slice(&rect); + } else { + boundary_batch.vertex_offsets.push(next_offset); + boundary_batch.vertices.extend_from_slice(&boundary.vertices); + next_offset += (boundary.vertices.len() / 2) as u32; + } + } + if !rect_batch.rects.is_empty() { + rect_batches.push(rect_batch); + } + if !boundary_batch.vertex_offsets.is_empty() { + boundary_batches.push(boundary_batch); + } + } + + let sref_builder = cell_builder.field_builder::(2).unwrap(); + for sref in sref_plain { + append_sref(sref_builder, sref); + } + sref_builder.append(true); + + let aref_builder = cell_builder.field_builder::(3).unwrap(); + for aref in aref_plain { + append_aref(aref_builder, aref); + } + aref_builder.append(true); + + let sref_props_builder = cell_builder.field_builder::(4).unwrap(); + for sref in sref_props { + append_sref_prop(sref_props_builder, sref); + } + sref_props_builder.append(true); + + let aref_props_builder = cell_builder.field_builder::(5).unwrap(); + for aref in aref_props { + append_aref_prop(aref_props_builder, aref); + } + aref_props_builder.append(true); + + let rect_batches_builder = cell_builder.field_builder::(6).unwrap(); + for rect_batch in rect_batches { + append_rect_batch(rect_batches_builder, rect_batch); + } + rect_batches_builder.append(true); + + let boundary_batches_builder = cell_builder.field_builder::(7).unwrap(); + for boundary_batch in boundary_batches { + append_boundary_batch(boundary_batches_builder, boundary_batch); + } + boundary_batches_builder.append(true); + + let boundary_props_builder = cell_builder.field_builder::(8).unwrap(); + for boundary in boundary_props { + append_boundary_prop(boundary_props_builder, boundary); + } + boundary_props_builder.append(true); + + let paths_builder = cell_builder.field_builder::(9).unwrap(); + paths_builder.append(true); + let nodes_builder = cell_builder.field_builder::(10).unwrap(); + nodes_builder.append(true); + let boxes_builder = cell_builder.field_builder::(11).unwrap(); + boxes_builder.append(true); + let texts_builder = cell_builder.field_builder::(12).unwrap(); + texts_builder.append(true); + + Ok((input, ())) +} + + +pub fn scan_elements<'a>( + input: &'a [u8], + names: &mut HashMap, + ) -> IResult<'a, Vec> { + let mut input = input; + let mut refs = Vec::::new(); + let (_, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDSTR { + (input, _) = match header.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, header.tag, names)?; + refs.push(scan_ref_from_record(ref_record)); + (next_input, ()) + }, + records::RTAG_BOUNDARY + | records::RTAG_PATH + | records::RTAG_NODE + | records::RTAG_BOX + | records::RTAG_TEXT => skip_element(input)?, + _ => { + let result = take_bytes(input, header.data_size + 4)?; + (result.0, ()) + } + }; + (_, header) = RecordHeader::read(input)?; + } + refs.sort_by_key(|scan_ref| scan_ref.target_id); + (input, _) = take_bytes(input, 4_usize)?; + Ok((input, refs)) +} + + +fn append_sref( + sref_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_struct_builder = sref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + sref_struct_builder.append(true); +} + + +fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { + match ref_record { + RefRecord::SRef(sref) => ScanRefRecord { + target_id: sref.target_id, + invert_y: sref.invert_y, + scale: sref.scale, + angle_rad: sref.angle_rad, + xy: sref.xy, + xy0: ZERO_XY, + xy1: ZERO_XY, + counts: UNIT_COUNTS, + }, + RefRecord::ARef(aref) => ScanRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + xy0: aref.xy0, + xy1: aref.xy1, + counts: aref.counts, + }, + } +} + + +fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { + let mut input = input; + loop { + let (next_input, header) = RecordHeader::read(input)?; + let (next_input, _) = take_bytes(next_input, header.data_size)?; + input = next_input; + if header.tag == ENDEL::tag() { + return Ok((input, ())); + } + } +} + + +fn append_aref( + aref_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_struct_builder = aref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_struct_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_struct_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_struct_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + aref_struct_builder.append(true); +} + + +fn append_property(props_builder: &mut DListBuilder, property: PropertyRecord) { + let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); + let key_builder = prop_builder.field_builder::(0).unwrap(); + key_builder.append_value(property.key); + + let val_builder = prop_builder.field_builder::(1).unwrap(); + val_builder.append_value(property.value); + + prop_builder.append(true); +} + + +fn append_properties( + props_builder: &mut DListBuilder, + properties: Vec, + ) { + for prop in properties { + append_property(props_builder, prop); + } + props_builder.append(true); +} + + +fn append_sref_prop( + sref_props_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_prop_builder = sref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + let props_builder = sref_prop_builder.field_builder::(5).unwrap(); + append_properties(props_builder, sref.properties); + + sref_prop_builder.append(true); +} + + +fn append_aref_prop( + aref_props_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_prop_builder = aref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_prop_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_prop_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_prop_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + let props_builder = aref_prop_builder.field_builder::(8).unwrap(); + append_properties(props_builder, aref.properties); + + aref_prop_builder.append(true); +} + + +fn aref_to_sref(aref: ARefRecord) -> SRefRecord { + SRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + properties: aref.properties, + } +} + + +fn read_boundary<'a>( + input: &'a [u8], + layers: &mut HashMap, + ) -> IResult<'a, BoundaryRecord> { + let (input, _) = records::BOUNDARY::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + let layer_id = layer_id_for(layer, dtype, layers); + + let (input, xy_iter) = XY::read(input)?; + let mut vertices: Vec = xy_iter.collect(); + vertices.truncate(vertices.len().saturating_sub(2)); + + let (input, properties) = read_properties_vec(input)?; + + Ok((input, BoundaryRecord { layer_id, vertices, properties })) +} + + +fn append_boundary_batch( + boundary_batches_builder: &mut DListBuilder, + boundary_batch: BoundaryBatchRecord, + ) { + let boundary_batch_builder = boundary_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary_batch.layer_id); + + let vertices_builder = boundary_batch_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary_batch.vertices); + vertices_builder.append(true); + + let offsets_builder = boundary_batch_builder.field_builder::(2).unwrap(); + let offset_builder = offsets_builder.values().as_any_mut().downcast_mut::().unwrap(); + offset_builder.append_slice(&boundary_batch.vertex_offsets); + offsets_builder.append(true); + + boundary_batch_builder.append(true); +} + + +fn append_rect_batch( + rect_batches_builder: &mut DListBuilder, + rect_batch: RectBatchRecord, + ) { + let rect_batch_builder = rect_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = rect_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(rect_batch.layer_id); + + let rects_builder = rect_batch_builder.field_builder::(1).unwrap(); + let rect_builder = rects_builder.values().as_any_mut().downcast_mut::().unwrap(); + rect_builder.append_slice(&rect_batch.rects); + rects_builder.append(true); + + rect_batch_builder.append(true); +} + + +fn append_boundary_prop( + boundary_props_builder: &mut DListBuilder, + boundary: BoundaryRecord, + ) { + let boundary_prop_builder = boundary_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_prop_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary.layer_id); + + let vertices_builder = boundary_prop_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary.vertices); + vertices_builder.append(true); + + let props_builder = boundary_prop_builder.field_builder::(2).unwrap(); + append_properties(props_builder, boundary.properties); + + boundary_prop_builder.append(true); +} + + +pub fn read_path<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let paths_builder = cell_builder.field_builder::(9).unwrap(); + let path_builder = paths_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = records::PATH::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + insert_layer(layer, dtype, layers, path_builder, 0); + + let mut path_type = None; + let mut width = None; + let mut bgn_ext = None; + let mut end_ext = None; + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_PATHTYPE => { + let _path_type; + (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; + path_type = Some(_path_type); + }, + records::RTAG_WIDTH => { + let _width; + (input, _width) = WIDTH::read_data(input, header.data_size)?; + width = Some(_width); + }, + records::RTAG_BGNEXTN => { + let _bgn_ext; + (input, _bgn_ext) = BGNEXTN::read_data(input, header.data_size)?; + bgn_ext = Some(_bgn_ext); + }, + records::RTAG_ENDEXTN => { + let _end_ext; + (input, _end_ext) = ENDEXTN::read_data(input, header.data_size)?; + end_ext = Some(_end_ext); + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + }; + (input, header) = RecordHeader::read(input)?; + } + let path_type_builder = path_builder.field_builder::(1).unwrap(); + path_type_builder.append_option(path_type); + let ext0_builder = path_builder.field_builder::(2).unwrap(); + ext0_builder.append_option(bgn_ext); + let ext1_builder = path_builder.field_builder::(3).unwrap(); + ext1_builder.append_option(end_ext); + let width_builder = path_builder.field_builder::(4).unwrap(); + width_builder.append_option(width); + + let xys_builder = path_builder.field_builder::(5).unwrap(); + let (input, xy_iter) = XY::read_data(input, header.data_size)?; + for xy in xy_iter { + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = path_builder.field_builder::(6).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + path_builder.append(true); + Ok((input, ())) +} + +pub fn read_boxnode<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + tag: u16, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let field_num = match tag { + records::RTAG_NODE => 10, + records::RTAG_BOX => 11, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let boxnodes_builder = cell_builder.field_builder::(field_num).unwrap(); + let boxnode_builder = boxnodes_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = match tag { + records::RTAG_NODE => records::NODE::read(input)?, + records::RTAG_BOX => records::BOX::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = match tag { + records::RTAG_NODE => NODETYPE::read(input)?, + records::RTAG_BOX => BOXTYPE::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + insert_layer(layer, dtype, layers, boxnode_builder, 0); + + let xys_builder = boxnode_builder.field_builder::(1).unwrap(); + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + let (input, xy_iter) = XY::read(input)?; + for xy in xy_iter { + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = boxnode_builder.field_builder::(2).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + boxnode_builder.append(true); + Ok((input, ())) +} + +pub fn read_text<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let texts_builder = cell_builder.field_builder::(12).unwrap(); + let text_builder = texts_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let mut path_type = None; + let mut pres_hori = None; + let mut pres_vert = None; + let mut pres_font = None; + let mut invert_y = None; + let mut width = None; + let mut mag = None; + let mut angle_deg = None; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = TEXTTYPE::read(input)?; + insert_layer(layer, dtype, layers, text_builder, 0); + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + // TODO warn if repeat tags? + records::RTAG_PRESENTATION => { + let _presentation; + (input, _presentation) = PRESENTATION::read_data(input, header.data_size)?; + pres_hori = Some(_presentation[14] as u8 * 2 + _presentation[15] as u8); + pres_vert = Some(_presentation[12] as u8 * 2 + _presentation[13] as u8); + pres_font = Some(_presentation[10] as u8 * 2 + _presentation[11] as u8); + }, + records::RTAG_PATHTYPE => { + let _path_type; + (input, _path_type) = PATHTYPE::read_data(input, header.data_size)?; + path_type = Some(_path_type); + }, + records::RTAG_WIDTH => { + let _width; + (input, _width) = WIDTH::read_data(input, header.data_size)?; + width = Some(_width); + }, + records::RTAG_STRANS => { + let strans; + (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = Some(strans[0]); + }, + records::RTAG_MAG => { + let _mag; + (input, _mag) = MAG::read_data(input, header.data_size)?; + mag = Some(_mag); + }, + records::RTAG_ANGLE => { + let _angle_deg; + (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; + angle_deg = Some(_angle_deg); + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + } + (input, header) = RecordHeader::read(input)?; + } + + let pres_hori_builder = text_builder.field_builder::(1).unwrap(); + pres_hori_builder.append_option(pres_hori); + let pres_vert_builder = text_builder.field_builder::(2).unwrap(); + pres_vert_builder.append_option(pres_vert); + let pres_font_builder = text_builder.field_builder::(3).unwrap(); + pres_font_builder.append_option(pres_font); + let path_type_builder = text_builder.field_builder::(4).unwrap(); + path_type_builder.append_option(path_type); + let width_builder = text_builder.field_builder::(5).unwrap(); + width_builder.append_option(width); + let inv_builder = text_builder.field_builder::(6).unwrap(); + inv_builder.append_option(invert_y); + let mag_builder = text_builder.field_builder::(7).unwrap(); + mag_builder.append_option(mag); + let angle_builder = text_builder.field_builder::(8).unwrap(); + angle_builder.append_option(angle_deg); + + let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; + let xx = xy_iter.next().unwrap(); + let yy = xy_iter.next().unwrap(); + let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let xy_builder = text_builder.field_builder::(9).unwrap(); + xy_builder.append_value(xy); + + let (input, string_bytes) = STRING::read(input)?; + let string = String::from_utf8(string_bytes).unwrap(); + let string_builder = text_builder.field_builder::(10).unwrap(); + string_builder.append_value(string); + + let props_builder = text_builder.field_builder::(11).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + text_builder.append(true); + Ok((input, ())) +} + + +fn boundary_to_rect(vertices: &[i32]) -> Option<[i32; 4]> { + if vertices.len() != 8 { + return None; + } + + let xs = [vertices[0], vertices[2], vertices[4], vertices[6]]; + let ys = [vertices[1], vertices[3], vertices[5], vertices[7]]; + + let min_x = *xs.iter().min().unwrap(); + let max_x = *xs.iter().max().unwrap(); + let min_y = *ys.iter().min().unwrap(); + let max_y = *ys.iter().max().unwrap(); + if min_x >= max_x || min_y >= max_y { + return None; + } + + let mut seen_corners: u8 = 0; + for ii in 0..4 { + let jj = (ii + 1) % 4; + let same_x = xs[jj] == xs[ii]; + let same_y = ys[jj] == ys[ii]; + if same_x == same_y { + return None; + } + + let x = xs[ii]; + let y = ys[ii]; + if (x != min_x && x != max_x) || (y != min_y && y != max_y) { + return None; + } + let code = ((x == max_x) as u8) << 1 | ((y == max_y) as u8); + seen_corners |= 1 << code; + } + + if seen_corners != 0b1111 { + return None; + } + Some([min_x, min_y, max_x, max_y]) +} + + + +fn read_ref<'a>( + input: &'a [u8], + tag: u16, + names: &mut HashMap, + ) -> IResult<'a, RefRecord> { + let (input, _) = match tag { + records::RTAG_SREF => records::SREF::read(input)?, + records::RTAG_AREF => records::AREF::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let is_aref = tag == records::RTAG_AREF; + let mut invert_y = false; + let mut scale = 1.0; + let mut angle_rad = 0.0; + let mut colrow = None; + + let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; + let struct_name = String::from_utf8(struct_name_bytes).unwrap(); + let next_id = names.len(); + let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap()); + + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_XY { + match header.tag { + records::RTAG_STRANS => { + let strans; + (input, strans) = STRANS::read_data(input, header.data_size)?; + invert_y = strans[0]; + }, + records::RTAG_MAG => { + let _mag; + (input, _mag) = MAG::read_data(input, header.data_size)?; + scale = _mag; + }, + records::RTAG_ANGLE => { + let _angle_deg; + (input, _angle_deg) = ANGLE::read_data(input, header.data_size)?; + angle_rad = _angle_deg.to_radians(); + }, + records::RTAG_COLROW => { + if header.data_size != 4 { + return fail(input, "COLROW must contain two counts".to_string()); + } + let mut _colrow; + (input, _colrow) = COLROW::read_data(input, header.data_size)?; + colrow = Some((_colrow.next().unwrap(), _colrow.next().unwrap())); + if !is_aref { + return fail(input, "Got a COLROW record inside an SREF".to_string()); + } + }, + _ => + return fail(input, format!("Unexpected tag {:04x}", header.tag)), + }; + (input, header) = RecordHeader::read(input)?; + } + + let expected_xy_size = if is_aref { 24 } else { 8 }; + if header.data_size != expected_xy_size { + return fail(input, format!("Expected {expected_xy_size} bytes of reference coordinates, got {}", header.data_size)); + } + let (input, mut xy_iter) = XY::read_data(input, header.data_size)?; + let xx = xy_iter.next().unwrap(); + let yy = xy_iter.next().unwrap(); + let xy = ((xx as u32 as u64) << 32) | (yy as u32 as u64); + let (input, properties) = read_properties_vec(input)?; + + if is_aref { + if colrow.is_none() { + return fail(input, "AREF without COLROW before XY".to_string()) + } + let (count0, count1) = colrow.unwrap(); + if count0 <= 0 || count1 <= 0 { + return fail(input, "AREF column and row counts must be positive".to_string()); + } + + let pitch = |end: i32, origin: i32, count: i16| { + let delta = i64::from(end) - i64::from(origin); + let divisor = i64::from(count); + if delta % divisor != 0 { + return Err((input, crate::basic::ErrType::Failed( + "AREF pitch is not an integer database-unit increment; the Arrow schema cannot represent it exactly".to_string() + ))); + } + let value = delta / divisor; + i32::try_from(value).map_err(|_| (input, crate::basic::ErrType::Failed( + "AREF pitch exceeds signed 32-bit coordinate range".to_string() + ))) + }; + let x0a = pitch(xy_iter.next().unwrap(), xx, count0)?; + let y0a = pitch(xy_iter.next().unwrap(), yy, count0)?; + let x1a = pitch(xy_iter.next().unwrap(), xx, count1)?; + let y1a = pitch(xy_iter.next().unwrap(), yy, count1)?; + + let xy0a = ((x0a as u32 as u64) << 32) | (y0a as u32 as u64); + let xy1a = ((x1a as u32 as u64) << 32) | (y1a as u32 as u64); + let counts = ((count0 as u16 as u32) << 16) | (count1 as u16 as u32); + Ok((input, RefRecord::ARef(ARefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy, + xy0: xy0a, + xy1: xy1a, + counts, + properties, + }))) + } else { + Ok((input, RefRecord::SRef(SRefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy, + properties, + }))) + } +} + + +/// +/// Read element properties. +/// +/// Preserves record order and duplicate property attributes. +/// Stops reading after consuming ENDEL record. +/// +pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { + let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; + props_builder.append(true); + Ok((input, ())) +} + + +fn walk_properties<'a>( + input: &'a [u8], + mut on_property: impl FnMut(PropertyRecord), + ) -> IResult<'a, ()> { + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != ENDEL::tag() { + if header.tag == PROPATTR::tag() { + let key; + let value_bytes; + (input, key) = PROPATTR::read_data(input, header.data_size)?; + (input, value_bytes) = PROPVALUE::read(input)?; + + on_property(PropertyRecord { + key, + value: String::from_utf8(value_bytes).unwrap(), + }); + } + (input, header) = RecordHeader::read(input)?; + } + Ok((input, ())) +} + + +fn read_properties_vec<'a>(input: &'a [u8]) -> IResult<'a, Vec> { + let mut properties = Vec::new(); + let (input, ()) = walk_properties(input, |property| properties.push(property))?; + Ok((input, properties)) +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..0abcca5 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,307 @@ +pub mod basic; +pub mod record; +pub mod records; +pub mod elements; +pub mod library; +pub mod review; + +use crate::library::{read_library, read_selected_library, scan_library}; + +use std::any::Any; +use std::cell::RefCell; +use std::mem::size_of; +use std::ffi::{CStr, CString, c_char}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::ptr; +#[cfg(unix)] +use std::os::unix::ffi::OsStringExt; +use std::str; +use std::fs; +use std::slice; + +use arrow_array::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, StructArray}; + + +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} + +/// Stable C API version. Increment for incompatible function/ownership changes. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_abi_version() -> u32 { + 1 +} + +/// Layout of the full-read and hierarchy-scan Arrow payloads. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_arrow_schema_version() -> u32 { + 1 +} + + +fn clear_last_error() { + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = None); +} + + +fn set_last_error(message: String) -> i32 { + let sanitized = message.replace('\0', "\\0"); + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = Some(CString::new(sanitized).unwrap())); + -1 +} + + +fn describe_panic(payload: Box) -> String { + if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unexpected panic".to_string() + } +} + + +fn describe_parse_error(input: &[u8], (remaining, reason): (&[u8], crate::basic::ErrType)) -> String { + // Selected-cell errors may contain a bounded range rather than a file suffix. + let offset = remaining.as_ptr() as usize - input.as_ptr() as usize; + let message = match reason { + crate::basic::ErrType::Failed(message) => message, + crate::basic::ErrType::Incomplete(Some(size)) => format!("Unexpected end of input; need {size} bytes"), + crate::basic::ErrType::Incomplete(None) => "Unexpected end of input".to_string(), + }; + format!("at byte {offset}: {message}") +} + + +fn export_struct_array( + op_name: &str, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + build: impl FnOnce() -> Result, + ) -> i32 { + clear_last_error(); + if arr.is_null() || schema.is_null() { + return set_last_error(format!("{op_name}: Arrow output pointer was null")); + } + match catch_unwind(AssertUnwindSafe(build)) { + Ok(Ok(struct_arr)) => match to_ffi(&struct_arr.to_data()) { + Ok((ffi_arr, ffi_schema)) => { + unsafe { + *arr = ffi_arr; + *schema = ffi_schema; + } + 0 + }, + Err(err) => set_last_error(format!("{op_name}: failed to export Arrow data: {err}")), + }, + Ok(Err(message)) => set_last_error(format!("{op_name}: {message}")), + Err(payload) => set_last_error(format!("{op_name}: {}", describe_panic(payload))), + } +} + + +fn pathbuf_from_c(cpath: *const c_char) -> Result { + if cpath.is_null() { + return Err("path pointer was null".to_string()); + } + + let cstr = unsafe { CStr::from_ptr(cpath) }; + #[cfg(unix)] + { + return Ok(PathBuf::from(std::ffi::OsString::from_vec(cstr.to_bytes().to_vec()))); + } + #[cfg(windows)] + { + let ustr = str::from_utf8(cstr.to_bytes()) + .map_err(|_| "Non-UTF8 paths are not supported".to_string())?; + return Ok(PathBuf::from(ustr)); + } + #[allow(unreachable_code)] + Err("Unsupported OS".to_string()) +} + + +/// Return this thread's last parser error, or null if its last call succeeded. +/// The returned string remains valid until this thread's next parser call or exit. +#[unsafe(no_mangle)] +pub extern "C" fn last_error_message() -> *const c_char { + LAST_ERROR.with(|last_error| last_error.borrow() + .as_ref() + .map_or(ptr::null(), |message| message.as_ptr())) +} + + +/// Read a GDSII file and export its Arrow data. Return 0 on success or -1 on error. +/// +/// # Safety +/// `cpath` must point to a null-terminated path. Non-null output pointers must +/// point to initialized, writable Arrow FFI structures. Null pointers are rejected. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_path( + cpath: *const c_char, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_path", arr, schema, || { + let path = pathbuf_from_c(cpath)?; + let input = fs::read(Path::new(&path)) + .map_err(|err| format!("file read failed for {}: {err}", path.display()))?; + let (_input, struct_arr) = read_library(&input) + .map_err(|err| format!("read failed {}", describe_parse_error(&input, err)))?; + Ok(struct_arr) + }) +} + + +unsafe fn input_slice<'a, T>(data: *const T, size: usize, name: &str) -> Result<&'a [T], String> { + if size == 0 { + return Ok(&[]); + } + if data.is_null() { + return Err(format!("{name} pointer was null")); + } + if !data.is_aligned() { + return Err(format!("{name} pointer was misaligned")); + } + if size.checked_mul(size_of::()).is_none_or(|bytes| bytes > isize::MAX as usize) { + return Err(format!("{name} length is too large")); + } + Ok(unsafe { slice::from_raw_parts(data, size) }) +} + +/// Read a borrowed GDSII buffer, exporting Arrow data that owns its storage. +/// +/// # Safety +/// `data` must cover `size` readable bytes (null is allowed for zero bytes). +/// Outputs must be initialized Arrow FFI structures, as for `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, +) -> i32 { + export_struct_array("read_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = read_library(input) + .map_err(|err| format!("read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Scan cell ranges and hierarchy references. Return 0 on success or -1 on error. +/// +/// # Safety +/// Non-null `data` must point to `size` readable bytes. Null is allowed when +/// `size` is zero. Output pointers have the same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scan_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("scan_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = scan_library(input) + .map_err(|err| format!("scan failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Read the cells in half-open byte ranges. Return 0 on success or -1 on error. +/// +/// # Safety +/// `data` has the same requirements as `scan_bytes`. Non-null `ranges` must +/// point to `range_count * 2` readable, aligned u64 values, alternating start +/// and end offsets. Null is allowed for zero ranges. Output pointers have the +/// same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_cells_bytes( + data: *const u8, + size: usize, + ranges: *const u64, + range_count: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_cells_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let range_size = range_count.checked_mul(2) + .ok_or_else(|| "range count is too large".to_string())?; + let flat_ranges = unsafe { input_slice(ranges, range_size, "ranges") }?; + let cell_ranges: Vec<(u64, u64)> = flat_ranges + .as_chunks::<2>() + .0 + .iter() + .map(|&[start, end]| (start, end)) + .collect(); + let (_input, struct_arr) = read_selected_library(input, &cell_ranges) + .map_err(|err| format!("selected-cell read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +macro_rules! impl_intbe { + ($source:ty, $target:ty, $arr:ident, $size:ident) => {{ + let values = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; + for value in values { + // All source types represent the target bounds exactly in f64. + let number = *value as f64; + if !number.is_finite() + || number < <$target>::MIN as f64 + || number > <$target>::MAX as f64 + { + return *value; + } + let mut slot = [0; size_of::<$source>()]; + slot[..size_of::<$target>()].copy_from_slice(&(*value as $target).to_be_bytes()); + *value = <$source>::from_ne_bytes(slot); + } + 0 as $source + }}; +} + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i32, arr, size) } + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_intbe!(u16, i16, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_intbe!(i16, i16, arr, size) } diff --git a/src/library.rs b/src/library.rs new file mode 100644 index 0000000..1ea4c6b --- /dev/null +++ b/src/library.rs @@ -0,0 +1,588 @@ +/// +/// File-level read/write functionality. +/// + +use std::io::Write; +//use std::collections::HashMap; + +pub use crate::record; +pub use crate::record::{RecordHeader, Record}; +pub use crate::records; +pub use crate::elements; +pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +pub use crate::basic::{IResult, OResult, take_bytes, fail}; + +use std::string::String; +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field, Fields}; +use arrow_array::{ + StructArray, + builder::{ + BooleanBuilder, Float64Builder, Int16Builder, StringBuilder, StructBuilder, + UInt32Builder, UInt64Builder, + }, +}; + + +/// +/// Representation of the GDS file header. +/// +/// File header records: HEADER BGNLIB LIBNAME UNITS +/// Optional records are ignored if present and never written. +/// +/// Version is written as `600`. +/// +#[derive(Debug, Clone)] +pub struct FileHeader { + /// Number of user units in one database unit + user_units_per_db_unit: f64, + /// Number of meters in one database unit + meters_per_db_unit: f64, + /// Last-modified time [y, m, d, h, m, s] + mod_time: [i16; 6], + /// Last-accessed time [y, m, d, h, m, s] + acc_time: [i16; 6], + /// Library name + name: Vec, +} + +impl FileHeader { + pub fn new(name: &[u8], meters_per_db_unit: f64, user_units_per_db_unit: f64) -> Self { + FileHeader{ + mod_time: [0, 1, 1, 0, 0, 0], + acc_time: [0, 1, 1, 0, 0, 0], + name: name.to_owned(), + user_units_per_db_unit, + meters_per_db_unit, + } + } + + /// Read and construct a header from the provided input. + /// + /// Args: + /// input: Seekable input to read from + /// + /// Returns: + /// FileHeader object + /// + pub fn read(input: &[u8]) -> IResult<'_, Self> { + let (input, _version) = records::HEADER::read(input)?; + let (input, [mod_time, acc_time]) = records::BGNLIB::read(input)?; + let (input, name) = records::LIBNAME::skip_and_read(input)?; + let (input, (uu, dbu)) = records::UNITS::skip_and_read(input)?; + + Ok((input, FileHeader{ + mod_time, + acc_time, + name, + user_units_per_db_unit: uu, + meters_per_db_unit: dbu, + })) + } + + /// Write the header to a input + /// + /// Args: + /// input: input to write to + /// + /// Returns: + /// number of bytes written + /// + pub fn write(&self, ww: &mut W) -> OResult { + let mut size = 0; + size += records::HEADER::write(ww, &600)?; + size += records::BGNLIB::write(ww, &[self.mod_time, self.acc_time])?; + size += records::LIBNAME::write(ww, &self.name)?; + size += records::UNITS::write(ww, &(self.user_units_per_db_unit, self.meters_per_db_unit))?; + Ok(size) + } +} + + +fn make_name_list_type() -> DataType { + DataType::List(Arc::new( + Field::new_list_field(DataType::Utf8, false) + )) +} + + +fn make_time_type() -> DataType { + DataType::FixedSizeList(Arc::new( + Field::new_list_field(DataType::Int16, false), + ), + 6, + ) +} + + +fn make_top_level_builder(cells_field: Field, extra_fields: Vec) -> StructBuilder { + let mut fields = vec![ + Field::new("meters_per_db_unit", DataType::Float64, false), + Field::new("user_units_per_db_unit", DataType::Float64, false), + Field::new("lib_name", DataType::Utf8, false), + Field::new("mod_time", make_time_type(), false), + Field::new("acc_time", make_time_type(), false), + cells_field, + Field::new("cell_names", make_name_list_type(), false), + ]; + fields.extend(extra_fields); + StructBuilder::from_fields(fields, 0) +} + + +fn reference_type(is_array: bool, properties: Option<&DataType>) -> DataType { + let mut fields = vec![ + Field::new("target", DataType::UInt32, false), + Field::new("invert_y", DataType::Boolean, false), + Field::new("scale", DataType::Float64, false), + Field::new("angle_rad", DataType::Float64, false), + Field::new("xy", DataType::UInt64, false), + ]; + if is_array { + fields.extend([ + Field::new("xy0", DataType::UInt64, false), + Field::new("xy1", DataType::UInt64, false), + Field::new("counts", DataType::UInt32, false), + ]); + } + if let Some(properties) = properties { + fields.push(Field::new("properties", properties.clone(), true)); + } + DataType::Struct(Fields::from(fields)) +} + + +fn make_library_builder() -> StructBuilder { + let property_t = DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Int16, false), + Field::new("value", DataType::Utf8, false), + ])); + + let property_list_t = DataType::List(Arc::new( + Field::new_list_field(property_t, false) + )); + + let sref_struct_t = reference_type(false, None); + let aref_struct_t = reference_type(true, None); + let sref_prop_struct_t = reference_type(false, Some(&property_list_t)); + let aref_prop_struct_t = reference_type(true, Some(&property_list_t)); + + let text_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("presentation_horiz", DataType::UInt8, true), + Field::new("presentation_vert", DataType::UInt8, true), + Field::new("presentation_font", DataType::UInt8, true), + Field::new("path_type", DataType::Int16, true), + Field::new("width", DataType::Int32, true), + Field::new("invert_y", DataType::Boolean, true), + Field::new("mag", DataType::Float64, true), + Field::new("angle_deg", DataType::Float64, true), + Field::new("xy", DataType::UInt64, false), + Field::new("string", DataType::Utf8, false), + Field::new("properties", property_list_t.clone(), true), + ])); + + + let coords_t = DataType::List(Arc::new( + Field::new_list_field(DataType::Int32, false) + )); + + let boundary_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new( + "vertex_offsets", + DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false))), + false, + ), + ])); + + let rect_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("rects", coords_t.clone(), false), + ])); + + let boundary_prop_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let path_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("path_type", DataType::Int16, true), + Field::new("extension_start", DataType::Int32, true), + Field::new("extension_end", DataType::Int32, true), + Field::new("width", DataType::Int32, true), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let boxnode_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let sref_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_struct_t, false) + )); + + let aref_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_struct_t, false) + )); + + let sref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_prop_struct_t, false) + )); + + let aref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_prop_struct_t, false) + )); + + let text_list_t = DataType::List(Arc::new( + Field::new_list_field(text_struct_t, false) + )); + + let boundary_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_batch_struct_t, false) + )); + + let rect_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(rect_batch_struct_t, false) + )); + + let boundary_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_prop_struct_t, false) + )); + + let path_list_t = DataType::List(Arc::new( + Field::new_list_field(path_struct_t, false) + )); + + let boxnode_list_t = DataType::List(Arc::new( + Field::new_list_field(boxnode_struct_t, false) + )); + + let layer_list_t = DataType::List(Arc::new( + Field::new_list_field(DataType::UInt32, false) + )); + + let cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("file_offset", DataType::UInt64, false), + Field::new("srefs", sref_list_t, false), + Field::new("arefs", aref_list_t, false), + Field::new("sref_props", sref_prop_list_t, false), + Field::new("aref_props", aref_prop_list_t, false), + Field::new("rect_batches", rect_batch_list_t, false), + Field::new("boundary_batches", boundary_batch_list_t, false), + Field::new("boundary_props", boundary_prop_list_t, false), + Field::new("paths", path_list_t, false), + Field::new("nodes", boxnode_list_t.clone(), true), + Field::new("boxes", boxnode_list_t.clone(), true), + Field::new("texts", text_list_t, false), + ])); + + let cells_list_t = DataType::List(Arc::new( + Field::new_list_field(cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", cells_list_t, false), + vec![ + Field::new("layers", layer_list_t, false), + ], + ) +} + + +fn make_scan_builder() -> StructBuilder { + let scan_ref_struct_t = reference_type(true, None); + + let scan_ref_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_ref_struct_t, false) + )); + + let scan_cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("struct_start_offset", DataType::UInt64, false), + Field::new("struct_end_offset", DataType::UInt64, false), + Field::new("refs", scan_ref_list_t, false), + ])); + + let scan_cells_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", scan_cells_list_t, false), + Vec::new(), + ) +} + + +fn append_file_header( + lib_builder: &mut StructBuilder, + header: &FileHeader, + ) { + let dbu_builder = lib_builder.field_builder::(0).unwrap(); + dbu_builder.append_value(header.meters_per_db_unit); + let uu_builder = lib_builder.field_builder::(1).unwrap(); + uu_builder.append_value(header.user_units_per_db_unit); + let libname_builder = lib_builder.field_builder::(2).unwrap(); + libname_builder.append_value(String::from_utf8(header.name.clone()).unwrap()); + let mtl_builder = lib_builder.field_builder::(3).unwrap(); + let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); + mt_builder.append_values(&header.mod_time, &[true; 6]); + mtl_builder.append(true); + let atl_builder = lib_builder.field_builder::(4).unwrap(); + let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); + at_builder.append_values(&header.acc_time, &[true; 6]); + atl_builder.append(true); +} + + +fn append_names( + lib_builder: &mut StructBuilder, + field_index: usize, + names: HashMap, + ) { + // Interned IDs are consecutive, so a vector directly represents ID order. + let mut ordered_names = vec![String::new(); names.len()]; + for (name, id) in names { + ordered_names[id as usize] = name; + } + let names_builder = lib_builder.field_builder::(field_index).unwrap(); + let name_builder = names_builder.values().as_any_mut().downcast_mut::().unwrap(); + for name in ordered_names { + name_builder.append_value(name); + } + names_builder.append(true); +} + + +fn append_layers( + lib_builder: &mut StructBuilder, + layers: HashMap, + ) { + let mut ordered_layers = vec![0; layers.len()]; + for (layer, id) in layers { + ordered_layers[id as usize] = layer; + } + let layers_builder = lib_builder.field_builder::(7).unwrap(); + let layer_builder = layers_builder.values().as_any_mut().downcast_mut::().unwrap(); + layer_builder.append_slice(&ordered_layers); + layers_builder.append(true); +} + + +fn append_scan_ref( + refs_builder: &mut DListBuilder, + scan_ref: ScanRefRecord, + ) { + let ref_builder = refs_builder.values().as_any_mut().downcast_mut::().unwrap(); + let target_builder = ref_builder.field_builder::(0).unwrap(); + target_builder.append_value(scan_ref.target_id); + let invert_builder = ref_builder.field_builder::(1).unwrap(); + invert_builder.append_value(scan_ref.invert_y); + let scale_builder = ref_builder.field_builder::(2).unwrap(); + scale_builder.append_value(scan_ref.scale); + let angle_builder = ref_builder.field_builder::(3).unwrap(); + angle_builder.append_value(scan_ref.angle_rad); + let xy_builder = ref_builder.field_builder::(4).unwrap(); + xy_builder.append_value(scan_ref.xy); + let xy0_builder = ref_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(scan_ref.xy0); + let xy1_builder = ref_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(scan_ref.xy1); + let counts_builder = ref_builder.field_builder::(7).unwrap(); + counts_builder.append_value(scan_ref.counts); + ref_builder.append(true); +} + + +fn intern_name(names: &mut HashMap, name: String) -> u32 { + let next_id = names.len(); + *names.entry(name).or_insert(next_id.try_into().unwrap()) +} + + +fn append_full_cell<'a>( + cells_builder: &mut DListBuilder, + names: &mut HashMap, + layers: &mut HashMap, + struct_input: &'a [u8], + id: u32, + file_offset: u64, + ) -> IResult<'a, ()> { + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let offset_builder = cell_builder.field_builder::(1).unwrap(); + offset_builder.append_value(file_offset); + + let (next_input, _) = read_elements(struct_input, cell_builder, names, layers)?; + cell_builder.append(true); + Ok((next_input, ())) +} + + +fn walk_structs<'a, F>( + input: &'a [u8], + base_offset: usize, + names: &mut HashMap, + mut on_struct: F, + ) -> IResult<'a, ()> +where + F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, +{ + let input_size = input.len(); + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != records::RTAG_ENDLIB { + let record_start = base_offset + input_size - input.len() - 4; + (input, _) = take_bytes(input, header.data_size)?; + if header.tag == records::RTAG_BGNSTR { + let name_bytes; + (input, name_bytes) = records::STRNAME::read(input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let id = intern_name(names, name); + let (next_input, ()) = on_struct(input, record_start, id, names)?; + input = next_input; + } + (input, header) = RecordHeader::read(input)?; + } + Ok((input, ())) +} + + +pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_library_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, _record_start, id, names| { + let position = base_offset + walk_input_size - struct_input.len(); + append_full_cell( + cells_builder, + names, + &mut layers, + struct_input, + id, + position.try_into().unwrap(), + ) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Scan cell byte ranges and references without decoding geometry. +pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_scan_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, record_start, id, names| { + let (after_cell, refs) = scan_elements(struct_input, names)?; + let struct_end = base_offset + walk_input_size - after_cell.len(); + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let start_builder = cell_builder.field_builder::(1).unwrap(); + start_builder.append_value(record_start.try_into().unwrap()); + + let end_builder = cell_builder.field_builder::(2).unwrap(); + end_builder.append_value(struct_end.try_into().unwrap()); + + let refs_builder = cell_builder.field_builder::(3).unwrap(); + for scan_ref in refs { + append_scan_ref(refs_builder, scan_ref); + } + refs_builder.append(true); + cell_builder.append(true); + Ok((after_cell, ())) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Read exact half-open cell ranges returned by `scan_library`, in caller order. +/// Each range includes BGNSTR through ENDSTR. Cell and layer IDs are local to +/// this result. This random-access operation returns the original input slice +/// as its remainder; it does not advance a sequential parsing cursor. +pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { + let mut lib_builder = make_library_builder(); + let (_, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + for (start_u64, end_u64) in ranges { + if start_u64 >= end_u64 || *end_u64 > input.len() as u64 { + return fail(input, format!("Cell byte range out of bounds: [{start_u64}, {end_u64})")); + } + let start = usize::try_from(*start_u64).unwrap(); + let end = usize::try_from(*end_u64).unwrap(); + + let struct_input = &input[start..end]; + let (struct_input, _) = records::BGNSTR::read(struct_input)?; + let (struct_input, name_bytes) = records::STRNAME::read(struct_input)?; + let name = String::from_utf8(name_bytes).unwrap(); + let file_offset = end - struct_input.len(); + + let id = intern_name(&mut names, name); + let (remaining_struct, _) = append_full_cell( + cells_builder, + &mut names, + &mut layers, + struct_input, + id, + file_offset.try_into().unwrap(), + )?; + if !remaining_struct.is_empty() { + return fail(remaining_struct, "Cell byte range extends past ENDSTR".to_string()); + } + } + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} diff --git a/src/misc.py b/src/misc.py new file mode 100644 index 0000000..1e507c7 --- /dev/null +++ b/src/misc.py @@ -0,0 +1,39 @@ +''' +https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py +''' +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes +import pyarrow as pa +from pyarrow.cffi import ffi + + c_schema = ffi.new('struct ArrowSchema*') + ptr_schema = int(ffi.cast('uintptr_t', c_schema)) + c_array = ffi.new('struct ArrowArray*') + ptr_array = int(ffi.cast('uintptr_t', c_array)) + + # pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32())) + + # import gc + # gc.collect() # Make sure no Arrow data dangles in a ref cycle + # pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema) + + arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema) + + diff --git a/src/record.rs b/src/record.rs new file mode 100644 index 0000000..cb64ea2 --- /dev/null +++ b/src/record.rs @@ -0,0 +1,430 @@ +/// +/// Generic record-level read/write functionality. +/// +use std::io::{self, Write}; + + +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] +use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType, +use crate::records; + + +//#[no_mangle] +//pub extern "C" fn write_record_header( + + +#[repr(C)] +#[derive(Debug, Clone)] +pub struct RecordHeader { + pub tag: u16, + pub data_size: u16, +} + +impl RecordHeader { + pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { + let (input, size) = parse_u16(input)?; + let (input, tag) = parse_u16(input)?; + let Some(data_size) = size.checked_sub(4) else { + return fail(input, format!("Record size {size} is smaller than its header")); + }; + Ok((input, RecordHeader{tag, data_size})) + } + + pub fn pack_into(&self) -> [u8; 4] { + let size = self.data_size.checked_add(4).expect("Record too big!"); + let [size_hi, size_lo] = size.to_be_bytes(); + let [tag_hi, tag_lo] = self.tag.to_be_bytes(); + [size_hi, size_lo, tag_hi, tag_lo] + } + + pub fn write(&self, ww: &mut W) -> OResult { + let bytes = self.pack_into(); + ww.write_all(&bytes)?; + Ok(bytes.len()) + } +} + + +pub trait RecordData { + type BareData<'a>; + type InData : ?Sized; + type ByteData : AsRef<[u8]>; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; + fn pack_into(buf: &mut [u8], data: &Self::InData); + //fn size(data: &Self::BareData<'_>) -> u16; + fn pack(data: &Self::InData) -> Self::ByteData; +} + + +pub trait Record { + fn tag() -> u16; + fn expected_size() -> Option; + + fn check_size(actual_size: u16) -> Result<(), String> { + match Self::expected_size() { + Some(size) => if size == actual_size { + Ok(()) + } else { + Err(format!("Expected record size {}, got {}", size, actual_size)) + }, + None => Ok(()), + } + } + + fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> { + RecordHeader::read(input) + } + + fn write_header(ww: &mut W, data_size: u16) -> OResult { + RecordHeader{tag: Self::tag(), data_size}.write(ww) + } + + fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + RData::read(input, size) + } + + fn pack_data(buf: &mut [u8], data: &RData::InData) { + RData::pack_into(buf, data) + } + + /// + /// Skip to the end of the next occurence of this record. + /// + /// Return: + /// True if the record was encountered and skipped. + /// False if the end of the library was reached. + /// + fn skip_past(input: &[u8]) -> IResult<'_, bool> { + let original_input = input; + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != Self::tag() { + (input, _) = take_bytes(input, header.data_size)?; + if header.tag == records::RTAG_ENDLIB { + return Ok((original_input, false)) + } + (input, header) = RecordHeader::read(input)?; + } + (input, _) = take_bytes(input, header.data_size)?; + Ok((input, true)) + } + + fn skip_and_read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let (mut input, mut header) = RecordHeader::read(input)?; + while header.tag != Self::tag() { + (input, _) = take_bytes(input, header.data_size)?; + (input, header) = RecordHeader::read(input)?; + } + let (input, data) = Self::read_data(input, header.data_size)?; + Ok((input, data)) + } + + fn expect_header(input: &[u8]) -> IResult<'_, u16> { + let (input, header) = RecordHeader::read(input)?; + if header.tag != Self::tag() { + fail(input, format!("Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", header.tag, Self::tag())) + } else { + Ok((input, header.data_size)) + } + } + + fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let (input, size) = Self::expect_header(input)?; + Self::check_size(size).unwrap(); + let (input, data) = Self::read_data(input, size)?; + Ok((input, data)) + } + + fn write(ww: &mut W, data: &RData::InData) -> OResult { + let packed_data = RData::pack(data); + let data_bytes = packed_data.as_ref(); + let len = u16::try_from(data_bytes.len()).ok() + .filter(|len| *len <= u16::MAX - 4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + Self::write_header(ww, len)?; + ww.write_all(data_bytes)?; + Ok(usize::from(len) + 4) + } +} + +pub struct BitArray; +impl RecordData for BitArray { + type BareData<'a> = [bool; 16]; + type InData = [bool; 16]; + type ByteData = [u8; 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2); + parse_bitarray(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_bitarray(buf, data); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct Int2; +impl RecordData for Int2 { + type BareData<'a> = i16; + type InData = i16; + type ByteData = [u8; 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2); + parse_int2(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_int2(buf, *data) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int4; +impl RecordData for Int4 { + type BareData<'a> = i32; + type InData = i32; + type ByteData = [u8; 4]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 4); + parse_int4(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_int4(buf, *data) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 4]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct Int2Array; +impl RecordData for Int2Array { + type BareData<'a> = Int2ArrayReader<'a>; + type InData = [i16]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers"); + //let mut input = input; + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int2ArrayReader{bytes})) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + assert_eq!(buf.len(), data.len() * 2); + for (bytes, value) in buf.as_chunks_mut::<2>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() * 2]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int2ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int2ArrayReader<'_> { + type Item = i16; + + fn next(&mut self) -> Option { + if self.bytes.len() < 2 { + None + } else { + let (remaining, val) = parse_int2(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + +pub struct Int4Array; +impl RecordData for Int4Array { + type BareData<'a> = Int4ArrayReader<'a>; + type InData = [i32]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers"); + //let mut input = input; + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int4ArrayReader{bytes})) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + assert_eq!(buf.len(), data.len() * 4); + for (bytes, value) in buf.as_chunks_mut::<4>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() * 4]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Int4ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int4ArrayReader<'_> { + type Item = i32; + + fn next(&mut self) -> Option { + if self.bytes.len() < 4 { + None + } else { + let (remaining, val) = parse_int4(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + +pub struct Real8; +impl RecordData for Real8 { + type BareData<'a> = f64; + type InData = f64; + type ByteData = [u8; 8]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 8); + parse_real8(input) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_real8(buf, *data).unwrap_or_else(|_| panic!("Float {0} too big for Real8", data)) + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 8]; + Self::pack_into(&mut buf, data); + buf + } +} + +pub struct Real8Pair; +impl RecordData for Real8Pair { + type BareData<'a> = (f64, f64); + type InData = (f64, f64); + type ByteData = [u8; 2 * 8]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2 * 8); + let (input, data0) = parse_real8(input)?; + let (input, data1) = parse_real8(input)?; + Ok((input, (data0, data1))) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_real8(buf, data.0).unwrap_or_else(|_| panic!("Float.0 {0} too big for Real8", data.0)); + pack_real8(&mut buf[8 * 1..], data.1).unwrap_or_else(|_| panic!("Float.1 {0} too big for Real8", data.1)); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2 * 8]; + Self::pack_into(&mut buf, data); + buf + } + + //fn write(ww: &mut W, data: &Self::BareData) -> OResult { + // let mut buf = [u8; 2 * 6 * 2]; + // Self::pack_into(&mut buf, data) + //} +} + +pub struct ASCII; +impl RecordData for ASCII { + type BareData<'a> = Vec; + type InData = [u8]; + type ByteData = Vec; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + parse_ascii(input, size) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_ascii(buf, data); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = vec![0; data.len() + data.len() % 2]; + Self::pack_into(&mut buf, data); + buf + } +} + + +pub struct DateTimePair; +impl RecordData for DateTimePair { + type BareData<'a> = [[i16; 6]; 2]; + type InData = [[i16; 6]; 2]; + type ByteData = [u8; 2 * 6 * 2]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 2 * 6 * 2); + let (input, data0) = parse_datetime(input)?; + let (input, data1) = parse_datetime(input)?; + Ok((input, [data0, data1])) + } + + fn pack_into(buf: &mut [u8], data: &Self::InData) { + pack_datetime(buf, &data[0]); + pack_datetime(&mut buf[6 * 2 * 1..], &data[1]); + } + + fn pack(data: &Self::InData) -> Self::ByteData { + let mut buf = [0; 2 * 6 * 2]; + Self::pack_into(&mut buf, data); + buf + } + + //fn write(ww: &mut W, data: &Self::BareData) -> OResult { + // let mut buf = [u8; 2 * 6 * 2]; + // Self::pack_into(&mut buf, data) + //} +} + +pub struct Empty; +impl RecordData for Empty { + type BareData<'a> = (); + type InData = (); + type ByteData = [u8; 0]; + + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + assert!(size == 0); + Ok((input, ())) + } + + fn pack_into(_buf: &mut [u8], _data: &Self::InData) { + } + + fn pack(_data: &Self::InData) -> Self::ByteData { + [] + } + //fn write(ww: &mut W, data: &Self::BareData) { + //} +} diff --git a/src/records.rs b/src/records.rs new file mode 100644 index 0000000..4e86404 --- /dev/null +++ b/src/records.rs @@ -0,0 +1,567 @@ +/// +/// Record type and tag definitions +/// + +use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; + +//use std::io::Write; + +// record tags +pub const RTAG_HEADER: u16 = 0x0002; +pub const RTAG_BGNLIB: u16 = 0x0102; +pub const RTAG_LIBNAME: u16 = 0x0206; +pub const RTAG_UNITS: u16 = 0x0305; // (user_units_per_db_unit, db_units_per_meter) +pub const RTAG_ENDLIB: u16 = 0x0400; +pub const RTAG_BGNSTR: u16 = 0x0502; +pub const RTAG_STRNAME: u16 = 0x0606; +pub const RTAG_ENDSTR: u16 = 0x0700; +pub const RTAG_BOUNDARY: u16 = 0x0800; +pub const RTAG_PATH: u16 = 0x0900; +pub const RTAG_SREF: u16 = 0x0a00; +pub const RTAG_AREF: u16 = 0x0b00; +pub const RTAG_TEXT: u16 = 0x0c00; +pub const RTAG_LAYER: u16 = 0x0d02; +pub const RTAG_DATATYPE: u16 = 0x0e02; +pub const RTAG_WIDTH: u16 = 0x0f03; +pub const RTAG_XY: u16 = 0x1003; +pub const RTAG_ENDEL: u16 = 0x1100; +pub const RTAG_SNAME: u16 = 0x1206; +pub const RTAG_COLROW: u16 = 0x1302; +pub const RTAG_NODE: u16 = 0x1500; +pub const RTAG_TEXTTYPE: u16 = 0x1602; +pub const RTAG_PRESENTATION: u16 = 0x1701; +pub const RTAG_SPACING: u16 = 0x1802; // unused; not sure about 02 +pub const RTAG_STRING: u16 = 0x1906; +pub const RTAG_STRANS: u16 = 0x1a01; +pub const RTAG_MAG: u16 = 0x1b05; +pub const RTAG_ANGLE: u16 = 0x1c05; +pub const RTAG_UINTEGER: u16 = 0x1d02; // unused; not sure about 02 +pub const RTAG_USTRING: u16 = 0x1e06; // unused; not sure about 06 +pub const RTAG_REFLIBS: u16 = 0x1f06; +pub const RTAG_FONTS: u16 = 0x2006; +pub const RTAG_PATHTYPE: u16 = 0x2102; +pub const RTAG_GENERATIONS: u16 = 0x2202; +pub const RTAG_ATTRTABLE: u16 = 0x2306; +pub const RTAG_STYPTABLE: u16 = 0x2406; // unused; not sure about 06 +pub const RTAG_STRTYPE: u16 = 0x2502; // unused +pub const RTAG_ELFLAGS: u16 = 0x2601; +pub const RTAG_ELKEY: u16 = 0x2703; // unused +pub const RTAG_LINKTYPE: u16 = 0x2803; // unused +pub const RTAG_LINKKEYS: u16 = 0x2903; // unused +pub const RTAG_NODETYPE: u16 = 0x2a02; +pub const RTAG_PROPATTR: u16 = 0x2b02; +pub const RTAG_PROPVALUE: u16 = 0x2c06; +pub const RTAG_BOX: u16 = 0x2d00; +pub const RTAG_BOXTYPE: u16 = 0x2e02; +pub const RTAG_PLEX: u16 = 0x2f03; +pub const RTAG_BGNEXTN: u16 = 0x3003; +pub const RTAG_ENDEXTN: u16 = 0x3103; +pub const RTAG_TAPENUM: u16 = 0x3202; +pub const RTAG_TAPECODE: u16 = 0x3302; +pub const RTAG_STRCLASS: u16 = 0x3401; +pub const RTAG_RESERVED: u16 = 0x3503; +pub const RTAG_FORMAT: u16 = 0x3602; +pub const RTAG_MASK: u16 = 0x3706; // list of Layers and dtypes +pub const RTAG_ENDMASKS: u16 = 0x3800; // end of MASKS records +pub const RTAG_LIBDIRSIZE: u16 = 0x3902; +pub const RTAG_SRFNAME: u16 = 0x3a06; +pub const RTAG_LIBSECUR: u16 = 0x3b02; +pub const RTAG_BORDER: u16 = 0x3c00; +pub const RTAG_SOFTFENCE: u16 = 0x3d00; +pub const RTAG_HARDFENCE: u16 = 0x3f00; +pub const RTAG_SOFTWIRE: u16 = 0x3f00; +pub const RTAG_HARDWIRE: u16 = 0x4000; +pub const RTAG_PATHPORT: u16 = 0x4100; +pub const RTAG_NODEPORT: u16 = 0x4200; +pub const RTAG_USERCONSTRAINT: u16 = 0x4300; +pub const RTAG_SPACERERROR: u16 = 0x4400; +pub const RTAG_CONTACT: u16 = 0x4500; + +/* +// data types +pub const DATA_TYPE_NONE: u16 = 0x00; +pub const DATA_TYPE_BIT: u16 = 0x01; +pub const DATA_TYPE_INT16: u16 = 0x02; +pub const DATA_TYPE_INT32: u16 = 0x03; +pub const DATA_TYPE_REAL32: u16 = 0x04; +pub const DATA_TYPE_REAL64: u16 = 0x05; +pub const DATA_TYPE_STR: u16 = 0x06; + +pub const MAX_DATA_SIZE: usize = 8; + +/// Returns the size of the given data type in bytes. +pub fn data_size(t: u16) -> Option { + match t { + x if x == DATA_TYPE_NONE => 0, + x if x == DATA_TYPE_BIT => 2, + x if x == DATA_TYPE_INT16 => 2, + x if x == DATA_TYPE_INT32 => 4, + x if x == DATA_TYPE_REAL32 => 4, + x if x == DATA_TYPE_REAL64 => 8, + _ => 0 + } +*/ + +pub struct HEADER; +impl Record for HEADER { + fn tag() -> u16 { RTAG_HEADER } + fn expected_size() -> Option { Some(2) } +} +//impl Record for HEADER; + +pub struct BGNLIB; +impl Record for BGNLIB { + fn tag() -> u16 { RTAG_BGNLIB } + fn expected_size() -> Option { Some(2 * 2 * 6) } +} + +pub struct LIBNAME; +impl Record for LIBNAME { + fn tag() -> u16 { RTAG_LIBNAME } + fn expected_size() -> Option { None } +} + +pub struct UNITS; +impl Record for UNITS { + // (user_units_per_db_unit, db_units_per_meter) + fn tag() -> u16 { RTAG_UNITS } + fn expected_size() -> Option { Some(2 * 8) } +} + +pub struct ENDLIB; +impl Record for ENDLIB { + fn tag() -> u16 { RTAG_ENDLIB } + fn expected_size() -> Option { Some(0) } +} + +pub struct BGNSTR; +impl Record for BGNSTR { + fn tag() -> u16 { RTAG_BGNSTR } + fn expected_size() -> Option { Some(2 * 6 * 2) } +} + +pub struct STRNAME; +impl Record for STRNAME { + fn tag() -> u16 { RTAG_STRNAME } + fn expected_size() -> Option { None } +} + +pub struct ENDSTR; +impl Record for ENDSTR { + fn tag() -> u16 { RTAG_ENDSTR } + fn expected_size() -> Option { Some(0) } +} + +pub struct BOUNDARY; +impl Record for BOUNDARY { + fn tag() -> u16 { RTAG_BOUNDARY } + fn expected_size() -> Option { Some(0) } +} + +pub struct PATH; +impl Record for PATH { + fn tag() -> u16 { RTAG_PATH } + fn expected_size() -> Option { Some(0) } +} + +pub struct SREF; +impl Record for SREF { + fn tag() -> u16 { RTAG_SREF } + fn expected_size() -> Option { Some(0) } +} + +pub struct AREF; +impl Record for AREF { + fn tag() -> u16 { RTAG_AREF } + fn expected_size() -> Option { Some(0) } +} + +pub struct TEXT; +impl Record for TEXT { + fn tag() -> u16 { RTAG_TEXT } + fn expected_size() -> Option { Some(0) } +} + +pub struct LAYER; +impl Record for LAYER { + fn tag() -> u16 { RTAG_LAYER } + fn expected_size() -> Option { Some(2) } +} + +pub struct DATATYPE; +impl Record for DATATYPE { + fn tag() -> u16 { RTAG_DATATYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct WIDTH; +impl Record for WIDTH { + fn tag() -> u16 { RTAG_WIDTH } + fn expected_size() -> Option { Some(4) } +} + +pub struct XY; +impl Record for XY { + fn tag() -> u16 { RTAG_XY } + fn expected_size() -> Option { None } +} + +pub struct ENDEL; +impl Record for ENDEL { + fn tag() -> u16 { RTAG_ENDEL } + fn expected_size() -> Option { Some(0) } +} + +pub struct SNAME; +impl Record for SNAME { + fn tag() -> u16 { RTAG_SNAME } + fn expected_size() -> Option { None } +} + +pub struct COLROW; +impl Record for COLROW { + fn tag() -> u16 { RTAG_COLROW } + fn expected_size() -> Option { Some(4) } +} + +pub struct NODE; +impl Record for NODE { + fn tag() -> u16 { RTAG_NODE } + fn expected_size() -> Option { Some(0) } +} + +pub struct TEXTTYPE; +impl Record for TEXTTYPE { + fn tag() -> u16 { RTAG_TEXTTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PRESENTATION; +impl Record for PRESENTATION { + fn tag() -> u16 { RTAG_PRESENTATION } + fn expected_size() -> Option { Some(2) } +} + +pub struct SPACING; +impl Record for SPACING { + fn tag() -> u16 { RTAG_SPACING } + fn expected_size() -> Option { Some(2) } +} + +pub struct STRING; +impl Record for STRING { + fn tag() -> u16 { RTAG_STRING } + fn expected_size() -> Option { None } +} + +pub struct STRANS; +impl Record for STRANS { + fn tag() -> u16 { RTAG_STRANS } + fn expected_size() -> Option { Some(2) } +} + +pub struct MAG; +impl Record for MAG { + fn tag() -> u16 { RTAG_MAG } + fn expected_size() -> Option { Some(8) } +} + +pub struct ANGLE; +impl Record for ANGLE { + fn tag() -> u16 { RTAG_ANGLE } + fn expected_size() -> Option { Some(8) } +} + +pub struct UINTEGER; +impl Record for UINTEGER { + fn tag() -> u16 { RTAG_UINTEGER } + fn expected_size() -> Option { Some(2) } +} + +pub struct USTRING; +impl Record for USTRING { + fn tag() -> u16 { RTAG_USTRING } + fn expected_size() -> Option { None } +} + +pub struct REFLIBS; +impl Record for REFLIBS { + fn tag() -> u16 { RTAG_REFLIBS } + fn expected_size() -> Option { None } +} +impl REFLIBS { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct FONTS; +impl Record for FONTS { + fn tag() -> u16 { RTAG_FONTS } + fn expected_size() -> Option { None } +} +impl FONTS { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct PATHTYPE; +impl Record for PATHTYPE { + fn tag() -> u16 { RTAG_PATHTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct GENERATIONS; +impl Record for GENERATIONS { + fn tag() -> u16 { RTAG_GENERATIONS } + fn expected_size() -> Option { Some(2) } +} + +pub struct ATTRTABLE; +impl Record for ATTRTABLE { + fn tag() -> u16 { RTAG_ATTRTABLE } + fn expected_size() -> Option { None } +} +impl ATTRTABLE { + pub fn check_size(actual_size: usize) -> Result<(), String> { + if actual_size.is_multiple_of(44) { + Ok(()) + } else { + Err(format!("Expected record size divisible by 44, got {}", actual_size)) + } + } +} + +pub struct STYPTABLE; +impl Record for STYPTABLE { + fn tag() -> u16 { RTAG_STYPTABLE } + fn expected_size() -> Option { None } +} + +pub struct STRTYPE; +impl Record for STRTYPE { + fn tag() -> u16 { RTAG_STRTYPE } + fn expected_size() -> Option { None } +} + +pub struct ELFLAGS; +impl Record for ELFLAGS { + fn tag() -> u16 { RTAG_ELFLAGS } + fn expected_size() -> Option { Some(2) } +} + +pub struct ELKEY; +impl Record for ELKEY { + fn tag() -> u16 { RTAG_ELKEY } + fn expected_size() -> Option { Some(2) } +} + +pub struct LINKTYPE; +impl Record for LINKTYPE { + fn tag() -> u16 { RTAG_LINKTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct LINKKEYS; +impl Record for LINKKEYS { + fn tag() -> u16 { RTAG_LINKKEYS } + fn expected_size() -> Option { Some(2) } +} + +pub struct NODETYPE; +impl Record for NODETYPE { + fn tag() -> u16 { RTAG_NODETYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PROPATTR; +impl Record for PROPATTR { + fn tag() -> u16 { RTAG_PROPATTR } + fn expected_size() -> Option { Some(2) } +} + +pub struct PROPVALUE; +impl Record for PROPVALUE { + fn tag() -> u16 { RTAG_PROPVALUE } + fn expected_size() -> Option { None } +} + +pub struct BOX; +impl Record for BOX { + fn tag() -> u16 { RTAG_BOX } + fn expected_size() -> Option { Some(0) } +} + +pub struct BOXTYPE; +impl Record for BOXTYPE { + fn tag() -> u16 { RTAG_BOXTYPE } + fn expected_size() -> Option { Some(2) } +} + +pub struct PLEX; +impl Record for PLEX { + fn tag() -> u16 { RTAG_PLEX } + fn expected_size() -> Option { Some(4) } +} + +pub struct BGNEXTN; +impl Record for BGNEXTN { + fn tag() -> u16 { RTAG_BGNEXTN } + fn expected_size() -> Option { Some(4) } +} + +pub struct ENDEXTN; +impl Record for ENDEXTN { + fn tag() -> u16 { RTAG_ENDEXTN } + fn expected_size() -> Option { Some(4) } +} + +pub struct TAPENUM; +impl Record for TAPENUM { + fn tag() -> u16 { RTAG_TAPENUM } + fn expected_size() -> Option { Some(2) } +} + +pub struct TAPECODE; +impl Record for TAPECODE { + fn tag() -> u16 { RTAG_TAPECODE } + fn expected_size() -> Option { Some(2 * 6) } +} + +pub struct STRCLASS; +impl Record for STRCLASS { + fn tag() -> u16 { RTAG_STRCLASS } + fn expected_size() -> Option { Some(2) } +} + +pub struct RESERVED; +impl Record for RESERVED { + fn tag() -> u16 { RTAG_RESERVED } + fn expected_size() -> Option { Some(2) } +} + +pub struct FORMAT; +impl Record for FORMAT { + fn tag() -> u16 { RTAG_FORMAT } + fn expected_size() -> Option { Some(2) } +} + +pub struct MASK; +impl Record for MASK { + fn tag() -> u16 { RTAG_MASK } + fn expected_size() -> Option { None } +} + +/// End of MASKS records +pub struct ENDMASKS; +impl Record for ENDMASKS { + fn tag() -> u16 { RTAG_ENDMASKS } + fn expected_size() -> Option { Some(0) } +} + +pub struct LIBDIRSIZE; +impl Record for LIBDIRSIZE { + fn tag() -> u16 { RTAG_LIBDIRSIZE } + fn expected_size() -> Option { Some(2) } +} + +pub struct SRFNAME; +impl Record for SRFNAME { + fn tag() -> u16 { RTAG_SRFNAME } + fn expected_size() -> Option { None } +} + +pub struct LIBSECUR; +impl Record for LIBSECUR { + fn tag() -> u16 { RTAG_LIBSECUR } + fn expected_size() -> Option { Some(2) } +} + +pub struct BORDER; +impl Record for BORDER { + fn tag() -> u16 { RTAG_BORDER } + fn expected_size() -> Option { Some(0) } +} + +pub struct SOFTFENCE; +impl Record for SOFTFENCE { + fn tag() -> u16 { RTAG_SOFTFENCE } + fn expected_size() -> Option { Some(0) } +} + +pub struct HARDFENCE; +impl Record for HARDFENCE { + fn tag() -> u16 { RTAG_HARDFENCE } + fn expected_size() -> Option { Some(0) } +} + +pub struct SOFTWIRE; +impl Record for SOFTWIRE { + fn tag() -> u16 { RTAG_SOFTWIRE } + fn expected_size() -> Option { Some(0) } +} + +pub struct HARDWIRE; +impl Record for HARDWIRE { + fn tag() -> u16 { RTAG_HARDWIRE } + fn expected_size() -> Option { Some(0) } +} + +pub struct PATHPORT; +impl Record for PATHPORT { + fn tag() -> u16 { RTAG_PATHPORT } + fn expected_size() -> Option { Some(0) } +} + +pub struct NODEPORT; +impl Record for NODEPORT { + fn tag() -> u16 { RTAG_NODEPORT } + fn expected_size() -> Option { Some(0) } +} + +pub struct USERCONSTRAINT; +impl Record for USERCONSTRAINT { + fn tag() -> u16 { RTAG_USERCONSTRAINT } + fn expected_size() -> Option { Some(0) } +} + +pub struct SPACERERROR; +impl Record for SPACERERROR { + fn tag() -> u16 { RTAG_SPACERERROR } + fn expected_size() -> Option { Some(0) } +} + +pub struct CONTACT; +impl Record for CONTACT { + fn tag() -> u16 { RTAG_CONTACT } + fn expected_size() -> Option { Some(0) } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bgnstr_reads_both_timestamps() { + let mut bytes = vec![0, 28, 0x05, 0x02]; + for value in [126_i16, 11, 14, 12, 30, 0, 125, 10, 13, 9, 15, 1] { + bytes.extend_from_slice(&value.to_be_bytes()); + } + bytes.extend_from_slice(&[0, 4, 0x07, 0]); + + let (remaining, timestamps) = BGNSTR::read(&bytes).unwrap(); + + assert_eq!(timestamps, [[2026, 11, 14, 12, 30, 0], [2025, 10, 13, 9, 15, 1]]); + assert_eq!(remaining, &[0, 4, 0x07, 0]); + } +} diff --git a/src/review.rs b/src/review.rs new file mode 100644 index 0000000..918ddeb --- /dev/null +++ b/src/review.rs @@ -0,0 +1,1492 @@ +//! Deterministic GDS object review without the normalization performed by the +//! Arrow reader. Raw payloads borrow the file; only polygon keys are copied. + +use crate::{basic::decode_real8, record::RecordHeader, records::*}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet}, + fmt::Write, +}; + +mod names; +pub use names::{parse_layer, parse_layer_name, write_layer_names}; +mod selection; +pub use selection::{DisplayFilter, Scope}; +mod changes; +pub use changes::Replacement; +pub mod report; + +/// A comparison limitation with source context independent of its wording. +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub code: &'static str, + pub message: String, + pub offset: Option, + pub cell: Option, + pub layer: Option, + pub target: Option, +} + +impl Diagnostic { + fn at(code: &'static str, offset: usize, message: String) -> Self { + Self { + code, + message: error(offset, message), + offset: Some(offset), + cell: None, + layer: None, + target: None, + } + } +} + +pub const PROFILE: &str = "klamath-object-v2"; +pub const LAYER_SENTINEL: &str = "KLAMATH_LAYER_NAMES_V1"; +pub type Layer = (u16, u16); +pub type Bounds = [i32; 4]; + +#[derive(Clone, Debug)] +pub struct Record<'a> { + pub offset: usize, + pub tag: u16, + pub data: &'a [u8], +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct Key<'a> { + kind: u16, + fields: Vec<(u16, Cow<'a, [u8]>)>, + properties: Vec<(u16, &'a [u8])>, +} + +#[derive(Debug)] +pub struct Object<'a> { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub multiplicity: u64, + key: Key<'a>, + metadata: bool, + issues: Vec, +} + +impl Object<'_> { + fn field(&self, tag: u16) -> Option<&[u8]> { + self.key + .fields + .iter() + .find(|(t, _)| *t == tag) + .map(|(_, v)| v.as_ref()) + } + + pub fn description(&self) -> String { + let mut out = kind_name(self.kind).to_string(); + for (tag, value) in &self.key.fields { + write!(out, " {}={}", tag_name(*tag), format_data(*tag, value)).unwrap(); + } + for (attribute, value) in &self.key.properties { + write!(out, " property[{attribute}]={}", quoted(value)).unwrap(); + } + out + } +} + +#[derive(Debug)] +pub struct Cell<'a> { + pub offset: usize, + /// Start of ENDSTR, for inserting records without rewriting the cell. + pub end_offset: usize, + pub objects: Vec>, + attributes: Vec<(u16, &'a [u8])>, + issues: Vec, +} + +#[derive(Debug)] +pub struct Document<'a> { + pub records: Vec>, + pub cells: BTreeMap>, + pub layer_names: BTreeMap, + /// Unsupported features or invalid reference graphs prevent an equality claim. + pub issues: Vec, + pub diagnostics: Vec, + header: Vec<(u16, &'a [u8])>, + global_issues: Vec, + known_cells: BTreeSet, + pub scope: Scope, + present: bool, +} + +fn error(offset: usize, message: impl std::fmt::Display) -> String { + format!("at byte {offset}: {message}") +} + +fn string(data: &[u8], offset: usize) -> Result { + decoded_string(unpad(data), offset) +} + +fn decoded_string(data: &[u8], offset: usize) -> Result { + if data.contains(&0) { + return Err(error(offset, "name or metadata contains an embedded NUL")); + } + std::str::from_utf8(data) + .map(str::to_owned) + .map_err(|_| error(offset, "name or metadata is not UTF-8")) +} + +fn unpad(data: &[u8]) -> &[u8] { + if data.last() == Some(&0) { + &data[..data.len() - 1] + } else { + data + } +} + +fn integer(data: &[u8]) -> u16 { + u16::from_be_bytes(data.try_into().unwrap()) +} + +fn points(data: &[u8]) -> Vec<[i32; 2]> { + data.as_chunks::<8>() + .0 + .iter() + .map(|&[x0, x1, x2, x3, y0, y1, y2, y3]| { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + }) + .collect() +} + +// Booth's least-rotation algorithm avoids quadratic work on repeated vertices. +fn least_rotation(points: &[[i32; 2]]) -> Vec<[i32; 2]> { + let n = points.len(); + let (mut i, mut j, mut k) = (0, 1, 0); + while i < n && j < n && k < n { + match points[(i + k) % n].cmp(&points[(j + k) % n]) { + std::cmp::Ordering::Equal => k += 1, + std::cmp::Ordering::Greater => { + i += k + 1; + if i <= j { + i = j + 1; + } + k = 0; + } + std::cmp::Ordering::Less => { + j += k + 1; + if j <= i { + j = i + 1; + } + k = 0; + } + } + } + let start = i.min(j); + (0..n).map(|k| points[(start + k) % n]).collect() +} + +fn canonical_ring(data: &[u8]) -> Vec { + let mut ring = points(data); + ring.pop(); // The caller has checked closure. + let forward = least_rotation(&ring); + ring.reverse(); + let backward = least_rotation(&ring); + let mut ring = forward.min(backward); + ring.push(ring[0]); + ring.into_iter() + .flatten() + .flat_map(i32::to_be_bytes) + .collect() +} + +fn validate_payload(record: &Record<'_>) -> Result<(), String> { + let size = record.data.len(); + let valid = match record.tag { + RTAG_BGNLIB | RTAG_BGNSTR => size == 24, + RTAG_UNITS => size == 16, + RTAG_HEADER | RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE + | RTAG_PROPATTR | RTAG_PATHTYPE | RTAG_STRANS | RTAG_PRESENTATION | RTAG_ELFLAGS + | RTAG_STRCLASS | RTAG_GENERATIONS => size == 2, + RTAG_WIDTH | RTAG_BGNEXTN | RTAG_ENDEXTN | RTAG_PLEX | RTAG_COLROW => size == 4, + RTAG_MAG | RTAG_ANGLE => size == 8, + RTAG_XY => size.is_multiple_of(8), + RTAG_ENDLIB | RTAG_ENDSTR | RTAG_ENDEL | RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF + | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX | RTAG_ENDMASKS => size == 0, + _ => true, + }; + if !valid { + return Err(error( + record.offset, + format!("invalid payload size {size} for {}", tag_name(record.tag)), + )); + } + if record.tag == RTAG_UNITS + && record.data.as_chunks::<8>().0.iter().any(|&bytes| { + let value = decode_real8(u64::from_be_bytes(bytes)); + !value.is_finite() || value <= 0.0 + }) + { + return Err(error(record.offset, "UNITS must be positive finite values")); + } + Ok(()) +} + +fn is_element(tag: u16) -> bool { + matches!( + tag, + RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX + ) +} + +fn allowed_field(kind: u16, tag: u16) -> bool { + if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) { + return true; + } + match kind { + RTAG_BOUNDARY => matches!(tag, RTAG_LAYER | RTAG_DATATYPE), + RTAG_BOX => matches!(tag, RTAG_LAYER | RTAG_BOXTYPE), + RTAG_NODE => matches!(tag, RTAG_LAYER | RTAG_NODETYPE), + RTAG_PATH => matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_WIDTH | RTAG_PATHTYPE | RTAG_BGNEXTN | RTAG_ENDEXTN + ), + RTAG_SREF => matches!(tag, RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE), + RTAG_AREF => matches!( + tag, + RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE | RTAG_COLROW + ), + RTAG_TEXT => matches!( + tag, + RTAG_LAYER + | RTAG_TEXTTYPE + | RTAG_STRING + | RTAG_STRANS + | RTAG_MAG + | RTAG_ANGLE + | RTAG_PRESENTATION + | RTAG_PATHTYPE + | RTAG_WIDTH + ), + _ => false, + } +} + +fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result, String> { + let first = &records[0]; + let mut fields = Vec::new(); + let mut properties = Vec::new(); + let mut seen = BTreeSet::new(); + let mut i = 1; + while i + 1 < records.len() { + let record = &records[i]; + if record.tag == RTAG_PROPATTR { + let value = records + .get(i + 1) + .filter(|r| r.tag == RTAG_PROPVALUE) + .ok_or_else(|| error(record.offset, "PROPATTR without PROPVALUE"))?; + properties.push((integer(record.data), unpad(value.data))); + i += 2; + continue; + } + if !allowed_field(first.tag, record.tag) { + issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!( + "unsupported record 0x{:04x} in {}", + record.tag, + kind_name(first.tag) + ), + )); + } + if !seen.insert(record.tag) { + return Err(error(record.offset, "duplicate element field")); + } + let mut data = Cow::Borrowed(record.data); + if record.tag & 0xff == 6 { + data = Cow::Borrowed(unpad(record.data)); + } + if record.tag == RTAG_XY && matches!(first.tag, RTAG_BOUNDARY | RTAG_BOX) { + let p = points(record.data); + if p.len() < 4 || p.first() != p.last() { + return Err(error( + record.offset, + "polygon must have at least three vertices and be closed", + )); + } + data = Cow::Owned(canonical_ring(record.data)); + } + fields.push((record.tag, data)); + i += 1; + } + fields.sort(); + properties.sort(); + let mut object = Object { + offset: first.offset, + end_offset: records.last().unwrap().offset + 4, + kind: first.tag, + layer: None, + bounds: None, + target: None, + multiplicity: 1, + key: Key { + kind: first.tag, + fields, + properties, + }, + metadata: false, + issues: Vec::new(), + }; + let xy = object + .field(RTAG_XY) + .ok_or_else(|| error(first.offset, "element has no XY record"))?; + let p = points(xy); + let valid_count = match first.tag { + RTAG_TEXT | RTAG_SREF => p.len() == 1, + RTAG_AREF => p.len() == 3, + RTAG_PATH => p.len() >= 2, + RTAG_NODE => !p.is_empty(), + RTAG_BOX => p.len() == 5, + _ => true, + }; + if !valid_count { + return Err(error(first.offset, "invalid coordinate count")); + } + object.bounds = Some([ + p.iter().map(|p| p[0]).min().unwrap(), + p.iter().map(|p| p[1]).min().unwrap(), + p.iter().map(|p| p[0]).max().unwrap(), + p.iter().map(|p| p[1]).max().unwrap(), + ]); + if matches!(first.tag, RTAG_SREF | RTAG_AREF) { + object.target = Some(decoded_string( + object + .field(RTAG_SNAME) + .ok_or_else(|| error(first.offset, "reference has no SNAME"))?, + first.offset, + )?); + if first.tag == RTAG_AREF { + let counts = object + .field(RTAG_COLROW) + .ok_or_else(|| error(first.offset, "array has no COLROW"))?; + let a = i16::from_be_bytes(counts[..2].try_into().unwrap()); + let b = i16::from_be_bytes(counts[2..].try_into().unwrap()); + if a <= 0 || b <= 0 { + return Err(error(first.offset, "array counts must be positive")); + } + object.multiplicity = a as u64 * b as u64; + } + } else { + let dtype = match first.tag { + RTAG_TEXT => RTAG_TEXTTYPE, + RTAG_BOX => RTAG_BOXTYPE, + RTAG_NODE => RTAG_NODETYPE, + _ => RTAG_DATATYPE, + }; + object.layer = Some(( + integer( + object + .field(RTAG_LAYER) + .ok_or_else(|| error(first.offset, "element has no LAYER"))?, + ), + integer( + object + .field(dtype) + .ok_or_else(|| error(first.offset, "element has no datatype/texttype"))?, + ), + )); + if first.tag == RTAG_TEXT && object.field(RTAG_STRING).is_none() { + return Err(error(first.offset, "text has no STRING")); + } + } + Ok(object) +} + +fn read_records(input: &[u8]) -> Result>, String> { + let mut records = Vec::new(); + let mut offset = 0; + while offset < input.len() { + let (payload, header) = RecordHeader::read(&input[offset..]) + .map_err(|(_, e)| error(offset, format!("invalid record header: {e:?}")))?; + let size = header.data_size as usize; + if !(size + 4).is_multiple_of(2) { + return Err(error(offset, "odd record length")); + } + let data = payload + .get(..size) + .ok_or_else(|| error(offset, "truncated record"))?; + let record = Record { + offset, + tag: header.tag, + data, + }; + offset += size + 4; + let ended = record.tag == RTAG_ENDLIB; + records.push(record); + if ended { + if input[offset..].iter().any(|b| *b != 0) { + return Err(error(offset, "nonzero data after ENDLIB")); + } + break; + } + } + Ok(records) +} + +impl<'a> Document<'a> { + pub fn parse(input: &'a [u8]) -> Result { + let records = read_records(input)?; + for record in &records { + validate_payload(record)?; + } + if records.first().map(|r| r.tag) != Some(RTAG_HEADER) + || records.get(1).map(|r| r.tag) != Some(RTAG_BGNLIB) + || records.last().map(|r| r.tag) != Some(RTAG_ENDLIB) + { + return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string()); + } + let mut doc = Self { + records, + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + issues: Vec::new(), + diagnostics: Vec::new(), + header: Vec::new(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: Scope::default(), + present: true, + }; + let mut i = 2; + let mut global_seen = BTreeSet::new(); + let mut cells_started = false; + doc.header.push((RTAG_HEADER, doc.records[0].data)); + while i + 1 < doc.records.len() { + let record = &doc.records[i]; + if record.tag != RTAG_BGNSTR { + if cells_started { + return Err(error( + record.offset, + "library field outside the library header", + )); + } + if record.tag != RTAG_MASK && !global_seen.insert(record.tag) { + return Err(error(record.offset, "duplicate library field")); + } + if !matches!( + record.tag, + RTAG_LIBNAME + | RTAG_UNITS + | RTAG_REFLIBS + | RTAG_FONTS + | RTAG_GENERATIONS + | RTAG_ATTRTABLE + | RTAG_FORMAT + | RTAG_MASK + | RTAG_ENDMASKS + | RTAG_LIBDIRSIZE + | RTAG_SRFNAME + | RTAG_LIBSECUR + ) { + doc.global_issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported library record 0x{:04x}", record.tag), + )); + } + doc.header.push(( + record.tag, + if record.tag & 0xff == 6 { + unpad(record.data) + } else { + record.data + }, + )); + i += 1; + continue; + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err(error(record.offset, "LIBNAME and UNITS must precede cells")); + } + cells_started = true; + let offset = record.offset; + i += 1; + let name_record = doc + .records + .get(i) + .filter(|r| r.tag == RTAG_STRNAME) + .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; + let name = string(name_record.data, name_record.offset)?; + if name.is_empty() { + return Err(error(name_record.offset, "empty cell name")); + } + i += 1; + let mut cell = Cell { + offset, + end_offset: 0, + objects: Vec::new(), + attributes: Vec::new(), + issues: Vec::new(), + }; + loop { + let record = doc + .records + .get(i) + .ok_or_else(|| error(offset, "missing ENDSTR"))?; + if record.tag == RTAG_ENDSTR { + cell.end_offset = record.offset; + i += 1; + break; + } + if is_element(record.tag) { + let start = i; + i += 1; + while doc.records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { + if matches!(doc.records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) + || is_element(doc.records[i].tag) + { + return Err(error(doc.records[i].offset, "missing ENDEL")); + } + i += 1; + } + if i == doc.records.len() { + return Err(error(offset, "missing ENDEL")); + } + let mut issues = Vec::new(); + let mut parsed = object(&doc.records[start..=i], &mut issues)?; + parsed.issues = issues; + cell.objects.push(parsed); + i += 1; + } else { + if matches!(record.tag, RTAG_ENDLIB | RTAG_BGNSTR) { + return Err(error(record.offset, "missing ENDSTR")); + } + if record.tag != RTAG_STRCLASS { + cell.issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported cell record 0x{:04x}", record.tag), + )); + } else if !cell.objects.is_empty() + || cell.attributes.iter().any(|(tag, _)| *tag == RTAG_STRCLASS) + { + return Err(error( + record.offset, + "STRCLASS must occur once before elements", + )); + } + cell.attributes.push((record.tag, record.data)); + i += 1; + } + } + cell.attributes.sort(); + cell.objects.sort_by(|a, b| a.key.cmp(&b.key)); + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err("library must contain LIBNAME and UNITS".to_string()); + } + doc.header.sort(); + doc.read_metadata()?; + doc.known_cells = doc.cells.keys().cloned().collect(); + doc.refresh_issues(); + Ok(doc) + } + + fn read_metadata(&mut self) -> Result<(), String> { + for (cell_name, cell) in &mut self.cells { + for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { + let text = object.field(RTAG_STRING).unwrap(); + if text.starts_with(b"KLAMATH_LAYER_NAMES_") && text != LAYER_SENTINEL.as_bytes() { + // Layer names apply across the library. An unknown version + // may describe layers used outside this carrier or cell, so + // geometry scope selection cannot establish its relevance. + let mut diagnostic = Diagnostic::at( + "unsupported_metadata_version", + object.offset, + "unsupported layer-name metadata version".to_string(), + ); + diagnostic.cell = Some(cell_name.clone()); + diagnostic.layer = object.layer; + diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); + self.global_issues.push(diagnostic); + } + } + let carriers: BTreeSet<_> = cell + .objects + .iter() + .filter(|o| { + o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) + }) + .filter_map(|o| o.layer) + .collect(); + for object in &mut cell.objects { + if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) + { + continue; + } + let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; + if text != LAYER_SENTINEL { + let ((layer, dtype), name) = + parse_layer_name(&text).map_err(|message| error(object.offset, message))?; + if self + .layer_names + .get(&(layer, dtype)) + .is_some_and(|old| old != &name) + { + return Err(error( + object.offset, + format!("conflicting names for {layer}/{dtype}"), + )); + } + self.layer_names.insert((layer, dtype), name.to_string()); + } + object.metadata = true; + } + } + Ok(()) + } + + fn check_graph(&mut self) { + let mut unresolved = BTreeMap::new(); + for (parent, cell) in &self.cells { + for object in &cell.objects { + if let Some(target) = &object.target + && !self.known_cells.contains(target) + { + unresolved.entry((parent, target)).or_insert(object.offset); + } + } + } + for ((parent, target), offset) in unresolved { + self.diagnostics.push(Diagnostic { + code: "unresolved_reference", + message: format!("cell {parent:?}: unresolved reference {target:?}"), + offset: Some(offset), + cell: Some(parent.clone()), + layer: None, + target: Some(target.clone()), + }); + } + let mut state = BTreeMap::<&str, u8>::new(); + for root in self.cells.keys() { + let mut stack = vec![(root.as_str(), false)]; + while let Some((name, leaving)) = stack.pop() { + if leaving { + state.insert(name, 2); + continue; + } + match state.get(name) { + Some(2) => continue, + Some(1) => { + self.diagnostics.push(Diagnostic { + code: "hierarchy_cycle", + message: format!("cyclic hierarchy involving {name:?}"), + offset: self.cells.get(name).map(|c| c.offset), + cell: Some(name.to_string()), + layer: None, + target: None, + }); + continue; + } + _ => (), + } + state.insert(name, 1); + stack.push((name, true)); + if let Some(cell) = self.cells.get(name) { + for object in &cell.objects { + if let Some(target) = &object.target { + stack.push((target, false)); + } + } + } + } + } + } + + pub fn edges(&self) -> BTreeMap<(String, String), u64> { + let mut edges = BTreeMap::new(); + for (name, cell) in &self.cells { + for object in &cell.objects { + if let Some(target) = &object.target { + let count = edges.entry((name.clone(), target.clone())).or_insert(0_u64); + *count = count.saturating_add(object.multiplicity); + } + } + } + edges + } + + pub fn canonical_dump(&self) -> String { + let mut out = format!( + "profile {PROFILE}\n{}\ncoordinates: integer database units; bounds: coordinate envelopes\n", + self.scope.description() + ); + for (tag, data) in &self.header { + writeln!(out, "{}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + } + for ((layer, dtype), name) in &self.layer_names { + writeln!(out, "layer {layer}/{dtype} name={name:?}").unwrap(); + } + for (name, cell) in &self.cells { + writeln!(out, "cell {name:?}").unwrap(); + for (tag, data) in &cell.attributes { + writeln!(out, " {}={}", tag_name(*tag), format_data(*tag, data)).unwrap(); + } + for object in cell.objects.iter().filter(|o| !o.metadata) { + writeln!(out, " {}", object.description()).unwrap(); + } + } + for issue in &self.issues { + writeln!(out, "INCOMPLETE: {issue}").unwrap(); + } + out + } + + pub fn record_dump(&self) -> String { + format_records(&self.records) + } +} + +/// Inspect record boundaries even when object grammar or metadata is invalid. +pub fn dump_records(input: &[u8]) -> Result { + Ok(format_records(&read_records(input)?)) +} + +fn format_records(records: &[Record<'_>]) -> String { + let mut out = String::new(); + for record in records { + writeln!( + out, + "{:010} {:04x} {} size={} {}", + record.offset, + record.tag, + tag_name(record.tag), + record.data.len() + 4, + format_data( + record.tag, + if record.tag & 0xff == 6 { + unpad(record.data) + } else { + record.data + } + ) + ) + .unwrap(); + } + out +} + +/// A retained comparison field. Polygon coordinates and string padding follow +/// the comparison profile; REAL encodings and unsupported payloads are exact. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FieldValue { + pub tag: u16, + pub data: Vec, +} + +fn owned_fields<'a>(fields: impl IntoIterator) -> Vec { + fields + .into_iter() + .map(|(tag, data)| FieldValue { + tag, + data: data.to_vec(), + }) + .collect() +} + +#[derive(Debug)] +pub struct ObjectChange { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub origin: Option<[i32; 2]>, + pub width: Option, + pub counts: Option<[u16; 2]>, + pub description: String, + pub fields: Vec, + pub properties: Vec<(u16, Vec)>, +} + +impl From<&Object<'_>> for ObjectChange { + fn from(object: &Object<'_>) -> Self { + Self { + offset: object.offset, + end_offset: object.end_offset, + kind: object.kind, + layer: object.layer, + bounds: object.bounds, + target: object.target.clone(), + origin: object.field(RTAG_XY).map(|data| { + [ + i32::from_be_bytes(data[..4].try_into().unwrap()), + i32::from_be_bytes(data[4..8].try_into().unwrap()), + ] + }), + width: object + .field(RTAG_WIDTH) + .map(|data| i32::from_be_bytes(data.try_into().unwrap())), + counts: object.target.as_ref().map(|_| { + if object.kind == RTAG_AREF { + let data = object.field(RTAG_COLROW).unwrap(); + [integer(&data[..2]), integer(&data[2..])] + } else { + [1, 1] + } + }), + description: object.description(), + fields: owned_fields( + object + .key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + ), + properties: object + .key + .properties + .iter() + .map(|(a, v)| (*a, v.to_vec())) + .collect(), + } + } +} + +#[derive(Debug)] +pub struct CellChange { + pub name: String, + pub old_offset: Option, + pub new_offset: Option, + pub attributes_changed: bool, + pub old_attributes: Vec, + pub new_attributes: Vec, + pub removed: Vec, + pub added: Vec, + pub replacements: Vec, +} + +#[derive(Debug)] +pub struct LayerNameChange { + pub layer: Layer, + pub old: Option, + pub new: Option, +} + +#[derive(Debug)] +pub struct LibraryFieldChange { + pub field: String, + pub tag: Option, + pub old_fields: Vec, + pub new_fields: Vec, + pub old: Vec, + pub new: Vec, +} + +#[derive(Debug)] +pub struct Comparison { + pub scope: Scope, + /// Unchanged names are displayed once when objects on that layer change. + pub layer_legend: BTreeMap, + pub library_fields: Vec, + pub layer_names: Vec, + pub cells: Vec, + pub issues: Vec, +} + +impl Comparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + let mut result = Self { + scope: old.scope.clone(), + layer_legend: old + .layer_names + .iter() + .filter(|(pair, name)| new.layer_names.get(pair) == Some(*name)) + .map(|(pair, name)| (*pair, name.clone())) + .collect(), + library_fields: Vec::new(), + layer_names: Vec::new(), + cells: Vec::new(), + issues: Vec::new(), + }; + if old.scope != new.scope { + result.issues.push("comparison scopes differ".to_string()); + } + if old.present != new.present { + result.library_fields.push(LibraryFieldChange { + field: "presence".to_string(), + tag: None, + old_fields: Vec::new(), + new_fields: Vec::new(), + old: vec![if old.present { "present" } else { "absent" }.to_string()], + new: vec![if new.present { "present" } else { "absent" }.to_string()], + }); + } + for (side, doc) in [("old", old), ("new", new)] { + result + .issues + .extend(doc.issues.iter().map(|i| format!("{side}: {i}"))); + } + let header_tags: BTreeSet<_> = old + .header + .iter() + .chain(&new.header) + .map(|(tag, _)| *tag) + .collect(); + for tag in header_tags { + let a: Vec<_> = old + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + let b: Vec<_> = new + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + if a != b { + result.library_fields.push(LibraryFieldChange { + field: tag_name(tag), + tag: Some(tag), + old_fields: owned_fields(a.iter().map(|data| (tag, *data))), + new_fields: owned_fields(b.iter().map(|data| (tag, *data))), + old: a.iter().map(|data| format_data(tag, data)).collect(), + new: b.iter().map(|data| format_data(tag, data)).collect(), + }); + } + } + let layers: BTreeSet<_> = old + .layer_names + .keys() + .chain(new.layer_names.keys()) + .copied() + .collect(); + for layer in layers { + if old.layer_names.get(&layer) != new.layer_names.get(&layer) { + result.layer_names.push(LayerNameChange { + layer, + old: old.layer_names.get(&layer).cloned(), + new: new.layer_names.get(&layer).cloned(), + }); + } + } + let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect(); + for name in names { + let a = old.cells.get(name); + let b = new.cells.get(name); + let mut change = CellChange { + name: name.clone(), + old_offset: a.map(|c| c.offset), + new_offset: b.map(|c| c.offset), + attributes_changed: a.map(|c| &c.attributes) != b.map(|c| &c.attributes), + old_attributes: owned_fields( + a.into_iter().flat_map(|c| c.attributes.iter().copied()), + ), + new_attributes: owned_fields( + b.into_iter().flat_map(|c| c.attributes.iter().copied()), + ), + removed: Vec::new(), + added: Vec::new(), + replacements: Vec::new(), + }; + let a: Vec<_> = a + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let b: Vec<_> = b + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let (mut i, mut j) = (0, 0); + while i < a.len() || j < b.len() { + let order = match (a.get(i), b.get(j)) { + (Some(a), Some(b)) => a.key.cmp(&b.key), + (Some(_), None) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Greater, + }; + match order { + std::cmp::Ordering::Equal => { + i += 1; + j += 1; + } + std::cmp::Ordering::Less => { + change.removed.push(a[i].into()); + i += 1; + } + std::cmp::Ordering::Greater => { + change.added.push(b[j].into()); + j += 1; + } + } + } + if change.old_offset.is_none() + || change.new_offset.is_none() + || change.attributes_changed + || !change.removed.is_empty() + || !change.added.is_empty() + { + change.replacements = changes::replacements(&change.removed, &change.added); + result.cells.push(change); + } + } + result + } + + pub fn is_equal(&self) -> bool { + self.issues.is_empty() + && self.library_fields.is_empty() + && self.layer_names.is_empty() + && self.cells.is_empty() + } + + pub fn exit_code(&self) -> u8 { + if !self.issues.is_empty() { + 2 + } else if self.is_equal() { + 0 + } else { + 1 + } + } + + /// Limit presentation only: the complete comparison has already been made. + pub fn text(&self, max_objects: usize) -> String { + self.text_filtered(max_objects, &DisplayFilter::default()) + } + + pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { + let mut out = format!("profile {PROFILE}\n{}\n", self.scope.description()); + if show.cell.is_some() || show.layer.is_some() { + writeln!(out, "display-only filter cell={:?} layer={:?}; status, totals and hierarchy are unfiltered", show.cell, show.layer).unwrap(); + } + for change in &self.library_fields { + writeln!( + out, + "library {}: {:?} -> {:?}", + change.field, change.old, change.new + ) + .unwrap(); + } + if self.library_fields.iter().any(|c| c.field == "UNITS") { + out.push_str("UNITS changed; coordinates remain in each file's database units\n"); + } + for change in &self.layer_names { + writeln!( + out, + "layer {}/{} name: {:?} -> {:?}", + change.layer.0, change.layer.1, change.old, change.new + ) + .unwrap(); + } + let changed_layers: BTreeSet<_> = self + .cells + .iter() + .filter(|cell| show.cell.as_ref().is_none_or(|name| name == &cell.name)) + .flat_map(|cell| cell.removed.iter().chain(&cell.added)) + .filter_map(|object| object.layer) + .filter(|layer| show.layer.is_none_or(|selected| selected == *layer)) + .collect(); + for layer in changed_layers { + if let Some(name) = self.layer_legend.get(&layer) { + writeln!(out, "layer {}/{} name={name:?}", layer.0, layer.1).unwrap(); + } + } + let total: usize = self + .cells + .iter() + .map(|c| c.removed.len() + c.added.len()) + .sum(); + let mut shown: usize = 0; + for cell in &self.cells { + if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + continue; + } + writeln!( + out, + "cell {:?} old@{:?} new@{:?}{}", + cell.name, + cell.old_offset, + cell.new_offset, + if cell.attributes_changed { + " [cell/header changed]" + } else { + "" + } + ) + .unwrap(); + let mut paired_old = BTreeSet::new(); + let mut paired_new = BTreeSet::new(); + for replacement in &cell.replacements { + let a = &cell.removed[replacement.removed]; + let b = &cell.added[replacement.added]; + if shown.saturating_add(2) <= max_objects + && show + .layer + .is_none_or(|layer| a.layer == Some(layer) && b.layer == Some(layer)) + { + writeln!(out, " ~ {}", replacement.explanation).unwrap(); + writeln!( + out, + " - @{} bbox={:?} {}", + a.offset, a.bounds, a.description + ) + .unwrap(); + writeln!( + out, + " + @{} bbox={:?} {}", + b.offset, b.bounds, b.description + ) + .unwrap(); + paired_old.insert(replacement.removed); + paired_new.insert(replacement.added); + shown += 2; + } + } + for (sign, objects, paired) in [ + ("-", &cell.removed, &paired_old), + ("+", &cell.added, &paired_new), + ] { + for (index, object) in objects.iter().enumerate() { + if paired.contains(&index) { + continue; + } + if show.layer.is_some_and(|layer| object.layer != Some(layer)) { + continue; + } + if shown < max_objects { + writeln!( + out, + " {sign} @{} bbox={:?} {}", + object.offset, object.bounds, object.description + ) + .unwrap(); + shown += 1; + } + } + } + } + writeln!( + out, + "{} changed cells; {shown} of {total} object changes shown; {} layer-name changes", + self.cells.len(), + self.layer_names.len() + ) + .unwrap(); + for issue in &self.issues { + writeln!(out, "INCOMPLETE: {issue}").unwrap(); + } + if self.is_equal() { + out.push_str("equal under this profile\n"); + } + out + } +} + +/// The graph joins both revisions, retaining removed branches and old callers. +pub fn hierarchy( + old: &Document<'_>, + new: &Document<'_>, + diff: Option<&Comparison>, + mermaid: bool, + max_depth: usize, +) -> String { + let old_edges = old.edges(); + let new_edges = new.edges(); + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = diff + .into_iter() + .flat_map(|d| d.cells.iter().map(|c| c.name.clone())) + .collect(); + let changed_refs: BTreeSet<_> = diff + .into_iter() + .flat_map(|d| &d.cells) + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + let visible: BTreeSet<_> = if diff.is_some() { + affected + .iter() + .cloned() + .chain(changed_refs.iter().map(|(_, t)| t.clone())) + .collect() + } else { + old.cells + .keys() + .chain(new.cells.keys()) + .cloned() + .chain(edges.iter().map(|(_, t)| t.clone())) + .collect() + }; + let mut children = BTreeMap::>::new(); + let mut targets = BTreeSet::new(); + for (parent, target) in &edges { + if visible.contains(parent) && visible.contains(target) { + children + .entry(parent.clone()) + .or_default() + .push(target.clone()); + targets.insert(target.clone()); + } + } + let roots: Vec<_> = visible.difference(&targets).cloned().collect(); + // Distinguish rootless cycles from descendants hidden by the depth limit. + let mut reachable = BTreeSet::new(); + let mut pending = roots.clone(); + while let Some(name) = pending.pop() { + if reachable.insert(name.clone()) { + pending.extend(children.get(&name).into_iter().flatten().cloned()); + } + } + let mut included = BTreeSet::new(); + let mut rows = Vec::new(); + let mut stack: Vec<_> = roots + .iter() + .rev() + .map(|n| (n.clone(), 0, None::)) + .collect(); + while let Some((name, depth, parent)) = stack.pop() { + let seen = !included.insert(name.clone()); + let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); + rows.push(( + name.clone(), + depth, + parent, + seen, + depth >= max_depth && !descendants.is_empty(), + )); + if !seen && depth < max_depth { + for child in descendants.iter().rev() { + stack.push((child.clone(), depth + 1, Some(name.clone()))); + } + } + } + for name in visible.difference(&reachable) { + rows.push((name.clone(), 0, None, false, true)); + } + let status = |name: &String| { + if !old.cells.contains_key(name) && !new.cells.contains_key(name) { + if old.known_cells.contains(name) || new.known_cells.contains(name) { + " [outside scope]" + } else { + " [unresolved]" + } + } else if direct.contains(name) { + " [cell changed]" + } else if affected.contains(name) { + " [affected]" + } else if diff.is_some() { + " [cell unchanged]" + } else { + "" + } + }; + let edge_label = |parent: &String, target: &String| { + let key = (parent.clone(), target.clone()); + let a = old_edges.get(&key).copied().unwrap_or(0); + let b = new_edges.get(&key).copied().unwrap_or(0); + let count = if a == b { + format!("x{b}") + } else { + format!("x{a} -> x{b}") + }; + format!( + "{count}{}", + if changed_refs.contains(&key) { + " [reference changed]" + } else { + "" + } + ) + }; + if mermaid { + let ids: BTreeMap<_, _> = rows + .iter() + .map(|r| &r.0) + .collect::>() + .into_iter() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + let mut out = format!("flowchart TD\n %% {}\n", new.scope.description()); + for (name, id) in &ids { + let label = format!("{name}{}", status(name)); + let label = label + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") + .replace(['\r', '\n'], " "); + writeln!(out, " n{id}[\"{label}\"]").unwrap(); + } + for (parent, target) in &edges { + if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { + writeln!(out, " n{a} -->|\"{}\"| n{b}", edge_label(parent, target)).unwrap(); + } + } + if rows.is_empty() { + out.push_str(" empty[\"No changed hierarchy\"]\n"); + } + out + } else { + let mut out = format!("{}\n", new.scope.description()); + for (name, depth, parent, seen, limited) in rows { + let count = parent + .map(|p| edge_label(&p, &name)) + .unwrap_or_else(|| "x1".to_string()); + writeln!( + out, + "{}{name:?} {count}{}{}{}", + " ".repeat(depth), + status(&name), + if seen { " [shown above]" } else { "" }, + if limited { " [children omitted]" } else { "" } + ) + .unwrap(); + } + if visible.is_empty() { + out.push_str("No changed hierarchy\n"); + } + out + } +} + +fn quoted(data: &[u8]) -> String { + let mut out = "\"".to_string(); + for byte in data { + for escaped in std::ascii::escape_default(*byte) { + out.push(escaped as char); + } + } + out.push('"'); + out +} + +fn format_data(tag: u16, data: &[u8]) -> String { + if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_BOXTYPE | RTAG_NODETYPE + ) && data.len() == 2 + { + return integer(data).to_string(); + } + match tag & 0xff { + 0 if data.is_empty() => String::new(), + 2 if data.len().is_multiple_of(2) => format!( + "{:?}", + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + 3 if data.len().is_multiple_of(4) => format!( + "{:?}", + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + 5 if data.len().is_multiple_of(8) => format!( + "{:?} raw={}", + data.as_chunks::<8>() + .0 + .iter() + .map(|&b| decode_real8(u64::from_be_bytes(b))) + .collect::>(), + hex(data) + ), + 6 => quoted(data), + _ => hex(data), + } +} + +fn hex(data: &[u8]) -> String { + let mut out = "0x".to_string(); + for byte in data { + write!(out, "{byte:02x}").unwrap(); + } + out +} + +fn kind_name(tag: u16) -> &'static str { + match tag { + RTAG_BOUNDARY => "BOUNDARY", + RTAG_PATH => "PATH", + RTAG_SREF => "SREF", + RTAG_AREF => "AREF", + RTAG_TEXT => "TEXT", + RTAG_NODE => "NODE", + RTAG_BOX => "BOX", + _ => "UNKNOWN", + } +} + +fn tag_name(tag: u16) -> String { + let name = match tag { + RTAG_HEADER => "HEADER", + RTAG_BGNLIB => "BGNLIB", + RTAG_LIBNAME => "LIBNAME", + RTAG_UNITS => "UNITS", + RTAG_ENDLIB => "ENDLIB", + RTAG_BGNSTR => "BGNSTR", + RTAG_STRNAME => "STRNAME", + RTAG_ENDSTR => "ENDSTR", + RTAG_ENDEL => "ENDEL", + RTAG_LAYER => "layer", + RTAG_DATATYPE => "datatype", + RTAG_TEXTTYPE => "texttype", + RTAG_XY => "xy", + RTAG_SNAME => "target", + RTAG_COLROW => "counts", + RTAG_STRANS => "strans", + RTAG_MAG => "mag", + RTAG_ANGLE => "angle", + RTAG_WIDTH => "width", + RTAG_PATHTYPE => "pathtype", + RTAG_STRING => "string", + RTAG_PROPATTR => "PROPATTR", + RTAG_PROPVALUE => "PROPVALUE", + RTAG_PRESENTATION => "presentation", + RTAG_BGNEXTN => "bgnextn", + RTAG_ENDEXTN => "endextn", + RTAG_BOXTYPE => "boxtype", + RTAG_NODETYPE => "nodetype", + RTAG_ELFLAGS => "elflags", + RTAG_PLEX => "plex", + RTAG_STRCLASS => "STRCLASS", + tag if is_element(tag) => kind_name(tag), + _ => return format!("record_{tag:04x}"), + }; + name.to_string() +} diff --git a/src/review/changes.rs b/src/review/changes.rs new file mode 100644 index 0000000..d4c2f9a --- /dev/null +++ b/src/review/changes.rs @@ -0,0 +1,127 @@ +use super::*; + +#[derive(Debug)] +pub struct Replacement { + pub removed: usize, + pub added: usize, + pub explanation: String, + /// Explanation matching is never object identity or geometric equivalence. + pub match_rule: &'static str, +} + +type Group = (u16, Option, Option, Option); + +fn group(object: &ObjectChange, bounds: bool) -> Group { + ( + object.kind, + object.layer, + object.target.clone(), + if bounds && object.target.is_none() { + object.bounds + } else { + None + }, + ) +} + +/// Pair only unique candidates in the remaining multisets. Ambiguous groups +/// remain additions/removals; no identity or geometric equivalence is inferred. +pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + let (mut used_old, mut used_new) = (BTreeSet::new(), BTreeSet::new()); + let mut result = Vec::new(); + for bounds in [true, false] { + let (mut a, mut b) = ( + BTreeMap::>::new(), + BTreeMap::>::new(), + ); + for (i, object) in old + .iter() + .enumerate() + .filter(|(i, _)| !used_old.contains(i)) + { + a.entry(group(object, bounds)).or_default().push(i); + } + for (i, object) in new + .iter() + .enumerate() + .filter(|(i, _)| !used_new.contains(i)) + { + b.entry(group(object, bounds)).or_default().push(i); + } + for (key, indices) in a { + if let (1, Some(others)) = (indices.len(), b.get(&key)) + && others.len() == 1 + { + let (i, j) = (indices[0], others[0]); + result.push(Replacement { + removed: i, + added: j, + explanation: explain(&old[i], &new[j]), + match_rule: if old[i].target.is_some() { + "unique_kind_target" + } else if bounds { + "unique_kind_layer_bounds" + } else { + "unique_kind_layer" + }, + }); + used_old.insert(i); + used_new.insert(j); + } + } + } + result.sort_by_key(|r| (r.removed, r.added)); + result +} + +fn explain(old: &ObjectChange, new: &ObjectChange) -> String { + let mut parts = vec![format!( + "{} modified{}", + kind_name(old.kind), + old.target + .as_ref() + .map(|t| format!(" target={t:?}")) + .unwrap_or_default() + )]; + if old.target.is_some() + && old.origin != new.origin + && let (Some(a), Some(b)) = (old.origin, new.origin) + { + parts.push(format!( + "origin {a:?} -> {b:?}, delta=({}, {}) dbu", + i64::from(b[0]) - i64::from(a[0]), + i64::from(b[1]) - i64::from(a[1]) + )); + } + if old.counts != new.counts { + let counts = |c: Option<[u16; 2]>| { + c.map(|[a, b]| format!("{a} x {b}")) + .unwrap_or_else(|| "absent".to_string()) + }; + parts.push(format!( + "columns/rows {} -> {}", + counts(old.counts), + counts(new.counts) + )); + } + if old.width != new.width { + let width = |n: Option| { + n.map(|n| n.to_string()) + .unwrap_or_else(|| "absent".to_string()) + }; + parts.push(format!( + "width {} -> {} dbu", + width(old.width), + width(new.width) + )); + } + if old.bounds == new.bounds { + parts.push("coordinate bounds unchanged".to_string()); + } else { + parts.push(format!( + "coordinate bounds {:?} -> {:?}", + old.bounds, new.bounds + )); + } + parts.join("; ") +} diff --git a/src/review/names.rs b/src/review/names.rs new file mode 100644 index 0000000..a3040d5 --- /dev/null +++ b/src/review/names.rs @@ -0,0 +1,120 @@ +use super::*; + +pub fn parse_layer(value: &str) -> Result { + let (a, b) = value + .split_once('/') + .ok_or("expected layer/type, for example 900/7")?; + Ok((decimal(a)?, decimal(b)?)) +} + +fn decimal(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err("layer numbers must be unsigned decimal integers".to_string()); + } + value + .parse() + .map_err(|_| "layer number exceeds 65535".to_string()) +} + +pub fn parse_layer_name(value: &str) -> Result<(Layer, String), String> { + let mut parts = value.splitn(3, ':'); + let a = parts.next().ok_or("expected layer:datatype:name")?; + let b = parts.next().ok_or("expected layer:datatype:name")?; + let name = parts + .next() + .filter(|s| !s.is_empty() && !s.contains('\0')) + .ok_or("layer name must be nonempty and contain no NUL")?; + Ok(((decimal(a)?, decimal(b)?), name.to_string())) +} + +fn record(output: &mut Vec, tag: u16, payload: &[u8]) -> Result<(), String> { + let padding = usize::from(tag & 0xff == 6 && !payload.len().is_multiple_of(2)); + let size = payload + .len() + .checked_add(padding + 4) + .and_then(|n| u16::try_from(n).ok()) + .ok_or("metadata label exceeds GDS record size")?; + output.extend_from_slice(&size.to_be_bytes()); + output.extend_from_slice(&tag.to_be_bytes()); + output.extend_from_slice(payload); + if padding != 0 { + output.push(0); + } + Ok(()) +} + +fn label(output: &mut Vec, carrier: Layer, text: &str) -> Result<(), String> { + record(output, RTAG_TEXT, &[])?; + record(output, RTAG_LAYER, &carrier.0.to_be_bytes())?; + record(output, RTAG_TEXTTYPE, &carrier.1.to_be_bytes())?; + record(output, RTAG_XY, &[0; 8])?; + record(output, RTAG_STRING, text.as_bytes())?; + record(output, RTAG_ENDEL, &[]) +} + +/// Consolidate recognized metadata into an explicitly chosen existing cell and +/// carrier. Other records, including timestamps and unknown records, are copied +/// byte-for-byte. Updates override existing names; the input is never modified. +pub fn write_layer_names( + input: &[u8], + cell_name: &str, + carrier: Layer, + updates: &BTreeMap, +) -> Result, String> { + let doc = Document::parse(input)?; + if doc + .diagnostics + .iter() + .any(|i| i.code == "unsupported_metadata_version") + { + return Err("cannot rewrite an unsupported layer-name metadata version".to_string()); + } + let cell = doc + .cells + .get(cell_name) + .ok_or_else(|| format!("cell {cell_name:?} does not exist"))?; + if cell + .objects + .iter() + .any(|o| o.layer == Some(carrier) && !o.metadata) + { + return Err( + "carrier pair already contains ordinary design objects in the selected cell" + .to_string(), + ); + } + let mut names = doc.layer_names.clone(); + for (&pair, name) in updates { + if name.is_empty() || name.contains('\0') { + return Err("layer names must be nonempty and contain no NUL".to_string()); + } + names.insert(pair, name.clone()); + } + let mut legend = Vec::new(); + label(&mut legend, carrier, LAYER_SENTINEL)?; + for ((layer, dtype), name) in names { + label(&mut legend, carrier, &format!("{layer}:{dtype}:{name}"))?; + } + let mut edits: Vec<_> = doc + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| o.metadata) + .map(|o| (o.offset, o.end_offset, None)) + .collect(); + edits.push((cell.end_offset, cell.end_offset, Some(legend.as_slice()))); + edits.sort_by_key(|(start, end, _)| (*start, *end)); + let mut result = Vec::with_capacity(input.len() + legend.len()); + let mut cursor = 0; + for (start, end, replacement) in edits { + result.extend_from_slice(&input[cursor..start]); + if let Some(bytes) = replacement { + result.extend_from_slice(bytes); + } + cursor = end; + } + result.extend_from_slice(&input[cursor..]); + // Validate the final declaration set before the CLI creates an output file. + Document::parse(&result)?; + Ok(result) +} diff --git a/src/review/report.rs b/src/review/report.rs new file mode 100644 index 0000000..7c0c445 --- /dev/null +++ b/src/review/report.rs @@ -0,0 +1,407 @@ +//! Versioned machine-readable diff reports. No consumer needs to parse prose. +use super::*; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; + +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, Default)] +pub struct ReportInputs { + pub old: Option, + pub new: Option, +} + +/// Limits affect evidence only. The default includes every changed object. +#[derive(Clone, Debug, Default)] +pub struct ReportOptions { + pub filter: DisplayFilter, + pub max_objects: Option, + pub hierarchy: bool, +} + +fn layer(pair: Layer) -> Value { + json!({"number": pair.0, "purpose": pair.1}) +} + +fn scope(value: &Scope) -> Value { + json!({"top": value.top, "cell": value.cell, "layer": value.layer.map(layer)}) +} + +fn input(path: Option<&Path>, doc: Option<&Document<'_>>) -> Value { + let units = doc + .filter(|d| d.present) + .and_then(|d| d.header.iter().find(|(tag, _)| *tag == RTAG_UNITS)) + .map(|(tag, data)| field(*tag, data)); + json!({ + "path": path.and_then(Path::to_str), + "path_encoded_hex": path.filter(|p| p.to_str().is_none()) + .map(|p| hex(p.as_os_str().as_encoded_bytes())), + "present": doc.map(|d| d.present), + "units": units + }) +} + +fn envelope(inputs: &ReportInputs, selection: &Scope, options: &ReportOptions) -> Value { + json!({ + "schema_version": SCHEMA_VERSION, + "kind": "diff", + "producer": {"name": "pheon", "version": env!("CARGO_PKG_VERSION")}, + "comparison_profile": PROFILE, + "inputs": {"old": input(inputs.old.as_deref(), None), "new": input(inputs.new.as_deref(), None)}, + "scope": scope(selection), + "coordinates": {"unit": "database_unit", "frame": "cell_local", "bounds": "coordinate_envelope"}, + "status": "failed", + "result": "unknown", + "exit_code": 2, + "summary": null, + "report_selection": { + "cell": options.filter.cell, + "layer": options.filter.layer.map(layer), + "max_objects": options.max_objects.map(|n| n.to_string()), + "truncated": false, + "filtered_cells": "0", "filtered_objects": "0", "limited_objects": "0", "emitted_objects": "0" + }, + "layer_legend": [], + "library_changes": [], "layer_name_changes": [], "cell_changes": [], + "hierarchy": null, + "diagnostics": [] + }) +} + +/// Input/selection failures still produce a valid report. Unknown locations are +/// null rather than inferred by parsing an error's human-readable message. +pub fn failure_report( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + code: &str, + message: &str, + side: Option<&str>, +) -> Value { + let mut report = envelope(inputs, selection, options); + report["diagnostics"] = json!([{ + "code": code, "severity": "error", "message": message, + "side": side, "byte_offset": null, "cell": null, "layer": null, "target": null + }]); + report +} + +fn field(tag: u16, data: &[u8]) -> Value { + let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { + ("xy", json!(points(data))) + } else if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE + ) && data.len().is_multiple_of(2) + { + ( + "uint16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ) + } else { + match tag & 0xff { + 0 if data.is_empty() => ("none", Value::Null), + 1 if data.len().is_multiple_of(2) => ( + "bits", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ), + 2 if data.len().is_multiple_of(2) => ( + "int16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + ), + 3 if data.len().is_multiple_of(4) => ( + "int32", + json!( + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + ), + 5 if data.len().is_multiple_of(8) => ( + "real8", + json!( + data.as_chunks::<8>() + .0 + .iter() + .map(|&v| decode_real8(u64::from_be_bytes(v))) + .collect::>() + ), + ), + 6 => ("ascii", json!(std::str::from_utf8(data).ok())), + _ => ("bytes", Value::Null), + } + }; + json!({"tag": tag, "name": tag_name(tag), "encoding": encoding, "value": value, "payload_hex": hex(data)}) +} + +fn fields(values: &[FieldValue]) -> Vec { + values.iter().map(|f| field(f.tag, &f.data)).collect() +} + +fn object(value: &ObjectChange, id: &str, side: &str) -> Value { + json!({ + "id": id, "kind": kind_name(value.kind), "tag": value.kind, + "source": {"side": side, "byte_offset": value.offset.to_string(), "end_offset": value.end_offset.to_string()}, + "layer": value.layer.map(layer), "bounds": value.bounds, "target": value.target, + "origin": value.origin.filter(|_| matches!(value.kind, RTAG_SREF | RTAG_AREF | RTAG_TEXT)), "width": value.width, "counts": value.counts, + "fields": fields(&value.fields), + "properties": value.properties.iter().map(|(attribute, bytes)| json!({ + "attribute": attribute, "value": std::str::from_utf8(bytes).ok(), "payload_hex": hex(bytes) + })).collect::>() + }) +} + +fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { + let a: BTreeMap<_, _> = old.fields.iter().map(|f| (f.tag, &f.data)).collect(); + let b: BTreeMap<_, _> = new.fields.iter().map(|f| (f.tag, &f.data)).collect(); + a.keys() + .chain(b.keys()) + .copied() + .collect::>() + .into_iter() + .filter(|tag| a.get(tag) != b.get(tag)) + .map(|tag| { + json!({ + "tag": tag, + "before": a.get(&tag).map(|data| field(tag, data)), + "after": b.get(&tag).map(|data| field(tag, data)) + }) + }) + .collect() +} + +impl Comparison { + /// Render evidence from the same documents used to construct this comparison. + /// IDs are local to this report, not persistent object identities. + pub fn json_report( + &self, + old: &Document<'_>, + new: &Document<'_>, + inputs: &ReportInputs, + options: &ReportOptions, + ) -> Value { + let mut report = envelope(inputs, &self.scope, options); + report["inputs"] = json!({"old": input(inputs.old.as_deref(), Some(old)), "new": input(inputs.new.as_deref(), Some(new))}); + let different = !self.cells.is_empty() + || !self.library_fields.is_empty() + || !self.layer_names.is_empty(); + report["status"] = json!(if self.issues.is_empty() { + "complete" + } else { + "incomplete" + }); + report["result"] = json!(if different { + "different" + } else if self.is_equal() { + "equal" + } else { + "unknown" + }); + report["exit_code"] = json!(self.exit_code()); + report["summary"] = json!({ + "counts": if self.issues.is_empty() { "exact" } else { "observed" }, + "changed_cells": self.cells.len().to_string(), + "removed_objects": self.cells.iter().map(|c| c.removed.len()).sum::().to_string(), + "added_objects": self.cells.iter().map(|c| c.added.len()).sum::().to_string(), + "layer_name_changes": self.layer_names.len().to_string(), + "library_field_changes": self.library_fields.len().to_string() + }); + report["layer_legend"] = json!( + self.layer_legend + .iter() + .map(|(pair, name)| json!({"layer": layer(*pair), "name": name})) + .collect::>() + ); + report["layer_name_changes"] = json!( + self.layer_names + .iter() + .map(|c| json!({ + "layer": layer(c.layer), "before": c.old, "after": c.new + })) + .collect::>() + ); + report["library_changes"] = json!(self.library_fields.iter().map(|c| { + if c.tag.is_none() { + json!({"field": "presence", "tag": null, "before": old.present, "after": new.present}) + } else { + json!({"field": c.field, "tag": c.tag, "before": fields(&c.old_fields), "after": fields(&c.new_fields)}) + } + }).collect::>()); + let (mut emitted, mut filtered_objects, mut limited, mut filtered_cells) = + (0_usize, 0_usize, 0_usize, 0_usize); + let mut cells = Vec::new(); + for (cell_index, cell) in self.cells.iter().enumerate() { + if options + .filter + .cell + .as_ref() + .is_some_and(|name| name != &cell.name) + { + filtered_cells += 1; + filtered_objects += cell.removed.len() + cell.added.len(); + continue; + } + let (mut removed, mut added) = (Vec::new(), Vec::new()); + let mut shown = BTreeSet::new(); + for (side, objects, output) in [ + ("old", &cell.removed, &mut removed), + ("new", &cell.added, &mut added), + ] { + for (index, change) in objects.iter().enumerate() { + if options + .filter + .layer + .is_some_and(|l| change.layer != Some(l)) + { + filtered_objects += 1; + } else if options.max_objects.is_some_and(|max| emitted >= max) { + limited += 1; + } else { + let id = format!("c{cell_index}:{side}:{index}"); + output.push(object(change, &id, side)); + shown.insert(id); + emitted += 1; + } + } + } + let replacements: Vec<_> = cell.replacements.iter().filter_map(|r| { + let a = format!("c{cell_index}:old:{}", r.removed); + let b = format!("c{cell_index}:new:{}", r.added); + (shown.contains(&a) && shown.contains(&b)).then(|| json!({ + "removed": a, "added": b, "match_rule": r.match_rule, + "explanation": r.explanation, + "field_changes": field_changes(&cell.removed[r.removed], &cell.added[r.added]), + "properties_changed": cell.removed[r.removed].properties != cell.added[r.added].properties + })) + }).collect(); + cells.push(json!({ + "name": cell.name, + "before": cell.old_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.old_attributes)})), + "after": cell.new_offset.map(|offset| json!({"byte_offset": offset.to_string(), "attributes": fields(&cell.new_attributes)})), + "attributes_changed": cell.attributes_changed, + "removed": removed, "added": added, "replacements": replacements + })); + } + report["cell_changes"] = json!(cells); + report["report_selection"]["truncated"] = + json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0); + for (key, count) in [ + ("filtered_cells", filtered_cells), + ("filtered_objects", filtered_objects), + ("limited_objects", limited), + ("emitted_objects", emitted), + ] { + report["report_selection"][key] = json!(count.to_string()); + } + let mut diagnostics: Vec<_> = [("old", old), ("new", new)] + .into_iter() + .flat_map(|(side, doc)| { + doc.diagnostics.iter().map(move |d| { + json!({ + "code": d.code, "severity": "error", "message": d.message, "side": side, + "byte_offset": d.offset.map(|n| n.to_string()), "cell": d.cell, + "layer": d.layer.map(layer), "target": d.target + }) + }) + }) + .collect(); + if old.scope != new.scope { + diagnostics.push(json!({"code": "scope_mismatch", "severity": "error", "message": "comparison scopes differ", "side": null, "byte_offset": null, "cell": null, "layer": null, "target": null})); + } + report["diagnostics"] = json!(diagnostics); + if options.hierarchy { + report["hierarchy"] = graph(old, new, self); + } + report + } +} + +fn graph(old: &Document<'_>, new: &Document<'_>, diff: &Comparison) -> Value { + let a = old.edges(); + let b = new.edges(); + let edges: BTreeSet<_> = a.keys().chain(b.keys()).cloned().collect(); + let names: BTreeSet<_> = old + .cells + .keys() + .chain(new.cells.keys()) + .cloned() + .chain( + edges + .iter() + .flat_map(|(parent, child)| [parent.clone(), child.clone()]), + ) + .collect(); + let direct: BTreeSet<_> = diff.cells.iter().map(|c| c.name.clone()).collect(); + let mut affected = direct.clone(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + let changed_refs: BTreeSet<_> = diff + .cells + .iter() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let a_targets: BTreeSet<_> = a.keys().map(|(_, t)| t.as_str()).collect(); + let b_targets: BTreeSet<_> = b.keys().map(|(_, t)| t.as_str()).collect(); + let state = |doc: &Document<'_>, targets: &BTreeSet<&str>, name: &String| { + if doc.cells.contains_key(name) { + "included" + } else if doc.known_cells.contains(name) { + "outside_scope" + } else if targets.contains(name.as_str()) { + "unresolved" + } else { + "absent" + } + }; + json!({ + "nodes": names.iter().map(|name| json!({ + "cell": name, "before": state(old, &a_targets, name), "after": state(new, &b_targets, name), + "change": if direct.contains(name) { "direct" } else if affected.contains(name) { "affected" } else { "unchanged" } + })).collect::>(), + "edges": edges.iter().map(|edge| json!({ + "parent": edge.0, "target": edge.1, + "before_count": a.get(edge).copied().unwrap_or(0).to_string(), + "after_count": b.get(edge).copied().unwrap_or(0).to_string(), + "reference_changed": changed_refs.contains(edge) + })).collect::>() + }) +} diff --git a/src/review/selection.rs b/src/review/selection.rs new file mode 100644 index 0000000..b87fbe6 --- /dev/null +++ b/src/review/selection.rs @@ -0,0 +1,233 @@ +use super::*; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Scope { + pub top: Option, + pub cell: Option, + pub layer: Option, +} + +/// Presentation filters never modify comparison status, totals, or hierarchy. +#[derive(Clone, Debug, Default)] +pub struct DisplayFilter { + pub cell: Option, + pub layer: Option, +} + +impl Scope { + pub fn description(&self) -> String { + let top = self + .top + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let cell = self + .cell + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let layer = self + .layer + .map(|(a, b)| format!("{a}/{b}")) + .unwrap_or_else(|| "all".to_string()); + format!("scope top={top} cell={cell} layer={layer}") + } + + pub fn validate_pair(&self, old: &Document<'_>, new: &Document<'_>) -> Result<(), String> { + for (kind, name) in [("top", &self.top), ("cell", &self.cell)] { + if let Some(name) = name + && !old.cells.contains_key(name) + && !new.cells.contains_key(name) + { + return Err(format!( + "selected {kind} {name:?} exists in neither revision" + )); + } + } + if let (Some(top), Some(cell)) = (&self.top, &self.cell) + && !old.reachable(top).contains(cell) + && !new.reachable(top).contains(cell) + { + return Err(format!( + "selected cell {cell:?} is not reachable from {top:?}" + )); + } + Ok(()) + } +} + +impl<'a> Document<'a> { + /// A missing Git revision has no cells or declarations. Retain the other + /// revision's library header to avoid inventing a unit/name change. + pub fn empty_like(&self) -> Self { + Self { + records: Vec::new(), + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + issues: Vec::new(), + diagnostics: Vec::new(), + header: self.header.clone(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: self.scope.clone(), + present: false, + } + } + fn reachable(&self, root: &str) -> BTreeSet { + let mut found = BTreeSet::new(); + let mut pending = vec![root.to_string()]; + while let Some(name) = pending.pop() { + if let Some(cell) = self.cells.get(&name) + && found.insert(name) + { + pending.extend(cell.objects.iter().filter_map(|o| o.target.clone())); + } + } + found + } + + pub(super) fn refresh_issues(&mut self) { + self.diagnostics = self.global_issues.clone(); + for (name, cell) in &self.cells { + for issue in &cell.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + for object in &cell.objects { + for issue in &object.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.layer = object.layer; + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + } + } + self.check_graph(); + self.issues = self.diagnostics.iter().map(|d| d.message.clone()).collect(); + } + + /// Project a parsed document. Missing selectors are allowed only for one + /// side of a comparison, after Scope::validate_pair has checked both sides. + pub fn selected(mut self, scope: &Scope, allow_missing: bool) -> Result { + if self.scope != Scope::default() { + return Err("document is already scoped; combine selectors in one Scope".to_string()); + } + if !allow_missing { + scope.validate_pair(&self, &self)?; + } + let mut wanted = scope + .top + .as_ref() + .map(|t| self.reachable(t)) + .unwrap_or_else(|| self.cells.keys().cloned().collect()); + if let Some(cell) = &scope.cell { + wanted.retain(|n| n == cell); + } + // Determine layer coverage before projecting cells: exact-cell scopes + // still need to know whether a referenced definition uses this layer. + let relevant = scope.layer.map(|layer| { + let mut relevant: BTreeSet<_> = self + .cells + .iter() + .filter(|(_, c)| { + !c.issues.is_empty() + || c.objects.iter().any(|o| { + !o.metadata + && (o.layer == Some(layer) + || (!o.issues.is_empty() && o.layer.is_none())) + }) + }) + .map(|(n, _)| n.clone()) + .collect(); + let mut parents = BTreeMap::>::new(); + for (parent, cell) in &self.cells { + for o in &cell.objects { + if let Some(target) = &o.target { + parents + .entry(target.clone()) + .or_default() + .push(parent.clone()); + if !self.known_cells.contains(target) { + relevant.insert(parent.clone()); + } + } + } + } + let mut pending: Vec<_> = relevant.iter().cloned().collect(); + // A cycle cannot be proven to have no effect on a layer. Preserve + // cycles (and their descendants) so graph validation remains visible. + let mut indegree: BTreeMap<_, usize> = + self.cells.keys().map(|n| (n.clone(), 0)).collect(); + for cell in self.cells.values() { + for target in cell.objects.iter().filter_map(|o| o.target.as_ref()) { + if let Some(n) = indegree.get_mut(target) { + *n += 1; + } + } + } + let mut ready: Vec<_> = indegree + .iter() + .filter(|(_, n)| **n == 0) + .map(|(n, _)| n.clone()) + .collect(); + while let Some(name) = ready.pop() { + for target in self.cells[&name] + .objects + .iter() + .filter_map(|o| o.target.as_ref()) + { + if let Some(n) = indegree.get_mut(target) { + *n -= 1; + if *n == 0 { + ready.push(target.clone()); + } + } + } + } + for (name, _) in indegree.iter().filter(|(_, n)| **n != 0) { + if relevant.insert(name.clone()) { + pending.push(name.clone()); + } + } + while let Some(child) = pending.pop() { + for parent in parents.get(&child).into_iter().flatten() { + if relevant.insert(parent.clone()) { + pending.push(parent.clone()); + } + } + } + relevant + }); + self.cells.retain(|name, _| { + wanted.contains(name) && relevant.as_ref().is_none_or(|r| r.contains(name)) + }); + if let (Some(layer), Some(relevant)) = (scope.layer, relevant) { + for cell in self.cells.values_mut() { + cell.objects.retain(|o| { + o.metadata + || o.layer == Some(layer) + || o.target + .as_ref() + .is_some_and(|t| relevant.contains(t) || !self.known_cells.contains(t)) + || (!o.issues.is_empty() && o.layer.is_none()) + }); + } + self.layer_names.retain(|pair, _| *pair == layer); + } else if scope.top.is_some() || scope.cell.is_some() { + let layers: BTreeSet<_> = self + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .filter_map(|o| o.layer) + .collect(); + self.layer_names.retain(|pair, _| layers.contains(pair)); + } + self.scope = scope.clone(); + self.refresh_issues(); + Ok(self) + } +} diff --git a/test.py b/test.py new file mode 100644 index 0000000..448e8e8 --- /dev/null +++ b/test.py @@ -0,0 +1,8 @@ +"""Read a GDSII file through Pheon's supported Python API.""" +import sys + +from pheon import arrow + + +if __name__ == "__main__": + print(arrow.read_path(sys.argv[1])) diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..993caa8 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,20 @@ +use pheon::{basic::encode_real8, records}; + +pub fn record(out: &mut Vec, tag: u16, data: &[u8]) { + out.extend_from_slice(&u16::try_from(data.len() + 4).unwrap().to_be_bytes()); + out.extend_from_slice(&tag.to_be_bytes()); + out.extend_from_slice(data); +} + +pub fn library_header() -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_BGNLIB, &[0; 24]); + record(&mut bytes, records::RTAG_LIBNAME, b"test"); + let units: Vec = [0.001, 1e-9] + .into_iter() + .flat_map(|v| encode_real8(v).unwrap().to_be_bytes()) + .collect(); + record(&mut bytes, records::RTAG_UNITS, &units); + bytes +} diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs new file mode 100644 index 0000000..0e74a5e --- /dev/null +++ b/tests/ffi_errors.rs @@ -0,0 +1,157 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{last_error_message, read_path, read_cells_bytes, scan_bytes, records}; +use std::{ffi::{CStr, CString}, fs, path::PathBuf, ptr}; + +struct TestFile(PathBuf); + +impl TestFile { + fn new(name: &str, contents: &[u8]) -> Self { + let path = std::env::temp_dir().join(format!("pheon-{name}-{}.gds", std::process::id())); + fs::write(&path, contents).unwrap(); + Self(path) + } + + fn cpath(&self) -> CString { + CString::new(self.0.to_str().unwrap()).unwrap() + } +} + +impl Drop for TestFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +fn error_message() -> String { + let message = last_error_message(); + assert!(!message.is_null()); + unsafe { CStr::from_ptr(message) }.to_str().unwrap().to_owned() +} + +#[test] +fn path_errors_preserve_outputs_and_convert_parser_panics() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("path pointer was null")); + assert!(array.is_released()); + + // A mismatched timestamp size currently makes Record::read panic. + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); + let file = TestFile::new("invalid-timestamps", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("Expected record size")); + assert!(array.is_released()); + + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("valid-empty", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(!array.is_released()); + assert!(last_error_message().is_null()); +} + +#[test] +fn other_threads_do_not_clear_or_free_error_messages() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + let pointer = last_error_message(); + let message = error_message(); + + std::thread::spawn(|| { + assert!(last_error_message().is_null()); + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("thread-empty", &bytes); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(last_error_message().is_null()); + }).join().unwrap(); + + assert_eq!(last_error_message(), pointer); + assert_eq!(unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(), message); +} + +#[test] +fn null_arrow_outputs_return_an_error() { + assert_eq!(unsafe { read_path(ptr::null(), ptr::null_mut(), ptr::null_mut()) }, -1); + assert!(error_message().contains("Arrow output pointer was null")); +} + +#[test] +fn large_malformed_inputs_report_offsets_without_dumping_contents() { + let mut bytes = vec![0; 1024 * 1024 + 4]; + bytes[..4].copy_from_slice(&[0, 4, 0xff, 0xff]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + let file = TestFile::new("large-malformed", &bytes); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let scan_message = error_message(); + assert_eq!( + unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, + -1 + ); + let read_message = error_message(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + -1 + ); + let selected_message = error_message(); + for message in [scan_message, read_message, selected_message] { + assert!(message.contains("at byte 4"), "{message}"); + assert!(message.contains("Unexpected record"), "{message}"); + assert!(message.len() < 256); + } + assert!(array.is_released()); +} + +#[test] +fn selected_cell_errors_use_file_offsets_for_bounded_ranges() { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let cell_end = bytes.len(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let range_end = bytes.len() as u64; + bytes.resize(bytes.len() + 1024, 0); + let ranges = [start, range_end]; + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!(error_message().contains(&format!( + "at byte {cell_end}: Cell byte range extends past ENDSTR" + ))); +} diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json new file mode 100644 index 0000000..75c2e4b --- /dev/null +++ b/tests/golden/report-equal.json @@ -0,0 +1,78 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 0, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "equal", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-failed.json b/tests/golden/report-failed.json new file mode 100644 index 0000000..c9b12d9 --- /dev/null +++ b/tests/golden/report-failed.json @@ -0,0 +1,64 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": null, + "cell": null, + "code": "invalid_input", + "layer": null, + "message": "truncated record", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": null, + "units": null + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": null, + "units": null + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "failed", + "summary": null +} diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json new file mode 100644 index 0000000..763de12 --- /dev/null +++ b/tests/golden/report-geometry.json @@ -0,0 +1,360 @@ +{ + "cell_changes": [ + { + "added": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:new:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "438", + "side": "new" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "after": { + "attributes": [], + "byte_offset": "346" + }, + "attributes_changed": false, + "before": { + "attributes": [], + "byte_offset": "346" + }, + "name": "leaf", + "removed": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:old:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "446", + "side": "old" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "replacements": [ + { + "added": "c0:new:0", + "explanation": "BOUNDARY modified; coordinate bounds unchanged", + "field_changes": [ + { + "after": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "before": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "tag": 4099 + } + ], + "match_rule": "unique_kind_layer_bounds", + "properties_changed": false, + "removed": "c0:old:0" + } + ] + } + ], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": { + "edges": [ + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "left" + }, + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "right" + }, + { + "after_count": "2", + "before_count": "2", + "parent": "left", + "reference_changed": false, + "target": "leaf" + }, + { + "after_count": "3", + "before_count": "3", + "parent": "right", + "reference_changed": false, + "target": "leaf" + } + ], + "nodes": [ + { + "after": "included", + "before": "included", + "cell": "CHIP", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "leaf", + "change": "direct" + }, + { + "after": "included", + "before": "included", + "cell": "left", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "right", + "change": "affected" + } + ] + }, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "2", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "1", + "changed_cells": "1", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "1" + } +} diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json new file mode 100644 index 0000000..4319c1f --- /dev/null +++ b/tests/golden/report-incomplete.json @@ -0,0 +1,105 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "old", + "target": null + }, + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "incomplete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "observed", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json new file mode 100644 index 0000000..3e1245a --- /dev/null +++ b/tests/golden/report-metadata.json @@ -0,0 +1,87 @@ +{ + "cell_changes": [], + "comparison_profile": "klamath-object-v2", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [ + { + "after": "CORE:WG", + "before": "WG", + "layer": { + "number": 1, + "purpose": 0 + } + } + ], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "1", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt new file mode 100644 index 0000000..8811a25 --- /dev/null +++ b/tests/golden/review-canonical.txt @@ -0,0 +1,9 @@ +profile klamath-object-v2 +scope top=all cell=all layer=all +coordinates: integer database units; bounds: coordinate envelopes +HEADER=[600] +LIBNAME="test" +UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54 +layer 1/0 name="WG" +cell "CHIP" + BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt new file mode 100644 index 0000000..57e6c09 --- /dev/null +++ b/tests/golden/review-diff.txt @@ -0,0 +1,7 @@ +profile klamath-object-v2 +scope top=all cell=all layer=all +cell "CHIP" old@Some(62) new@Some(62) + ~ BOUNDARY modified; coordinate bounds unchanged + - @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] + + @98 bbox=Some([0, 0, 10, 10]) BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 0, 0, 0] +1 changed cells; 2 of 2 object changes shown; 0 layer-name changes diff --git a/tests/golden/review-hierarchy.mmd b/tests/golden/review-hierarchy.mmd new file mode 100644 index 0000000..dcc963b --- /dev/null +++ b/tests/golden/review-hierarchy.mmd @@ -0,0 +1,10 @@ +flowchart TD + %% scope top=all cell=all layer=all + n0["CHIP [affected]"] + n1["leaf [cell changed]"] + n2["left [affected]"] + n3["right [affected]"] + n0 -->|"x1"| n2 + n0 -->|"x1"| n3 + n2 -->|"x2"| n1 + n3 -->|"x3"| n1 diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt new file mode 100644 index 0000000..fa40d79 --- /dev/null +++ b/tests/golden/review-hierarchy.txt @@ -0,0 +1,6 @@ +scope top=all cell=all layer=all +"CHIP" x1 [affected] + "left" x1 [affected] + "leaf" x2 [cell changed] + "right" x1 [affected] + "leaf" x3 [cell changed] [shown above] diff --git a/tests/numeric_packing.rs b/tests/numeric_packing.rs new file mode 100644 index 0000000..14a6109 --- /dev/null +++ b/tests/numeric_packing.rs @@ -0,0 +1,103 @@ +use pheon::*; +use std::{mem::size_of, slice}; + +// Read the packed prefix of each source slot, as the Python wrapper does. +fn packed_bytes(values: &[T], width: usize) -> Vec { + let bytes = unsafe { slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) }; + bytes + .chunks_exact(size_of::()) + .flat_map(|slot| slot[..width].iter().copied()) + .collect() +} + +macro_rules! check_int2 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 256 as $ty, 32767 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!(packed_bytes(&values, 2), [0, 0, 0, 1, 1, 0, 0x7f, 0xff]); + }}; +} + +macro_rules! check_int4 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 65536 as $ty, 2147483520_i64 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!( + packed_bytes(&values, 4), + [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0x7f, 0xff, 0xff, 0x80] + ); + }}; +} + +#[test] +fn all_source_types_pack_to_big_endian_integer_bytes() { + check_int2!(u16, u16_to_i16); + check_int2!(u32, u32_to_i16); + check_int2!(u64, u64_to_i16); + check_int2!(i16, i16_to_i16); + check_int2!(i32, i32_to_i16); + check_int2!(i64, i64_to_i16); + check_int2!(f32, f32_to_i16); + check_int2!(f64, f64_to_i16); + check_int4!(u32, u32_to_i32); + check_int4!(u64, u64_to_i32); + check_int4!(i32, i32_to_i32); + check_int4!(i64, i64_to_i32); + check_int4!(f32, f32_to_i32); + check_int4!(f64, f64_to_i32); +} + +#[test] +fn signed_limits_and_unsigned_overflows_are_checked() { + let mut signed = [i64::from(i16::MIN), -1]; + assert_eq!(unsafe { i64_to_i16(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!(packed_bytes(&signed, 2), [0x80, 0, 0xff, 0xff]); + let mut signed = [i64::from(i32::MIN), i64::from(i32::MAX)]; + assert_eq!(unsafe { i64_to_i32(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!( + packed_bytes(&signed, 4), + [0x80, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff] + ); + + for value in [32768_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i16(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } + for value in [2147483648_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i32(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } +} + +#[test] +fn nonfinite_floats_and_rounded_up_f32_limit_are_rejected() { + for convert in [f32_to_i16, f32_to_i32] { + for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let mut input = [value]; + let result = unsafe { convert(input.as_mut_ptr(), 1) }; + assert_ne!(result, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + for convert in [f64_to_i16, f64_to_i32] { + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut input = [value]; + assert_ne!(unsafe { convert(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + let mut input = [2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 2147483648_f32); + assert_eq!(input, [2147483648_f32]); + let mut input = [-2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(packed_bytes(&input, 4), [0x80, 0, 0, 0]); +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py new file mode 100644 index 0000000..30367b4 --- /dev/null +++ b/tests/python/test_arrow.py @@ -0,0 +1,105 @@ +"""Public Arrow readers, lifetime management and input validation.""" + +import gc +import gzip +import mmap +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy + +from pheon import PheonError, arrow +from pheon._native import ffi +from test_ffi import layout + + +class ArrowTests(unittest.TestCase): + def test_full_scan_and_selected_reads_own_their_data(self): + data = bytearray(layout()) + full = arrow.read_bytes(data) + scan = arrow.scan_bytes(data) + cells = scan.field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + ) + ) + selected = arrow.read_cells_bytes(data, ranges) + data[:] = b"\0" * len(data) + del data + gc.collect() + self.assertEqual(full.to_pylist(), selected.to_pylist()) + self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"]) + self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"]) + + def test_paths_gzip_magic_and_closed_mmap(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "layout with spaces.gds" + path.write_bytes(layout()) + expected = arrow.read_path(path).to_pylist() + with path.open("rb") as stream: + with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer: + result = arrow.read_bytes(buffer) + self.assertEqual(result.to_pylist(), expected) + path.write_bytes(gzip.compress(layout())) + self.assertEqual(arrow.read_path(path).to_pylist(), expected) + + def test_empty_and_noncontiguous_cell_selections(self): + data = layout() + empty = arrow.read_cells_bytes(data, []) + self.assertEqual(empty.field("cells")[0].as_py(), []) + values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2] + self.assertFalse(values.flags.c_contiguous) + self.assertEqual( + arrow.read_cells_bytes(data, values).to_pylist(), + arrow.read_bytes(data).to_pylist(), + ) + + def test_bad_inputs_fail_before_importing_arrow(self): + with self.assertRaises(PheonError): + arrow.read_bytes(b"invalid") + with self.assertRaises(PheonError): + arrow.scan_bytes(b"") + with self.assertRaises(PheonError): + arrow.read_cells_bytes(layout(), [[0, 10]]) + with self.assertRaises(ValueError): + arrow.read_bytes(memoryview(layout())[::2]) + for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]: + with self.assertRaises(ValueError): + arrow.read_cells_bytes(layout(), ranges) + with self.assertRaises(TypeError): + arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + + def test_import_failure_releases_both_native_outputs(self): + releases = [] + + @ffi.callback("void(struct ArrowArray *)") + def release_array(value): + releases.append("array") + value.release = ffi.NULL + + @ffi.callback("void(struct ArrowSchema *)") + def release_schema(value): + releases.append("schema") + value.release = ffi.NULL + + def export(array, schema): + array.release = release_array + schema.release = release_schema + return 0 + + def fail_import(*args): + raise ValueError("import failed") + + with patch.object( + arrow, + "pyarrow", + SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)), + ): + with self.assertRaisesRegex(ValueError, "import failed"): + arrow._export(export) + self.assertEqual(releases, ["array", "schema"]) diff --git a/tests/python/test_ffi.py b/tests/python/test_ffi.py new file mode 100644 index 0000000..4dba750 --- /dev/null +++ b/tests/python/test_ffi.py @@ -0,0 +1,70 @@ +"""Exercise the explicit C ABI declarations against the actual shared library.""" +import struct +import tempfile +from pathlib import Path +import unittest + +from pheon._native import ffi, lib + + +def record(tag, data=b""): + return struct.pack(">HH", len(data) + 4, tag) + data + + +def layout(): + return b"".join([ + record(0x0002, struct.pack(">h", 600)), + record(0x0102, b"\0" * 24), + record(0x0206, b"test"), + record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")), + record(0x0502, b"\0" * 24), + record(0x0606, b"cell"), + record(0x0700), + record(0x0400), + ]) + + +class FfiTests(unittest.TestCase): + def check_export(self, call): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + self.assertEqual(call(array, schema), 0) + self.assertEqual(lib.last_error_message(), ffi.NULL) + self.assertEqual(array.length, 1) + self.assertEqual(ffi.string(schema.format), b"+s") + self.assertNotEqual(array.release, ffi.NULL) + self.assertNotEqual(schema.release, ffi.NULL) + self.assertGreaterEqual(array.n_children, 7) + self.assertEqual(array.n_children, schema.n_children) + self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit") + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + def test_scan_selected_read_and_path_exports(self): + data = layout() + self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s)) + self.check_export(lambda a, s: lib.read_bytes(data, len(data), a, s)) + # The library header is 62 bytes; the one cell ends just before ENDLIB. + ranges = ffi.new("uint64_t[]", [62, len(data) - 4]) + self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "file with spaces.gds" + path.write_bytes(data) + self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s)) + + def test_failures_leave_outputs_empty_and_expose_error_message(self): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1) + self.assertNotEqual(lib.last_error_message(), ffi.NULL) + self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message())) + self.assertEqual(array.release, ffi.NULL) + self.assertEqual(schema.release, ffi.NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_loader.py b/tests/python/test_loader.py new file mode 100644 index 0000000..4724495 --- /dev/null +++ b/tests/python/test_loader.py @@ -0,0 +1,36 @@ +"""Reject incompatible native libraries before exposing callable functions.""" + +from pathlib import Path +import runpy +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import pheon._native + + +class LoaderTests(unittest.TestCase): + def load(self, library): + with patch("cffi.FFI.dlopen", return_value=library): + return runpy.run_path(str(Path(pheon._native.__file__))) + + def test_matching_versions_are_accepted(self): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: 1, pheon_arrow_schema_version=lambda: 1 + ) + ) + + def test_incompatible_abi_or_schema_is_rejected(self): + for abi, schema in [(2, 1), (1, 2)]: + with self.assertRaisesRegex(ImportError, "Incompatible Pheon library"): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: abi, + pheon_arrow_schema_version=lambda: schema, + ) + ) + + def test_unversioned_library_is_rejected(self): + with self.assertRaisesRegex(ImportError, "version API"): + self.load(SimpleNamespace()) diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py new file mode 100644 index 0000000..44eb84d --- /dev/null +++ b/tests/python/test_packing.py @@ -0,0 +1,66 @@ +import unittest + +import numpy as np + +from pheon import pack_int2, pack_int4 + + +class PackingTests(unittest.TestCase): + def test_scalars_lists_and_dimensions(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]: + for shape in [(), (1,), (2, 2), (2, 2, 2)]: + count = int(np.prod(shape)) if shape else 1 + data = np.arange(count, dtype=dtype).reshape(shape) + before = data.copy() + with self.subTest(width=width, dtype=dtype, shape=shape): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + np.testing.assert_array_equal(data, before) + for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]: + with self.subTest(width=width, data=data): + self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes()) + + def test_noncontiguous_readonly_and_endian_inputs(self): + original = np.arange(48, dtype=np.int64).reshape(6, 8) + readonly = original.copy() + readonly.setflags(write=False) + for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly, + original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]: + before = data.copy() + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + with self.subTest(width=width, strides=data.strides, dtype=data.dtype): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C")) + np.testing.assert_array_equal(data, before) + + def test_signed_limits_and_native_integer_widths(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1 + data = np.array([[low, -1], [0, high]], dtype=np.int64) + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + for dtype in [np.int8, np.uint8, np.int16, np.uint16]: + data = np.array([[1, 2], [3, 4]], dtype=dtype) + self.assertEqual(pack_int2(data), data.astype(">i2").tobytes()) + self.assertEqual(pack_int4(data), data.astype(">i4").tobytes()) + + def test_failed_conversions_preserve_caller_data(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + limit = 1 << (8 * width - 1) + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=np.int64) + before = data.copy() + with self.subTest(width=width, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + for dtype in [np.float32, np.float64]: + for invalid in [np.nan, np.inf, -np.inf]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, invalid=invalid, dtype=dtype): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/real8.rs b/tests/real8.rs new file mode 100644 index 0000000..fc632a1 --- /dev/null +++ b/tests/real8.rs @@ -0,0 +1,75 @@ +use pheon::basic::{decode_real8, encode_real8, pack_real8}; + +#[test] +fn normalized_real8_has_known_encodings_and_exact_values() { + for (value, expected) in [ + (0.0, 0), + (-0.0, 0), + (1.0, 0x4110_0000_0000_0000), + (-2.0, 0xc120_0000_0000_0000), + (0.5, 0x4080_0000_0000_0000), + (0.001, 0x3e41_8937_4bc6_a7f0), + (1e-9, 0x3944_b82f_a09b_5a54), + (2_f64.powi(-260), 0x0010_0000_0000_0000), + ] { + let bits = encode_real8(value).unwrap(); + assert_eq!(bits, expected, "{value:e}"); + assert_eq!(decode_real8(bits), value); + } +} + +#[test] +fn real8_round_trips_each_supported_binary_exponent_and_mantissa_boundary() { + for exponent in -260..=251_i32 { + for mantissa in [ + 0, + 1, + 0x0008_0000_0000_0000, + 0x000f_ffff_ffff_fffe, + 0x000f_ffff_ffff_ffff, + ] { + for sign in [0, 1_u64 << 63] { + let ieee = sign | (((exponent + 1023) as u64) << 52) | mantissa; + let value = f64::from_bits(ieee); + let encoded = encode_real8(value).unwrap(); + assert_ne!(encoded & 0x00f0_0000_0000_0000, 0, "{value:e}"); + assert_eq!(decode_real8(encoded).to_bits(), ieee, "{value:e}"); + } + } + } +} + +#[test] +fn real8_underflow_is_canonical_and_overflow_is_an_error() { + let minimum = 2_f64.powi(-260); + let maximum_exclusive = 2_f64.powi(252); + for magnitude in [ + 0.0, + f64::from_bits(1), + f64::MIN_POSITIVE, + 1e-90, + 1e-80, + f64::from_bits(minimum.to_bits() - 1), + ] { + for value in [magnitude, -magnitude] { + assert_eq!(encode_real8(value).unwrap(), 0, "{value:e}"); + } + } + let largest = f64::from_bits(maximum_exclusive.to_bits() - 1); + assert_eq!(encode_real8(largest).unwrap(), 0x7fff_ffff_ffff_fff8); + assert_eq!(decode_real8(encode_real8(largest).unwrap()), largest); + for value in [ + maximum_exclusive, + -maximum_exclusive, + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] { + assert!(encode_real8(value).is_err(), "{value}"); + let mut buffer = [0x55; 8]; + assert!(pack_real8(&mut buffer, value).is_err()); + assert_eq!(buffer, [0x55; 8]); + } +} diff --git a/tests/review.rs b/tests/review.rs new file mode 100644 index 0000000..71b895b --- /dev/null +++ b/tests/review.rs @@ -0,0 +1,1248 @@ +mod common; +#[path = "review/reports.rs"] +mod reports; + +use common::{library_header, record}; +use pheon::{ + records::*, + review::{Comparison, Document, LAYER_SENTINEL, hierarchy}, +}; +use std::{fs, path::PathBuf, process::Command}; + +fn ascii(bytes: &mut Vec, tag: u16, value: &str) { + let mut value = value.as_bytes().to_vec(); + if value.len() % 2 == 1 { + value.push(0); + } + record(bytes, tag, &value); +} + +fn xy(bytes: &mut Vec, points: &[[i32; 2]]) { + record( + bytes, + RTAG_XY, + &points + .iter() + .flatten() + .flat_map(|v| v.to_be_bytes()) + .collect::>(), + ); +} + +fn layout(cells: &[(&str, Vec>)]) -> Vec { + let mut bytes = library_header(); + for (name, objects) in cells { + record(&mut bytes, RTAG_BGNSTR, &[0; 24]); + ascii(&mut bytes, RTAG_STRNAME, name); + for object in objects { + bytes.extend_from_slice(object); + } + record(&mut bytes, RTAG_ENDSTR, &[]); + } + record(&mut bytes, RTAG_ENDLIB, &[]); + bytes +} + +fn polygon(points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_BOUNDARY, &[]); + record(&mut bytes, RTAG_LAYER, &1_u16.to_be_bytes()); + record(&mut bytes, RTAG_DATATYPE, &0_u16.to_be_bytes()); + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn square() -> Vec { + polygon(&[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]) +} + +fn triangle() -> Vec { + polygon(&[[0, 0], [10, 0], [0, 10], [0, 0]]) +} + +fn reference(target: &str, x: i32, count: i16) -> Vec { + let mut bytes = Vec::new(); + record( + &mut bytes, + if count == 1 { RTAG_SREF } else { RTAG_AREF }, + &[], + ); + ascii(&mut bytes, RTAG_SNAME, target); + if count == 1 { + xy(&mut bytes, &[[x, 0]]); + } else { + let counts: Vec<_> = [count, 1].into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, RTAG_COLROW, &counts); + xy(&mut bytes, &[[x, 0], [x + 100, 0], [x, 0]]); + } + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn label(carrier: (u16, u16), text: &str) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_TEXT, &[]); + record(&mut bytes, RTAG_LAYER, &carrier.0.to_be_bytes()); + record(&mut bytes, RTAG_TEXTTYPE, &carrier.1.to_be_bytes()); + xy(&mut bytes, &[[0, 0]]); + ascii(&mut bytes, RTAG_STRING, text); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn with_property(mut object: Vec, value: &str) -> Vec { + object.truncate(object.len() - 4); + record(&mut object, RTAG_PROPATTR, &17_u16.to_be_bytes()); + ascii(&mut object, RTAG_PROPVALUE, value); + record(&mut object, RTAG_ENDEL, &[]); + object +} + +fn shared(geometry: Vec) -> Vec { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 20, 1)], + ), + ("left", vec![reference("leaf", 0, 2)]), + ("right", vec![reference("leaf", 0, 3)]), + ("leaf", vec![geometry]), + ]) +} + +struct Pair { + name: &'static str, + old: Vec, + new: Vec, + changed_cells: usize, + removed: usize, + added: usize, + metadata: usize, +} + +// Ten deliberately small fixture pairs specify the first object profile. +fn corpus() -> Vec { + let base = layout(&[("CHIP", vec![square(), label((2, 0), "port")])]); + let mut timestamps = base.clone(); + timestamps[10] = 1; // BGNLIB payload, ignored by the object profile. + let meta = |name| { + layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), name), + ], + )]) + }; + vec![ + Pair { + name: "record reordering", + old: base.clone(), + new: layout(&[("CHIP", vec![label((2, 0), "port"), square()])]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "timestamp edit", + old: base, + new: timestamps, + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "polygon start and winding", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[( + "CHIP", + vec![polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])], + )]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "duplicate shape", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![square(), square()])]), + changed_cells: 1, + removed: 0, + added: 1, + metadata: 0, + }, + Pair { + name: "same bounds different geometry", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![triangle()])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "moved reference", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 5, 1)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "array count", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 2)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 0, 3)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "element property", + old: layout(&[("CHIP", vec![with_property(square(), "old")])]), + new: layout(&[("CHIP", vec![with_property(square(), "new")])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "layer name only", + old: meta("1:0:WG"), + new: meta("1:0:CORE:WG"), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 1, + }, + Pair { + name: "shared changed descendant", + old: shared(square()), + new: shared(triangle()), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + ] +} + +#[test] +fn ten_fixture_pairs_have_explicit_expected_changes() { + for pair in corpus() { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert!(diff.issues.is_empty(), "{}: {:?}", pair.name, diff.issues); + assert_eq!(diff.cells.len(), pair.changed_cells, "{}", pair.name); + assert_eq!( + diff.cells.iter().map(|c| c.removed.len()).sum::(), + pair.removed, + "{}", + pair.name + ); + assert_eq!( + diff.cells.iter().map(|c| c.added.len()).sum::(), + pair.added, + "{}", + pair.name + ); + assert_eq!(diff.layer_names.len(), pair.metadata, "{}", pair.name); + let equal = pair.changed_cells + pair.metadata == 0; + assert_eq!(diff.is_equal(), equal, "{}", pair.name); + assert_eq!( + old.canonical_dump() == new.canonical_dump(), + equal, + "{}", + pair.name + ); + assert_eq!(diff.exit_code(), if equal { 0 } else { 1 }); + } +} + +#[test] +fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { + let bytes = layout(&[("CHIP", vec![square()])]); + let doc = Document::parse(&bytes).unwrap(); + let dump = doc.canonical_dump(); + assert!( + dump.ends_with( + "cell \"CHIP\"\n BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0]\n" + ), + "{dump}" + ); + assert!(!dump.contains("BGNSTR")); + assert!( + doc.record_dump() + .contains("0000000000 0002 HEADER size=6 [600]") + ); + assert!(doc.record_dump().contains("BGNSTR")); +} + +#[test] +fn canonical_dump_and_hierarchy_match_versioned_goldens() { + let bytes = layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + )]); + assert_eq!( + Document::parse(&bytes).unwrap().canonical_dump(), + include_str!("golden/review-canonical.txt") + ); + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + hierarchy(&a, &b, Some(&diff), false, 8), + include_str!("golden/review-hierarchy.txt") + ); + assert_eq!( + hierarchy(&a, &b, Some(&diff), true, 8), + include_str!("golden/review-hierarchy.mmd") + ); +} + +#[test] +fn misplaced_library_fields_and_nul_names_are_rejected() { + let mut late = layout(&[("CHIP", vec![])]); + late.truncate(late.len() - 4); + ascii(&mut late, RTAG_LIBNAME, "late"); + record(&mut late, RTAG_ENDLIB, &[]); + assert!( + Document::parse(&late) + .unwrap_err() + .contains("library field outside") + ); + assert!( + Document::parse(&layout(&[("CH\0IP", vec![])])) + .unwrap_err() + .contains("NUL") + ); +} + +#[test] +fn cell_and_property_order_do_not_affect_comparison() { + let a = with_property(with_property(square(), "a"), "b"); + let b = with_property(with_property(square(), "b"), "a"); + let a = layout(&[("z", vec![a]), ("a", vec![])]); + let b = layout(&[("a", vec![]), ("z", vec![b])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + assert!(Comparison::between(&a, &b).is_equal()); + assert_eq!(a.canonical_dump(), b.canonical_dump()); +} + +#[test] +fn metadata_requires_a_sentinel_in_the_same_cell_and_carrier_pair() { + let bytes = layout(&[ + ( + "CHIP", + vec![ + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "2:0:ordinary text"), + ], + ), + ("OTHER", vec![label((900, 7), "3:0:ordinary text")]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!(doc.layer_names.len(), 1); + assert_eq!(doc.layer_names.get(&(1, 0)).unwrap(), "WG"); + assert!(doc.canonical_dump().contains("2:0:ordinary text")); + assert!(doc.canonical_dump().contains("3:0:ordinary text")); + assert!(!doc.canonical_dump().contains(LAYER_SENTINEL)); + let conflicts = layout(&[( + "CHIP", + vec![ + label((9, 0), LAYER_SENTINEL), + label((9, 0), "1:0:A"), + label((9, 0), "1:0:B"), + ], + )]); + assert!( + Document::parse(&conflicts) + .unwrap_err() + .contains("conflicting names") + ); + assert!( + pheon::review::dump_records(&conflicts) + .unwrap() + .contains("1:0:B") + ); + let future = layout(&[("CHIP", vec![label((9, 0), "KLAMATH_LAYER_NAMES_V2")])]); + let future = Document::parse(&future).unwrap(); + assert_eq!(Comparison::between(&future, &future).exit_code(), 2); +} + +#[test] +fn unknown_records_never_produce_a_successful_equality_claim() { + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![unknown])]); + let doc = Document::parse(&bytes).unwrap(); + let diff = Comparison::between(&doc, &doc); + assert!(!diff.is_equal()); + assert_eq!(diff.exit_code(), 2); + assert!(doc.record_dump().contains("7f00")); + assert!(diff.text(0).contains("INCOMPLETE")); +} + +#[test] +fn shared_hierarchy_reports_ancestors_and_marks_repeated_definitions() { + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + hierarchy(&a, &b, Some(&diff), false, 8), + concat!( + "scope top=all cell=all layer=all\n", + "\"CHIP\" x1 [affected]\n", + " \"left\" x1 [affected]\n", + " \"leaf\" x2 [cell changed]\n", + " \"right\" x1 [affected]\n", + " \"leaf\" x3 [cell changed] [shown above]\n", + ) + ); + let graph = hierarchy(&a, &b, Some(&diff), true, 8); + assert_eq!(graph.matches("leaf [cell changed]").count(), 1); + assert!(graph.contains("x2")); + assert!(graph.contains("x3")); + let limited = hierarchy(&a, &b, Some(&diff), false, 1); + assert!(!limited.contains("\"leaf\"")); + assert!(limited.contains("children omitted")); +} + +#[test] +fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { + for pair in corpus() + .into_iter() + .filter(|p| p.name == "moved reference" || p.name == "array count") + { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + let tree = hierarchy(&old, &new, Some(&diff), false, 8); + assert!(tree.contains("[cell unchanged]"), "{tree}"); + assert!(tree.contains("[reference changed]"), "{tree}"); + if pair.name == "array count" { + assert!(tree.contains("x2 -> x3"), "{tree}"); + } + } + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]); + let b = layout(&[("CHIP", vec![])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let tree = hierarchy(&a, &b, Some(&Comparison::between(&a, &b)), false, 8); + assert!(tree.contains("\"leaf\" x1 -> x0"), "{tree}"); +} + +#[test] +fn same_bounds_changes_and_truncated_reports_remain_differences() { + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle()])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + diff.cells[0].removed[0].bounds, + diff.cells[0].added[0].bounds + ); + assert!(diff.text(0).contains("0 of 2 object changes shown")); + assert_eq!(diff.exit_code(), 1); + assert!(diff.cells[0].removed[0].offset > 0); +} + +#[test] +fn unit_changes_report_values_without_claiming_geometry_equivalence() { + let a = layout(&[("CHIP", vec![square()])]); + let units = Document::parse(&a) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + let mut b = a.clone(); + b[units + 12..units + 20].copy_from_slice( + &pheon::basic::encode_real8(2e-9) + .unwrap() + .to_be_bytes(), + ); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!(diff.exit_code(), 1); + assert_eq!(diff.library_fields.len(), 1); + assert_eq!(diff.library_fields[0].field, "UNITS"); + assert!(diff.text(0).contains("UNITS changed")); + assert!(diff.cells.is_empty()); +} + +#[test] +fn dangling_references_cycles_and_duplicate_cell_names_are_reported() { + let missing = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let missing = Document::parse(&missing).unwrap(); + assert!( + missing + .issues + .iter() + .any(|s| s.contains("unresolved reference")) + ); + let cycle = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let cycle = Document::parse(&cycle).unwrap(); + assert!(cycle.issues.iter().any(|s| s.contains("cyclic"))); + assert!(hierarchy(&cycle, &cycle, None, false, 8).len() < 200); + let duplicate = layout(&[("a", vec![]), ("a", vec![])]); + assert!( + Document::parse(&duplicate) + .unwrap_err() + .contains("duplicate cell") + ); +} + +#[test] +fn absolute_transform_flags_are_not_discarded() { + let a = reference("leaf", 0, 1); + let mut b = Vec::new(); + record(&mut b, RTAG_SREF, &[]); + ascii(&mut b, RTAG_SNAME, "leaf"); + record(&mut b, RTAG_STRANS, &[0, 2]); + xy(&mut b, &[[0, 0]]); + record(&mut b, RTAG_ENDEL, &[]); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + assert_eq!( + Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .exit_code(), + 1 + ); +} + +#[test] +fn polygon_normalization_handles_repeated_minimum_vertices() { + let ring = [[0, 0], [4, 0], [4, 4], [0, 0], [-2, 2], [-2, 4], [0, 0]]; + let baseline = layout(&[("CHIP", vec![polygon(&ring)])]); + let baseline = Document::parse(&baseline).unwrap(); + let mut open = ring[..ring.len() - 1].to_vec(); + for _ in 0..2 { + for start in 0..open.len() { + let mut rotated = open[start..] + .iter() + .chain(&open[..start]) + .copied() + .collect::>(); + rotated.push(rotated[0]); + let candidate = layout(&[("CHIP", vec![polygon(&rotated)])]); + let candidate = Document::parse(&candidate).unwrap(); + assert!(Comparison::between(&baseline, &candidate).is_equal()); + } + open.reverse(); + } +} + +#[test] +fn malformed_lengths_fields_and_units_return_errors_without_panicking() { + let good = layout(&[("CHIP", vec![square()])]); + for length in 0..good.len() { + assert!( + Document::parse(&good[..length]).is_err(), + "accepted truncated length {length}" + ); + } + let mut bad = good.clone(); + bad[0..2].copy_from_slice(&3_u16.to_be_bytes()); + assert!(Document::parse(&bad).is_err()); + let mut bad = good.clone(); + let units = Document::parse(&good) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + bad[units + 4..units + 20].fill(0); + assert!(Document::parse(&bad).unwrap_err().contains("UNITS")); + let mut polygon = square(); + polygon.truncate(polygon.len() - 4); + record(&mut polygon, RTAG_LAYER, &[0, 1]); + record(&mut polygon, RTAG_ENDEL, &[]); + assert!( + Document::parse(&layout(&[("CHIP", vec![polygon])])) + .unwrap_err() + .contains("duplicate element field") + ); +} + +#[test] +fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { + let directory = std::env::temp_dir().join(format!("pheon review {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old file.gds"); + let new = directory.join("new file.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let equal = command().arg("diff").arg(&old).arg(&old).output().unwrap(); + assert_eq!(equal.status.code(), Some(0)); + let diff = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "mermaid", "--max-changes", "0"]) + .output() + .unwrap(); + assert_eq!(diff.status.code(), Some(1)); + let text = String::from_utf8(diff.stdout).unwrap(); + assert!(text.contains("0 of 2 object changes shown")); + assert!(text.contains("```mermaid\nflowchart TD")); + let dump = command() + .arg("dump") + .arg(&old) + .args(["--records"]) + .output() + .unwrap(); + assert_eq!(dump.status.code(), Some(0)); + assert!(String::from_utf8(dump.stdout).unwrap().contains("BGNSTR")); + let graph = command() + .arg("hierarchy") + .arg(&old) + .args(["--format", "mermaid"]) + .output() + .unwrap(); + assert_eq!(graph.status.code(), Some(0)); + assert!( + String::from_utf8(graph.stdout) + .unwrap() + .starts_with("flowchart TD") + ); + fs::write(&new, b"invalid").unwrap(); + assert_eq!( + command() + .arg("diff") + .arg(&old) + .arg(&new) + .output() + .unwrap() + .status + .code(), + Some(2) + ); +} + +#[test] +fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { + use pheon::review::write_layer_names; + use std::collections::BTreeMap; + let mut bytes = shared(with_property(square(), "keep this")); + bytes.extend_from_slice(&[0; 32]); + let updates = BTreeMap::from([ + ((1, 0), "WG".to_string()), + ((2, 3), "HEATER:draw".to_string()), + ]); + let output = write_layer_names(&bytes, "CHIP", (900, 7), &updates).unwrap(); + let doc = Document::parse(&output).unwrap(); + assert_eq!(doc.layer_names, updates); + let meta: Vec<_> = doc.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .collect(); + let start = meta.iter().map(|o| o.offset).min().unwrap(); + let end = meta.iter().map(|o| o.end_offset).max().unwrap(); + assert_eq!([&output[..start], &output[end..]].concat(), bytes); + let updated = write_layer_names( + &output, + "CHIP", + (900, 7), + &BTreeMap::from([((1, 0), "CORE".to_string())]), + ) + .unwrap(); + let parsed = Document::parse(&updated).unwrap(); + assert_eq!(parsed.layer_names[&(1, 0)], "CORE"); + assert_eq!(parsed.layer_names[&(2, 3)], "HEATER:draw"); + assert_eq!( + parsed.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .count(), + 3 + ); + let diff = Comparison::between(&Document::parse(&output).unwrap(), &parsed); + assert!(diff.cells.is_empty()); + assert_eq!(diff.layer_names.len(), 1); + assert_eq!( + write_layer_names(&updated, "CHIP", (900, 7), &BTreeMap::new()).unwrap(), + updated + ); +} + +#[test] +fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { + use pheon::review::{parse_layer_name, write_layer_names}; + use std::collections::BTreeMap; + let bytes = layout(&[("CHIP", vec![label((900, 7), "ordinary label")])]); + assert!( + write_layer_names(&bytes, "CHIP", (900, 7), &BTreeMap::new()) + .unwrap_err() + .contains("ordinary design") + ); + assert!( + write_layer_names(&bytes, "missing", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("does not exist") + ); + let bytes = layout(&[("CHIP", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")])]); + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("unsupported") + ); + let bytes = layout(&[("CHIP", vec![square()])]); + for value in ["".to_string(), "bad\0name".to_string(), "x".repeat(65536)] { + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::from([((1, 0), value)])) + .is_err() + ); + } + for text in ["+1:0:name", "1:-1:name", "1:0:", "65536:0:name"] { + assert!(parse_layer_name(text).is_err()); + } +} + +#[test] +fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { + let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let source = directory.join("source.gds"); + let output = directory.join("output.gds"); + let bytes = shared(square()); + fs::write(&source, &bytes).unwrap(); + let _ = fs::remove_file(&output); + let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); + command.arg("name-layers").arg(&source).arg(&output).args([ + "--cell", + "CHIP", + "--carrier", + "900/7", + "--name", + "1:0:WG", + ]); + let result = command.output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(fs::read(&source).unwrap(), bytes); + let generated = fs::read(&output).unwrap(); + assert_eq!( + Document::parse(&generated).unwrap().layer_names[&(1, 0)], + "WG" + ); + assert_eq!(command.output().unwrap().status.code(), Some(2)); + assert_eq!(fs::read(&output).unwrap(), generated); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { + use pheon::review::Scope; + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ("unused", vec![square()]), + ]); + let b = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![triangle()]), + ("unused", vec![triangle()]), + ]); + let top = Scope { + top: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&top, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&top, false).unwrap(); + assert_eq!( + old.cells.keys().map(String::as_str).collect::>(), + ["CHIP", "leaf"] + ); + assert_eq!(Comparison::between(&old, &new).cells.len(), 1); + assert!(old.canonical_dump().contains("scope top=\"CHIP\"")); + let cell = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&cell, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&cell, false).unwrap(); + assert!(old.issues.is_empty()); + assert!(Comparison::between(&old, &new).is_equal()); + let missing = Scope { + top: Some("missing".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&a) + .unwrap() + .selected(&missing, false) + .is_err() + ); + let wrong = Scope { + top: Some("CHIP".to_string()), + cell: Some("unused".to_string()), + layer: None, + }; + assert!( + wrong + .validate_pair(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .is_err() + ); +} + +#[test] +fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { + use pheon::review::Scope; + let mut other_layer = square(); + other_layer[8..10].copy_from_slice(&2_u16.to_be_bytes()); + let a = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 0, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer.clone()]), + ]); + let b = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 5, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer]), + ]); + for (layer, expected) in [((1, 0), 0), ((2, 0), 1)] { + let scope = Scope { + layer: Some(layer), + ..Scope::default() + }; + let old = Document::parse(&a) + .unwrap() + .selected(&scope, false) + .unwrap(); + let new = Document::parse(&b) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!(Comparison::between(&old, &new).exit_code(), expected); + assert!(old.issues.is_empty()); + } +} + +#[test] +fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { + use pheon::review::Scope; + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![square()]), ("unused", vec![unknown])]); + let scope = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .is_empty() + ); + let scope = Scope { + cell: Some("unused".to_string()), + ..Scope::default() + }; + assert!( + !Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .is_empty() + ); + let bytes = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let scope = Scope { + layer: Some((999, 0)), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .issues + .iter() + .any(|s| s.contains("cyclic")) + ); +} + +#[test] +fn display_filters_never_change_equality_or_totals() { + use pheon::review::DisplayFilter; + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text_filtered( + 100, + &DisplayFilter { + cell: None, + layer: Some((42, 0)), + }, + ); + assert!(text.contains("display-only")); + assert!(text.contains("0 of 2 object changes shown")); + assert_eq!(diff.exit_code(), 1); +} + +#[test] +fn git_external_driver_handles_changes_additions_deletions_and_errors() { + let directory = std::env::temp_dir().join(format!( + "pheon-git-review-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let inherited_path = std::env::var_os("PATH").unwrap_or_default(); + let git_path = std::env::join_paths( + std::iter::once( + PathBuf::from(env!("CARGO_BIN_EXE_pheon")) + .parent() + .unwrap() + .to_path_buf(), + ) + .chain(std::env::split_paths(&inherited_path)), + ) + .unwrap(); + let git = |args: &[&str]| { + let result = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(args) + .output() + .unwrap(); + assert!( + result.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&result.stderr) + ); + result + }; + git(&["init", "-q"]); + git(&["config", "user.name", "Review Fixture"]); + git(&["config", "user.email", "fixture@example.invalid"]); + git(&["config", "commit.gpgsign", "false"]); + git(&["config", "core.hooksPath", "/dev/null"]); + git(&[ + "config", + "include.path", + concat!(env!("CARGO_MANIFEST_DIR"), "/contrib/git/config"), + ]); + assert_eq!( + String::from_utf8(git(&["config", "--get", "diff.pheon.trustExitCode"]).stdout) + .unwrap() + .trim(), + "false" + ); + fs::write( + directory.join(".gitattributes"), + include_str!("../contrib/git/attributes"), + ) + .unwrap(); + let path = directory.join("chip layout.gds"); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + git(&["add", "."]); + git(&["commit", "-qm", "baseline"]); + fs::write(&path, layout(&[("CHIP", vec![triangle()])])).unwrap(); + let changed = git(&["diff", "--", "chip layout.gds"]); + assert!( + String::from_utf8(changed.stdout) + .unwrap() + .contains("2 of 2 object changes shown") + ); + fs::write(&path, b"invalid").unwrap(); + let invalid = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(["diff", "--", "chip layout.gds"]) + .output() + .unwrap(); + assert!(!invalid.status.success()); + assert!(String::from_utf8_lossy(&invalid.stderr).contains("pheon:")); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + let added = directory.join("added layout.gds"); + fs::write(&added, shared(square())).unwrap(); + git(&["add", "added layout.gds"]); + let result = git(&["diff", "--cached", "--", "added layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("layout added") + ); + fs::remove_file(&path).unwrap(); + git(&["add", "-u"]); + let result = git(&["diff", "--cached", "--", "chip layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("layout deleted") + ); + // Textconv is an alternative to the external driver, with a separate cache. + git(&["commit", "-qm", "replace fixture"]); + git(&["mv", "added layout.gds", "renamed layout.gds"]); + let renamed = directory.join("renamed layout.gds"); + fs::write(&renamed, shared(triangle())).unwrap(); + git(&["add", "renamed layout.gds"]); + let result = git(&["diff", "--cached", "--find-renames"]); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.contains("renamed layout.gds"), "{text}"); + assert!(text.contains("2 of 2 object changes shown"), "{text}"); + git(&["commit", "-qm", "rename fixture"]); + fs::write( + directory.join(".gitattributes"), + "*.gds diff=pheon-text\n", + ) + .unwrap(); + fs::write(&renamed, shared(square())).unwrap(); + let result = git(&["diff", "--", "renamed layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("- BOUNDARY") + ); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn replacement_explanations_are_unique_and_keep_full_evidence() { + for pair in corpus().into_iter().filter(|p| { + [ + "same bounds different geometry", + "moved reference", + "array count", + ] + .contains(&p.name) + }) { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert_eq!(diff.cells[0].replacements.len(), 1); + let text = diff.text(100); + assert!(text.contains(" ~ ")); + assert!(text.contains(" - @")); + assert!(text.contains(" + @")); + match pair.name { + "moved reference" => assert!(text.contains("delta=(5, 0) dbu")), + "array count" => assert!(text.contains("columns/rows 2 x 1 -> 3 x 1")), + _ => assert_eq!(text, include_str!("golden/review-diff.txt")), + } + assert!(!diff.text(0).contains(" ~ ")); + assert!(diff.text(1).contains("1 of 2 object changes shown")); + } + let a = layout(&[("CHIP", vec![square(), square()])]); + let b = layout(&[("CHIP", vec![triangle(), triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!(diff.cells[0].replacements.is_empty()); +} + +#[test] +fn path_width_explanation_preserves_absent_and_zero() { + let path = |width: Option| { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_PATH, &[]); + record(&mut bytes, RTAG_LAYER, &[0, 1]); + record(&mut bytes, RTAG_DATATYPE, &[0, 0]); + if let Some(n) = width { + record(&mut bytes, RTAG_WIDTH, &n.to_be_bytes()); + } + xy(&mut bytes, &[[0, 0], [10, 10]]); + record(&mut bytes, RTAG_ENDEL, &[]); + layout(&[("CHIP", vec![bytes])]) + }; + let a = path(None); + let b = path(Some(0)); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!(diff.text(100).contains("width absent -> 0 dbu")); +} + +#[test] +fn unchanged_layer_names_explain_geometry_changes_once() { + let named = |geometry| { + layout(&[( + "CHIP", + vec![ + geometry, + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + )]) + }; + let a = named(square()); + let b = named(triangle()); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text(100); + assert_eq!(text.matches("name=\"WG\"").count(), 1); + assert!(diff.layer_names.is_empty()); +} + +#[test] +fn an_added_empty_document_is_still_a_presence_change() { + let bytes = layout(&[]); + let doc = Document::parse(&bytes).unwrap(); + let diff = Comparison::between(&doc.empty_like(), &doc); + assert_eq!(diff.exit_code(), 1); + assert!(diff.text(0).contains("library presence:")); + assert!(!diff.text(0).contains("equal under")); +} + +#[test] +fn cli_scope_and_display_filters_have_distinct_exit_behavior() { + let directory = std::env::temp_dir().join(format!("pheon-scope-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + for (options, expected) in [ + (vec!["--cell", "CHIP"], 0), + (vec!["--top", "CHIP"], 1), + (vec!["--layer", "99/0"], 0), + (vec!["--show-cell", "missing"], 1), + (vec!["--show-layer", "99/0"], 1), + (vec!["--top", "missing"], 2), + ] { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(&options) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(expected), + "{options:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + fs::write(&old, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["diff", "--top", "CHIP"]) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn large_polygons_and_billion_instance_arrays_stay_compact() { + let mut points = vec![[0, 0], [0, 1]]; + points.extend((1..8187).map(|x| [x, 1])); + points.extend([[8186, 0], [0, 0]]); + let mut array = Vec::new(); + record(&mut array, RTAG_AREF, &[]); + ascii(&mut array, RTAG_SNAME, "leaf"); + record(&mut array, RTAG_COLROW, &[0x7f, 0xff, 0x7f, 0xff]); + xy(&mut array, &[[0, 0], [32767, 0], [0, 32767]]); + record(&mut array, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![array]), ("leaf", vec![polygon(&points)])]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.edges()[&("CHIP".to_string(), "leaf".to_string())], + 32767 * 32767 + ); + assert_eq!(doc.cells["CHIP"].objects.len(), 1); + assert!(hierarchy(&doc, &doc, None, false, 8).len() < 200); + assert!(Comparison::between(&doc, &doc).is_equal()); +} + +#[test] +fn deep_hierarchy_comparison_and_rendering_are_iterative() { + let names: Vec<_> = (0..2048).map(|i| format!("cell_{i}")).collect(); + let cells = |leaf: Vec| { + names + .iter() + .enumerate() + .map(|(i, name)| { + ( + name.as_str(), + if i + 1 == names.len() { + vec![leaf.clone()] + } else { + vec![reference(&names[i + 1], 0, 1)] + }, + ) + }) + .collect::>() + }; + let a = layout(&cells(square())); + let b = layout(&cells(triangle())); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + let tree = hierarchy(&a, &b, Some(&diff), false, 5); + assert_eq!(diff.cells.len(), 1); + assert!(tree.contains("children omitted")); + assert!(tree.len() < 500); +} diff --git a/tests/review/reports.rs b/tests/review/reports.rs new file mode 100644 index 0000000..f761a8c --- /dev/null +++ b/tests/review/reports.rs @@ -0,0 +1,633 @@ +use super::*; +use pheon::review::{ + DisplayFilter, Scope, + report::{ReportInputs, ReportOptions, failure_report}, +}; +use serde_json::{Value, json}; + +fn inputs() -> ReportInputs { + ReportInputs { + old: Some("before.gds".into()), + new: Some("after.gds".into()), + } +} + +fn report(old: &[u8], new: &[u8], options: &ReportOptions) -> Value { + let a = Document::parse(old).unwrap(); + let b = Document::parse(new).unwrap(); + Comparison::between(&a, &b).json_report(&a, &b, &inputs(), options) +} + +fn unknown(geometry: Vec) -> Vec { + let mut object = geometry; + object.truncate(object.len() - 4); + record(&mut object, 0x7f02, &[0, 42]); + record(&mut object, RTAG_ENDEL, &[]); + layout(&[("CHIP", vec![object])]) +} + +#[test] +fn reports_preserve_fixture_results_totals_and_replacement_links() { + for pair in corpus() { + let r = report(&pair.old, &pair.new, &ReportOptions::default()); + assert_eq!(r["schema_version"], 1); + assert_eq!(r["comparison_profile"], "klamath-object-v2"); + assert_eq!(r["producer"]["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(r["status"], "complete", "{}", pair.name); + assert_eq!( + r["exit_code"], + if pair.changed_cells + pair.metadata == 0 { + 0 + } else { + 1 + } + ); + assert_eq!( + r["summary"]["changed_cells"], + pair.changed_cells.to_string() + ); + assert_eq!(r["summary"]["added_objects"], pair.added.to_string()); + assert_eq!(r["summary"]["removed_objects"], pair.removed.to_string()); + assert_eq!( + r["summary"]["layer_name_changes"], + pair.metadata.to_string() + ); + assert_eq!(r["report_selection"]["truncated"], false); + for cell in r["cell_changes"].as_array().unwrap() { + for replacement in cell["replacements"].as_array().unwrap() { + assert!( + cell["removed"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["removed"]) + ); + assert!( + cell["added"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["added"]) + ); + } + } + assert_eq!(r, report(&pair.old, &pair.new, &ReportOptions::default())); + } +} + +#[test] +fn incomplete_reports_distinguish_known_differences_and_include_structured_locations() { + let a = unknown(square()); + let same = report(&a, &a, &ReportOptions::default()); + assert_eq!(same["status"], "incomplete"); + assert_eq!(same["result"], "unknown"); + assert_eq!(same["exit_code"], 2); + assert_eq!(same["summary"]["counts"], "observed"); + let diagnostic = &same["diagnostics"][0]; + assert_eq!(diagnostic["code"], "unsupported_record"); + assert_eq!(diagnostic["side"], "old"); + assert_eq!(diagnostic["cell"], "CHIP"); + assert_eq!(diagnostic["layer"], json!({"number":1,"purpose":0})); + let offset: usize = diagnostic["byte_offset"].as_str().unwrap().parse().unwrap(); + assert_eq!(&a[offset + 2..offset + 4], &0x7f02_u16.to_be_bytes()); + let changed = report(&a, &unknown(triangle()), &ReportOptions::default()); + assert_eq!(changed["status"], "incomplete"); + assert_eq!(changed["result"], "different"); + assert_eq!(changed["exit_code"], 2); + assert_eq!( + changed["cell_changes"][0]["removed"][0]["fields"] + .as_array() + .unwrap() + .last() + .unwrap()["payload_hex"], + "0x002a" + ); +} + +#[test] +fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { + for location in ["CHIP", "legend"] { + let bytes = if location == "CHIP" { + layout(&[( + "CHIP", + vec![square(), label((900, 7), "KLAMATH_LAYER_NAMES_V2")], + )]) + } else { + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ]) + }; + let source = Document::parse(&bytes).unwrap().diagnostics[0].offset; + for scope in [ + Scope::default(), + Scope { + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + layer: Some((999, 0)), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + ] { + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!(doc.diagnostics.len(), 1, "{scope:?}"); + let diagnostic = &doc.diagnostics[0]; + assert_eq!(diagnostic.code, "unsupported_metadata_version"); + assert_eq!(diagnostic.cell.as_deref(), Some(location)); + assert_eq!(diagnostic.layer, Some((900, 7))); + assert_eq!(diagnostic.offset, source); + let diff = Comparison::between(&doc, &doc); + assert_eq!(diff.exit_code(), 2); + assert!(!diff.is_equal()); + assert!(diff.text(0).contains("INCOMPLETE")); + let json = diff.json_report(&doc, &doc, &inputs(), &ReportOptions::default()); + assert_eq!(json["status"], "incomplete"); + assert_eq!(json["result"], "unknown"); + assert_eq!(json["exit_code"], 2); + assert_eq!(json["diagnostics"].as_array().unwrap().len(), 2); + } + } +} + +#[test] +fn json_is_unbounded_by_default_and_explicit_filters_do_not_change_totals() { + let a = layout(&[("CHIP", vec![])]); + let b = layout(&[("CHIP", vec![square(); 120])]); + let full = report(&a, &b, &ReportOptions::default()); + assert_eq!( + full["cell_changes"][0]["added"].as_array().unwrap().len(), + 120 + ); + let limited = report( + &a, + &b, + &ReportOptions { + max_objects: Some(1), + ..Default::default() + }, + ); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["exit_code"], 1); + assert_eq!(limited["report_selection"]["truncated"], true); + assert_eq!(limited["report_selection"]["limited_objects"], "119"); + assert_eq!(limited["report_selection"]["emitted_objects"], "1"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: Some("absent".into()), + layer: None, + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["cell_changes"], json!([])); + assert_eq!(filtered["report_selection"]["filtered_cells"], "1"); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + assert_eq!(filtered["cell_changes"][0]["added"], json!([])); +} + +#[test] +fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { + let mut a = reference("leaf", 0, 1); + a.truncate(a.len() - 4); + record(&mut a, RTAG_MAG, &0x4110000000000000_u64.to_be_bytes()); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_ENDEL, &[]); + let mut b = a.clone(); + let mag = Document::parse(&layout(&[("CHIP", vec![a.clone()]), ("leaf", vec![])])) + .unwrap() + .records + .into_iter() + .find(|r| r.tag == RTAG_MAG) + .unwrap() + .offset; + // Replace the REAL8 payload within the object, retaining the same decoded value. + let within = a + .windows(8) + .position(|v| v == 0x4110000000000000_u64.to_be_bytes()) + .unwrap(); + b[within..within + 8].copy_from_slice(&0x4201000000000000_u64.to_be_bytes()); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + let r = report(&a, &b, &ReportOptions::default()); + let replacement = &r["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_kind_target"); + let change = &replacement["field_changes"][0]; + assert_eq!(change["tag"], RTAG_MAG); + assert_eq!(change["before"]["value"], change["after"]["value"]); + assert_ne!( + change["before"]["payload_hex"], + change["after"]["payload_hex"] + ); + let removed = &r["cell_changes"][0]["removed"][0]; + assert_eq!(removed["properties"].as_array().unwrap().len(), 2); + assert_eq!(removed["properties"][0]["attribute"], 17); + assert_eq!(removed["properties"][0]["value"], Value::Null); + assert_eq!(removed["properties"][0]["payload_hex"], "0xfffe"); + let offset: usize = removed["source"]["byte_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert!(offset < mag); + let end: usize = removed["source"]["end_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert_eq!(&a[end - 4..end], &[0, 4, 17, 0]); + + let mut path = square(); + path[2..4].copy_from_slice(&RTAG_PATH.to_be_bytes()); + let mut width = path.clone(); + width.truncate(width.len() - 4); + record(&mut width, RTAG_WIDTH, &0_i32.to_be_bytes()); + record(&mut width, RTAG_ENDEL, &[]); + let r = report( + &layout(&[("CHIP", vec![path])]), + &layout(&[("CHIP", vec![width])]), + &ReportOptions::default(), + ); + let change = &r["cell_changes"][0]["replacements"][0]["field_changes"][0]; + assert_eq!(change["tag"], RTAG_WIDTH); + assert_eq!(change["before"], Value::Null); + assert_eq!(change["after"]["value"], json!([0])); +} + +#[test] +fn hierarchy_reports_shared_ancestors_and_symbolic_arrays() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let r = report(&shared(square()), &shared(triangle()), &options); + let graph = &r["hierarchy"]; + assert_eq!(graph["nodes"].as_array().unwrap().len(), 4); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "CHIP") + .unwrap()["change"], + "affected" + ); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "leaf") + .unwrap()["change"], + "direct" + ); + let edge = graph["edges"] + .as_array() + .unwrap() + .iter() + .find(|e| e["parent"] == "right") + .unwrap(); + assert_eq!(edge["before_count"], "3"); + assert_eq!(edge["reference_changed"], false); + let bytes = shared(square()); + let scope = Scope { + cell: Some("CHIP".into()), + ..Default::default() + }; + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + let r = Comparison::between(&doc, &doc).json_report(&doc, &doc, &inputs(), &options); + assert_eq!(r["scope"]["cell"], "CHIP"); + assert_eq!( + r["hierarchy"]["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "left") + .unwrap()["after"], + "outside_scope" + ); + let dangling = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let r = report(&dangling, &dangling, &options); + assert_eq!(r["diagnostics"][0]["code"], "unresolved_reference"); + assert_eq!(r["diagnostics"][0]["target"], "missing"); + assert_eq!(r["hierarchy"]["nodes"][1]["after"], "unresolved"); + let cycle = layout(&[("CHIP", vec![reference("CHIP", 0, 1)])]); + assert_eq!( + report(&cycle, &cycle, &options)["diagnostics"][0]["code"], + "hierarchy_cycle" + ); +} + +#[test] +fn library_attributes_empty_cells_units_and_absence_are_explicit() { + let a = layout(&[]); + let b = layout(&[("empty", vec![])]); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["cell_changes"][0]["before"], Value::Null); + assert_eq!(r["cell_changes"][0]["after"]["attributes"], json!([])); + let mut b = b; + let end = Document::parse(&b).unwrap().cells["empty"].end_offset; + let mut attribute = Vec::new(); + record(&mut attribute, RTAG_STRCLASS, &[0, 1]); + b.splice(end..end, attribute); + let r = report(&layout(&[("empty", vec![])]), &b, &ReportOptions::default()); + assert_eq!( + r["cell_changes"][0]["after"]["attributes"][0]["value"], + json!([1]) + ); + assert_eq!(r["inputs"]["old"]["units"]["encoding"], "real8"); + let doc = Document::parse(&a).unwrap(); + let absent = doc.empty_like(); + let r = Comparison::between(&absent, &doc).json_report( + &absent, + &doc, + &inputs(), + &ReportOptions::default(), + ); + assert_eq!(r["inputs"]["old"]["present"], false); + assert_eq!(r["inputs"]["old"]["units"], Value::Null); + assert_eq!( + r["library_changes"], + json!([{"field":"presence","tag":null,"before":false,"after":true}]) + ); + let mut b = a.clone(); + let units = doc + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + b[units + 12..units + 20].copy_from_slice( + &pheon::basic::encode_real8(2e-9) + .unwrap() + .to_be_bytes(), + ); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["library_changes"][0]["tag"], RTAG_UNITS); + assert_ne!(r["inputs"]["old"]["units"], r["inputs"]["new"]["units"]); +} + +#[test] +fn reports_keep_extreme_coordinates_unsigned_layers_and_timestamp_equality() { + let a = layout(&[("quoted\"\\\n日本", vec![])]); + let mut shape = polygon(&[ + [i32::MIN, i32::MIN], + [i32::MAX, i32::MIN], + [i32::MAX, i32::MAX], + [i32::MIN, i32::MAX], + [i32::MIN, i32::MIN], + ]); + shape[8..10].copy_from_slice(&u16::MAX.to_be_bytes()); + let b = layout(&[("quoted\"\\\n日本", vec![shape])]); + let r = report(&a, &b, &ReportOptions::default()); + let object = &r["cell_changes"][0]["added"][0]; + assert_eq!(object["layer"]["number"], u16::MAX); + assert_eq!( + object["bounds"], + json!([i32::MIN, i32::MIN, i32::MAX, i32::MAX]) + ); + let encoded = serde_json::to_string(&r).unwrap(); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), r); + let mut times = b.clone(); + let doc = Document::parse(&b).unwrap(); + for record in doc + .records + .iter() + .filter(|r| matches!(r.tag, RTAG_BGNLIB | RTAG_BGNSTR)) + { + times[record.offset + 4..record.offset + 28].fill(1); + } + let r = report(&b, ×, &ReportOptions::default()); + assert_eq!(r["result"], "equal"); + assert_eq!(r["exit_code"], 0); +} + +#[cfg(unix)] +#[test] +fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { + use std::os::unix::ffi::OsStringExt; + let directory = std::env::temp_dir().join(format!("pheon json paths {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let path = directory.join(std::ffi::OsString::from_vec(b"file\xff.gds".to_vec())); + fs::write(&path, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(["--format", "json", "--"]) + .arg(&path) + .arg(&path) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + let r: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(r["inputs"]["old"]["path"], Value::Null); + let expected = "0x".to_string() + + &path + .as_os_str() + .as_encoded_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + assert_eq!(r["inputs"]["old"]["path_encoded_hex"], expected); + for name in ["--format", "json"] { + fs::write(directory.join(name), layout(&[])).unwrap(); + } + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .current_dir(&directory) + .args(["diff", "--", "--format", "json"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + assert!( + String::from_utf8(output.stdout) + .unwrap() + .starts_with("profile ") + ); +} + +#[test] +fn machine_reports_match_versioned_goldens() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let equal = layout(&[("CHIP", vec![square()])]); + let metadata = corpus() + .into_iter() + .find(|p| p.name == "layer name only") + .unwrap(); + let samples = [ + ("equal", report(&equal, &equal, &ReportOptions::default())), + ( + "geometry", + report(&shared(square()), &shared(triangle()), &options), + ), + ( + "metadata", + report(&metadata.old, &metadata.new, &ReportOptions::default()), + ), + ( + "incomplete", + report( + &unknown(square()), + &unknown(square()), + &ReportOptions::default(), + ), + ), + ( + "failed", + failure_report( + &inputs(), + &Scope::default(), + &ReportOptions::default(), + "invalid_input", + "truncated record", + Some("new"), + ), + ), + ]; + for (name, mut actual) in samples { + actual["producer"]["version"] = json!("test"); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join(format!("tests/golden/report-{name}.json")); + if std::env::var_os("PHEON_UPDATE_REPORT_GOLDENS").is_some() { + fs::write(&path, serde_json::to_string_pretty(&actual).unwrap() + "\n").unwrap(); + } + let expected: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(actual, expected, "{name}"); + } +} + +#[test] +fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { + let directory = std::env::temp_dir().join(format!("pheon json {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let a = directory.join("before file.gds"); + let b = directory.join("after file.gds"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); + let command = |extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(["--format", "json"]) + .args(extra) + .output() + .unwrap(); + assert!(output.stderr.is_empty(), "{:?}", output.stderr); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + output.status.code().unwrap(), + report["exit_code"].as_i64().unwrap() as i32 + ); + report + }; + let full = command(&["--hierarchy", "graph"]); + assert_eq!(full["result"], "different"); + assert!(full["hierarchy"].is_object()); + let limited = command(&["--max-changes", "0"]); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["report_selection"]["emitted_objects"], "0"); + for extra in [ + &["--top", "missing"][..], + &["--unknown"], + &["--max-changes", "x"], + &["--hierarchy", "mermaid"], + &["--depth", "3"], + ] { + let failure = command(extra); + assert_eq!(failure["status"], "failed"); + assert_eq!(failure["result"], "unknown"); + assert_eq!(failure["summary"], Value::Null); + } + fs::write(&b, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "new"); + assert_eq!(invalid["inputs"]["new"]["path"], b.to_str().unwrap()); + fs::remove_file(&b).unwrap(); + assert_eq!(command(&[])["diagnostics"][0]["code"], "io_error"); + fs::write(&b, fs::read(&a).unwrap()).unwrap(); + assert_eq!(command(&[])["result"], "equal"); + fs::write(&b, unknown(square())).unwrap(); + assert_eq!(command(&[])["status"], "incomplete"); + fs::write( + &b, + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "KLAMATH_LAYER_NAMES_V2")]), + ]), + ) + .unwrap(); + for flags in [ + &["--layer", "1/0"][..], + &["--top", "CHIP"], + &["--cell", "CHIP", "--layer", "1/0"], + ] { + let report = command(flags); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["exit_code"], 2); + assert_eq!( + report["diagnostics"][0]["code"], + "unsupported_metadata_version" + ); + } +} diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs new file mode 100644 index 0000000..8db1d81 --- /dev/null +++ b/tests/selective_reads.rs @@ -0,0 +1,768 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; +use arrow_array::{ + Array, BooleanArray, Float64Array, Int16Array, Int32Array, ListArray, StringArray, StructArray, + UInt32Array, UInt64Array, +}; +use common::{library_header, record}; +use pheon::{ + basic::encode_real8, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_cells_bytes, read_path, records, scan_bytes, +}; +use std::{ + ffi::{CStr, CString}, + ptr, +}; + +fn xy(values: &[i32]) -> Vec { + values.iter().flat_map(|v| v.to_be_bytes()).collect() +} + +fn fixture() -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_XY, &xy(&[3, -4])); + record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]); + record( + &mut bytes, + records::RTAG_MAG, + &encode_real8(2.0).unwrap().to_be_bytes(), + ); + record( + &mut bytes, + records::RTAG_ANGLE, + &encode_real8(90.0).unwrap().to_be_bytes(), + ); + record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]); + record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray { + let list = array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + list.value(row) + .as_any() + .downcast_ref::() + .unwrap() + .clone() +} + +fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + let counts: Vec = counts.into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, records::RTAG_COLROW, &counts); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +#[test] +fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { + let bytes = aref_fixture( + [i32::MIN, i32::MAX], + [i32::MAX - 1, i32::MIN + 1], + [i32::MIN, i32::MIN + 1], + [2, 2], + ); + let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0); + let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0); + for refs in [ + list_struct(&full_cells, "arefs", 0), + list_struct(&scan_cells, "refs", 0), + ] { + let column = u64_value(&refs, "xy0", 0); + let row = u64_value(&refs, "xy1", 0); + assert_eq!((column >> 32) as i32, i32::MAX); + assert_eq!(column as i32, -i32::MAX); + assert_eq!((row >> 32) as i32, 0); + assert_eq!(row as i32, -i32::MAX); + } +} + +#[test] +fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { + for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { + let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); + assert!(read_library(&bytes).is_err()); + assert!(scan_library(&bytes).is_err()); + } + let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); + assert!(read_library(&bytes).is_err()); + assert!(scan_library(&bytes).is_err()); +} + +#[test] +fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { + for (column, row, counts) in [ + ([10, 0], [0, 20], [3, 2]), + ([-10, 0], [0, 20], [3, 2]), + ([0, 10], [20, 0], [3, 2]), + ([20, 0], [0, 10], [2, 3]), + ([20, 0], [-10, 0], [2, 3]), + ] { + let bytes = aref_fixture([0, 0], column, row, counts); + let range = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for error in [ + read_library(&bytes).unwrap_err().1, + scan_library(&bytes).unwrap_err().1, + read_selected_library(&bytes, &range).unwrap_err().1, + ] { + assert!(format!("{error:?}").contains("not an integer database-unit increment")); + } + } + + let bytes = aref_fixture([0, 0], [10, 0], [0, 20], [3, 2]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let range = [library_header().len() as u64, bytes.len() as u64 - 4]; + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + range.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + let path = + std::env::temp_dir().join(format!("pheon-fractional-aref-{}.gds", std::process::id())); + struct Cleanup(std::path::PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + let _cleanup = Cleanup(path.clone()); + std::fs::write(&path, &bytes).unwrap(); + let path = CString::new(path.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { read_path(path.as_ptr(), &mut array, &mut schema) }, + -1 + ); + let message = unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap(); + assert!(message.contains("not an integer database-unit increment")); + assert!(array.is_released()); + + // The object reviewer keeps the exact endpoints, including fractional pitches. + let mut bytes = bytes; + bytes.truncate(bytes.len() - 4); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let doc = pheon::review::Document::parse(&bytes).unwrap(); + assert_eq!( + pheon::review::Comparison::between(&doc, &doc).exit_code(), + 0 + ); +} + +#[test] +fn rectangle_detection_handles_extreme_coordinates_and_preserves_other_polygons() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + let wide = [ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MIN, + ]; + let reversed = [ + i32::MIN, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MIN, + ]; + let diagonal = [0, 0, 10, 10, 0, 10, 10, 0, 0, 0]; + let degenerate = [0, 0, 0, 0, 10, 0, 10, 0, 0, 0]; + for points in [&wide, &reversed, &diagonal, °enerate] { + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + record(&mut bytes, records::RTAG_XY, &xy(points)); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let full = read_library(&bytes).unwrap().1; + let scan = scan_library(&bytes).unwrap().1; + assert_eq!( + full, + read_selected_library(&bytes, &ranges(&scan)).unwrap().1 + ); + let cells = list_struct(&full, "cells", 0); + let rects = list_struct(&cells, "rect_batches", 0); + let values = rects + .column_by_name("rects") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + &[ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX + ] + ); + let boundaries = list_struct(&cells, "boundary_batches", 0); + let values = boundaries + .column_by_name("vertices") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + [&diagonal[..8], °enerate[..8]].concat() + ); +} + +#[test] +fn paths_preserve_missing_explicit_zero_and_negative_widths() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for width in [None, Some(0_i32), Some(-100)] { + record(&mut bytes, records::RTAG_PATH, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, records::RTAG_WIDTH, &width.to_be_bytes()); + } + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 10])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, full) = read_library(&bytes).unwrap(); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + let paths = list_struct(&cells, "paths", 0); + let widths = paths + .column_by_name("width") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + widths.iter().collect::>(), + [None, Some(0), Some(-100)] + ); +} + +fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 { + array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row) +} + +fn ranges(scan: &StructArray) -> Vec<(u64, u64)> { + let cells = list_struct(scan, "cells", 0); + (0..cells.len()) + .map(|i| { + ( + u64_value(&cells, "struct_start_offset", i), + u64_value(&cells, "struct_end_offset", i), + ) + }) + .collect() +} + +#[test] +fn scanned_ranges_reproduce_the_full_library_including_offsets() { + let bytes = fixture(); + let (_, full) = read_library(&bytes).unwrap(); + let (scan_remaining, scan) = scan_library(&bytes).unwrap(); + assert!(scan_remaining.is_empty()); + let ranges = ranges(&scan); + assert_eq!(ranges.len(), 2); + for &(start, end) in &ranges { + assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]); + assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]); + } + let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap(); + assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged. + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8); +} + +#[test] +fn scan_preserves_reference_transforms_and_skips_geometry() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let cells = list_struct(&scan, "cells", 0); + let refs = list_struct(&cells, "refs", 0); + assert_eq!(refs.len(), 2); + assert_eq!(list_struct(&cells, "refs", 1).len(), 0); + let target = refs + .column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(target.values().as_ref(), &[1, 1]); + assert_eq!( + u64_value(&refs, "xy", 0), + (3_u64 << 32) | (-4_i32 as u32 as u64) + ); + assert_eq!(u64_value(&refs, "xy0", 0), 0); + assert_eq!(u64_value(&refs, "xy1", 0), 0); + let counts = refs + .column_by_name("counts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]); + assert!( + refs.column_by_name("invert_y") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1) + ); + assert_eq!( + refs.column_by_name("scale") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + 2.0 + ); + assert_eq!( + refs.column_by_name("angle_rad") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + std::f64::consts::FRAC_PI_2 + ); + assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32); + assert_eq!(u64_value(&refs, "xy1", 1), 10); +} + +#[test] +fn selections_support_subsets_reordering_and_empty_results() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap(); + let leaf_cells = list_struct(&leaf, "cells", 0); + assert_eq!(leaf_cells.len(), 1); + let leaf_names = leaf + .column_by_name("cell_names") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!( + leaf_names + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "leaf" + ); + let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap(); + let reordered_cells = list_struct(&reordered, "cells", 0); + assert_eq!( + u64_value(&reordered_cells, "file_offset", 0), + u64_value(&leaf_cells, "file_offset", 0) + ); + let refs = list_struct(&reordered_cells, "sref_props", 1); + assert_eq!( + refs.column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + let (_, empty) = read_selected_library(&bytes, &[]).unwrap(); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); +} + +#[test] +fn id_tables_preserve_encounter_order_and_selection_local_ids() { + let mut bytes = library_header(); + for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha", 9, 4)] { + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); + if name == "zeta" { + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"alpha\0"); + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &datatype.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + } + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, full) = read_library(&bytes).unwrap(); + let cell_ranges = ranges(&scan); + + let check = |library: &StructArray, names: &[&str], layers: &[u32], ids: &[u32]| { + let values = |name| { + library + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + }; + let name_values = values("cell_names"); + assert_eq!( + name_values + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(Option::unwrap) + .collect::>(), + names + ); + let cells = list_struct(library, "cells", 0); + assert_eq!( + cells + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + ids + ); + if library.column_by_name("layers").is_some() { + let layer_values = values("layers"); + assert_eq!( + layer_values + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + layers + ); + } + }; + let layer_a = (9 << 16) | 4; + let layer_b = (2 << 16) | 8; + check( + &full, + &["zeta", "alpha", "middle"], + &[layer_a, layer_b], + &[0, 2, 1], + ); + check(&scan, &["zeta", "alpha", "middle"], &[], &[0, 2, 1]); + let (_, all_selected) = read_selected_library(&bytes, &cell_ranges).unwrap(); + assert_eq!(all_selected, full); + let (_, subset) = read_selected_library(&bytes, &cell_ranges[1..]).unwrap(); + check(&subset, &["middle", "alpha"], &[layer_b, layer_a], &[0, 1]); + let (_, reversed) = + read_selected_library(&bytes, &[cell_ranges[2], cell_ranges[1], cell_ranges[0]]).unwrap(); + check( + &reversed, + &["alpha", "middle", "zeta"], + &[layer_a, layer_b], + &[0, 1, 2], + ); +} + +#[test] +fn streamed_and_buffered_properties_preserve_duplicates_order_and_empty_lists() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for (kind, properties) in [ + (records::RTAG_BOUNDARY, true), + (records::RTAG_PATH, true), + (records::RTAG_PATH, false), + ] { + record(&mut bytes, kind, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + let vertices = if kind == records::RTAG_BOUNDARY { + xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]) + } else { + xy(&[0, 0, 10, 10]) + }; + record(&mut bytes, records::RTAG_XY, &vertices); + if properties { + for (key, value) in [(7_i16, b"first\0".as_slice()), (7, b"second"), (9, b"")] { + record(&mut bytes, records::RTAG_PROPATTR, &key.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, value); + } + } + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + + let (remaining, full) = read_library(&bytes).unwrap(); + assert!(remaining.is_empty()); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(selected, full); + let cells = list_struct(&full, "cells", 0); + let boundaries = list_struct(&cells, "boundary_props", 0); + let paths = list_struct(&cells, "paths", 0); + assert_eq!(boundaries.len(), 1); + assert_eq!(paths.len(), 2); + for elements in [&boundaries, &paths] { + let properties = list_struct(elements, "properties", 0); + let keys = properties + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(keys.values().as_ref(), &[7, 7, 9]); + let values = properties + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + values.iter().collect::>(), + [Some("first"), Some("second"), Some("")] + ); + } + let empty_properties = paths + .column_by_name("properties") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!empty_properties.is_null(1)); + assert_eq!(empty_properties.value_length(1), 0); +} + +#[test] +fn selected_ranges_must_cover_exactly_one_complete_cell() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (start, end) = ranges[0]; + for range in [ + (end, start), + (start, u64::MAX), + (start, end - 1), + (start + 4, end), + (start, end + 4), + (start, ranges[1].1), + ] { + assert!( + read_selected_library(&bytes, &[range]).is_err(), + "accepted {range:?}" + ); + } +} + +#[test] +fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + 0 + ); + let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); + + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + 0 + ); + let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + let ranges: Vec = ranges(&scan) + .into_iter() + .flat_map(|(a, b)| [a, b]) + .collect(); + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + ranges.len() / 2, + &mut array, + &mut schema, + ) + }, + 0 + ); + let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(selected, read_library(&bytes).unwrap().1); + assert!(last_error_message().is_null()); +} + +#[test] +fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("scan failed") + ); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) }, + -1 + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("ranges pointer was null") + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + usize::MAX, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("range count is too large") + ); + assert!(array.is_released()); +} diff --git a/tests/writing.rs b/tests/writing.rs new file mode 100644 index 0000000..caf8816 --- /dev/null +++ b/tests/writing.rs @@ -0,0 +1,138 @@ +use arrow_array::Array; +use pheon::{ + basic::{pack_bitarray, pack_int2, pack_int4, pack_real8}, + library::{FileHeader, read_library}, + record::{Int2Array, Int4Array, Record, RecordData, RecordHeader}, + records, +}; +use std::io::{self, Write}; + +#[test] +fn record_headers_include_their_own_size() { + let mut bytes = Vec::new(); + assert_eq!(records::HEADER::write(&mut bytes, &600).unwrap(), 6); + assert_eq!(bytes, [0, 6, 0, 2, 2, 88]); + let (remaining, value) = records::HEADER::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(value, 600); + let maximum = RecordHeader { + data_size: u16::MAX - 4, + tag: 0x1003, + }; + assert_eq!(maximum.pack_into(), [0xff, 0xff, 0x10, 0x03]); +} + +#[test] +fn integer_array_records_round_trip() { + let mut bytes = Vec::new(); + records::COLROW::write(&mut bytes, &[2, 3]).unwrap(); + let (remaining, values) = records::COLROW::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(values.collect::>(), [2, 3]); + + let mut bytes = Vec::new(); + let values = [i32::MIN, -1, 0, i32::MAX]; + records::XY::write(&mut bytes, &values).unwrap(); + let (remaining, parsed) = records::XY::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed.collect::>(), values); +} + +#[test] +fn scalar_packers_write_only_their_prefix_and_check_short_buffers() { + type Case = (fn(&mut [u8]), &'static [u8]); + let cases: &[Case] = &[ + (|bytes| pack_bitarray(bytes, &[true; 16]), &[0xff, 0xff]), + (|bytes| pack_int2(bytes, -2), &[0xff, 0xfe]), + (|bytes| pack_int4(bytes, -2), &[0xff, 0xff, 0xff, 0xfe]), + ( + |bytes| pack_real8(bytes, 1.0).unwrap(), + &[0x41, 0x10, 0, 0, 0, 0, 0, 0], + ), + ]; + for (pack, expected) in cases { + let mut bytes = [0xa5; 16]; + pack(&mut bytes); + assert_eq!(&bytes[..expected.len()], *expected); + assert!(bytes[expected.len()..].iter().all(|&byte| byte == 0xa5)); + let mut short = vec![0xa5; expected.len() - 1]; + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| pack(&mut short))).is_err() + ); + assert!(short.iter().all(|&byte| byte == 0xa5)); + } + // Invalid REAL8 values are rejected before accessing the output buffer. + assert!(pack_real8(&mut [], f64::INFINITY).is_err()); +} + +#[test] +fn array_packers_reject_nonexact_buffers_before_writing() { + for width in [2, 4] { + let required = 2 * width; + for size in [required - 1, required + 1, required + width] { + let mut bytes = vec![0xa5; size]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if width == 2 { + Int2Array::pack_into(&mut bytes, &[-1, 1]); + } else { + Int4Array::pack_into(&mut bytes, &[-1, 1]); + } + })); + assert!(result.is_err()); + assert!(bytes.iter().all(|&byte| byte == 0xa5)); + } + } +} + +#[test] +fn string_records_pad_only_when_needed_and_support_empty_strings() { + for value in [b"".as_slice(), b"odd", b"even"] { + let mut bytes = Vec::new(); + let expected_size = 4 + value.len() + value.len() % 2; + assert_eq!( + records::LIBNAME::write(&mut bytes, value).unwrap(), + expected_size + ); + assert_eq!(bytes.len(), expected_size); + let (remaining, parsed) = records::LIBNAME::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed, value); + if value.len() % 2 == 1 { + assert_eq!(bytes.last(), Some(&0)); + } + } +} + +struct ShortWriter(Vec); + +impl Write for ShortWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let size = bytes.len().min(1); + self.0.extend_from_slice(&bytes[..size]); + Ok(size) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn library_header_round_trips_through_a_short_writer() { + let header = FileHeader::new(b"odd", 1e-9, 0.001); + let mut writer = ShortWriter(Vec::new()); + let mut written = header.write(&mut writer).unwrap(); + written += records::ENDLIB::write(&mut writer, &()).unwrap(); + assert_eq!(written, writer.0.len()); + let (remaining, library) = read_library(&writer.0).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(library.len(), 1); +} + +#[test] +fn oversized_records_fail_before_writing() { + let mut bytes = Vec::new(); + let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(bytes.is_empty()); +} diff --git a/tools/check_installed.py b/tools/check_installed.py new file mode 100644 index 0000000..707ecbf --- /dev/null +++ b/tools/check_installed.py @@ -0,0 +1,26 @@ +"""Run outside the checkout, without PHEON_LIBRARY, after installing a wheel.""" +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +# A separate interpreter verifies that base packing does not import PyArrow. +subprocess.run( + [sys.executable, "-I", "-c", """ +import importlib.util +import sys +import pheon +from pheon._native import library +assert 'pyarrow' not in sys.modules +assert importlib.util.find_spec('klamath_rs_ext') is None +assert library.parent == __import__('pathlib').Path(pheon.__file__).parent +assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') +"""], + check=True, +) +suite = unittest.defaultTestLoader.discover(str(ROOT / "tests/python")) +if not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(): + raise SystemExit(1)