diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2d14a76 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: Native checks + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + rust: + runs-on: ubuntu-latest + strategy: + matrix: + rust: ['1.88.0', stable] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: rustup toolchain install ${{ matrix.rust }} --profile minimal + - run: cargo +${{ matrix.rust }} test --locked + - run: cargo +${{ matrix.rust }} test --locked --release diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..321f3c6 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,122 @@ +name: Package artifacts + +on: + pull_request: + workflow_dispatch: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - run: python -m pip install build + - run: python -m build --sdist + - uses: actions/upload-artifact@v4 + with: + name: source + path: dist/*.tar.gz + + wheels: + needs: sdist + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: source + path: source-dist + - name: Unpack the source release + shell: bash + run: | + mkdir source-build + tar -xzf source-dist/*.tar.gz -C source-build --strip-components=1 + - name: Build and repair Python-independent wheels from the sdist + uses: pypa/cibuildwheel@v4.3.5 + with: + package-dir: source-build + env: + CIBW_BUILD: cp311-* + CIBW_SKIP: '*-musllinux_* *-win32' + CIBW_ARCHS: native + CIBW_ENVIRONMENT_LINUX: 'CARGO_HOME=/tmp/pheon-cargo RUSTUP_HOME=/tmp/pheon-rustup PATH=/tmp/pheon-cargo/bin:$PATH' + CIBW_BEFORE_ALL_LINUX: 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable' + CIBW_TEST_COMMAND: 'python -I {project}/tools/check_installed.py' + - name: Check wheel ABI tags + shell: python + run: | + from pathlib import Path + wheels = list(Path('wheelhouse').glob('*.whl')) + assert wheels + assert all('-py3-none-' in p.name for p in wheels), wheels + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse/*.whl + + same-wheel: + needs: wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + python: ['3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v4 + with: + name: wheel-${{ matrix.os }} + path: wheelhouse + - name: Install the same artifact on another Python version + shell: bash + run: python -m pip install wheelhouse/*.whl + - run: python -I tools/check_installed.py + + cli: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: cargo test --release --locked + - run: cargo build --release --locked --bin pheon + - name: Assemble standalone CLI artifact + shell: bash + run: | + mkdir cli-dist + if [ "$RUNNER_OS" = Windows ]; then + cp target/release/pheon.exe cli-dist/ + else + cp target/release/pheon cli-dist/ + fi + cp LICENSE.md README.md cli-dist/ + cp -R contrib docs cli-dist/ + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.os }} + path: cli-dist/ diff --git a/.gitignore b/.gitignore index 5d80173..c2a3759 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ /target/ **/*.rs.bk -Cargo.lock *.swp *.swo + +__pycache__/ +/build/ +/dist/ +/.cache/ +/.venv/ +/*.egg-info/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..cfec364 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,664 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link 0.1.1", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pheon" +version = "0.3.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "flate2", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.1", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link 0.1.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link 0.1.1", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index a211306..93004b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,28 @@ [package] -name = "klamath_rs_ext" -version = "0.2.0" +name = "pheon" +version = "0.3.0" authors = ["jan "] -edition = "2021" +edition = "2024" +rust-version = "1.88" +description = "GDSII parsing, Arrow interchange, and structural layout review" +license = "GPL-3.0-only" +readme = "README.md" +repository = "https://mpxd.net/code/jan/pheon" [lib] -name = "klamath_rs_ext" +name = "pheon" crate-type = ["cdylib", "rlib"] [dependencies] -byteorder = "^1" +arrow-array = {version = "^54", features = ["ffi"]} +arrow-schema = "^54" +flate2 = "1" +serde = "1" +serde_json = "1" +termcolor = "1.4" + + +[profile.release] +lto = true diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..175443c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,595 @@ +GNU General Public License +========================== + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +* **a)** The work must carry prominent notices stating that you modified it, and giving a +relevant date. +* **b)** The work must carry prominent notices stating that it is released under this +License and any conditions added under section 7. This requirement modifies the +requirement in section 4 to “keep intact all notices”. +* **c)** You must license the entire work, as a whole, under this License to anyone who +comes into possession of a copy. This License will therefore apply, along with any +applicable section 7 additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no permission to license the +work in any other way, but it does not invalidate such permission if you have +separately received it. +* **d)** If the work has interactive user interfaces, each must display Appropriate Legal +Notices; however, if the Program has interactive interfaces that do not display +Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +* **a)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by the Corresponding Source fixed on a +durable physical medium customarily used for software interchange. +* **b)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by a written offer, valid for at least +three years and valid for as long as you offer spare parts or customer support for +that product model, to give anyone who possesses the object code either **(1)** a copy of +the Corresponding Source for all the software in the product that is covered by this +License, on a durable physical medium customarily used for software interchange, for +a price no more than your reasonable cost of physically performing this conveying of +source, or **(2)** access to copy the Corresponding Source from a network server at no +charge. +* **c)** Convey individual copies of the object code with a copy of the written offer to +provide the Corresponding Source. This alternative is allowed only occasionally and +noncommercially, and only if you received the object code with such an offer, in +accord with subsection 6b. +* **d)** Convey the object code by offering access from a designated place (gratis or for +a charge), and offer equivalent access to the Corresponding Source in the same way +through the same place at no further charge. You need not require recipients to copy +the Corresponding Source along with the object code. If the place to copy the object +code is a network server, the Corresponding Source may be on a different server +(operated by you or a third party) that supports equivalent copying facilities, +provided you maintain clear directions next to the object code saying where to find +the Corresponding Source. Regardless of what server hosts the Corresponding Source, +you remain obligated to ensure that it is available for as long as needed to satisfy +these requirements. +* **e)** Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are being +offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +* **a)** Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or +* **b)** Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed by works +containing it; or +* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that +modified versions of such material be marked in reasonable ways as different from the +original version; or +* **d)** Limiting the use for publicity purposes of names of licensors or authors of the +material; or +* **e)** Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or +* **f)** Requiring indemnification of licensors and authors of that material by anyone +who conveys the material (or modified versions of it) with contractual assumptions of +liability to the recipient, for any liability that these contractual assumptions +directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ff57a3d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include Cargo.toml Cargo.lock rust-toolchain.toml README.md LICENSE.md +recursive-include src *.rs +recursive-include pheon *.py *.h py.typed +recursive-include tests *.rs *.py *.json *.txt *.mmd +recursive-include docs *.md *.json +recursive-include contrib * +recursive-include tools *.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..eaf6d3c --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# Pheon + +Pheon reads GDSII layouts into Apache Arrow and provides command-line tools for +inspecting and comparing layout revisions. It is written in Rust, with Python +bindings and a C API. + +- Inspect cells, records, and reference hierarchies. +- Compare layout objects, with text or JSON reports and Git integration. +- Read geometry into Arrow, or scan the hierarchy before loading selected cells. + +Comparisons ignore timestamps and element order while preserving duplicates and +other object details. They compare object fields and coordinates exactly; see the +[comparison rules][comparison] for supported normalization and limits. + +## Try the command-line tools + +Install the CLI [from crates.io][crate] with Rust 1.88 or newer. +Use your own GDSII paths in the examples. + +```sh +cargo install pheon --locked +pheon dump chip.gds +pheon diff before.gds after.gds +pheon hierarchy chip.gds --format mermaid +``` + +`diff` returns 0 for equality, 1 for differences, and 2 for invalid or incomplete +comparisons. Start with the [CLI guide][cli], then set up +[Git integration][git] if you review layouts in a repository. + +## Read layouts in Python + +Install the Python package [from PyPI][pypi] with Python 3.11 or newer. + +```sh +python -m pip install pheon +``` + +Run this example from a directory containing `chip.gds`. + +```python +from pheon import arrow + +layout = arrow.read_path("chip.gds") +print(layout.field("lib_name")[0].as_py()) +``` + +The Python package installs PyArrow. See the [Python guide][python] for source +installs and selected reads, and the [Arrow reference][arrow] for the returned data. + +## Learn more + +Browse the [documentation][docs] for layer names, native integration, +JSON reports, development, and releases. For writing, Pheon provides low-level +GDSII record helpers in Rust. + +Licensed under [GPL-3.0-only][license]. + +[arrow]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/arrow-schema.md +[cli]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/cli.md +[comparison]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/comparison.md +[crate]: https://crates.io/crates/pheon +[docs]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/index.md +[git]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/git.md +[license]: https://mpxd.net/code/jan/pheon/src/branch/master/LICENSE.md +[pypi]: https://pypi.org/project/pheon/ +[python]: https://mpxd.net/code/jan/pheon/src/branch/master/docs/python.md diff --git a/contrib/gds-corpus.toml b/contrib/gds-corpus.toml new file mode 100644 index 0000000..25eeefb --- /dev/null +++ b/contrib/gds-corpus.toml @@ -0,0 +1,71 @@ +# Pinned GDSII test inputs. Each digest covers the original upstream bytes. + +[gdstk] +repo = "heitzmann/gdstk" +revision = "1099be134da34d4afd9e5fb1f4f0dbae9ef4c17a" +license = "BSL-1.0" +license_sha256 = "beb8e42e9d6b4284e03304d05a81a0755200a965fc8d0a5e0aea1e84cf805d6e" + +[gdstk.layouts] +"tests/proof_lib.gds" = "af7b95ed191ebe32438c0f0cb5ed824ba2a98673cfa9ce6f1a9ff5a827c3dbf7" + +[gdsfactory] +repo = "gdsfactory/gdsfactory" +revision = "7407836dd5d872c704fc06f38562a2ffc88049dc" +license = "MIT" +license_sha256 = "cdace89991b496bd052909790d00ece97feab566afd964d0883d2c0605462b5b" + +[gdsfactory.layouts] +"tests/gds/alphabet_3top_cells.gds" = "09b75d5bfeb0051e4972a4d9fc26df5d342604f7ed2a0994376cd7c77e929019" +"tests/gds/big_rect.gds" = "de71342365dc4bd84dbb03307537b7ca2b4dffab98efdf88713cc451e44e6ac2" +"tests/gds/mmi1x2.gds" = "e12e81a9d5b9dc8463cce05915f567bc7174f848a52c7e092dc8a602819dafa0" +"tests/gds/mzi2x2.gds" = "49499f13c888de137eb26342c45ea379d9ea2ecd82fc990780d38fa014002d38" +"tests/gds/small_rect.gds" = "345480096d17bc724ae32cc0f42bd915a39c6e64806b43054d687755afd5faba" +"tests/gds/straight.gds" = "dc5e971b5be91864895870ef70a5ae959bd52dab5971caa0dcfc9aa6e13fa40d" + +[klayout] +repo = "KLayout/klayout" +revision = "53911acd6dfaa113c188d11d20b2ae290aa60a6b" +license = "GPL-3.0 (upstream LICENSE)" +license_sha256 = "589ed823e9a84c56feb95ac58e7cf384626b9cbf4fda2a907bc36e103de1bad2" + +[klayout.layouts] +"samples/lvs/ringo.gds" = "f24d4d11ac7941021d0b0c15d8b32ced1c6735cbd4da68de1482830ca45be3bd" +"testdata/drc/texts.gds" = "ee8ddf7de65ebccf9772fd98cb5d45313e353f8a3455d162611320885061f907" +"testdata/algo/deep_texts_l1.gds" = "253d24491a097102a729ef7f101c9b7acf7684c4a90c16d212ac91d9087e3b4d" +"testdata/gds/arefs.gds" = "fadde330ade22d0e8b92e4040377dcccb6e7a572d9d69df6b3fa722a2bfbcdcd" +"testdata/gds/arefs_skew.gds" = "a681a47cd1d2d74b881d0ab10c3d6fd6c9674c819cf39c077936a4328511b1ae" +"testdata/gds/arefs_skew2.gds" = "363dd2650b4a3011cbe33a6d8febccc77b22dd9b7563f087029a1c5afbe2ae94" +"testdata/gds/basic_instances.gds" = "6ccfffd590c159a5ab450c8eed826afb8e1d68c9ae2f85ec089ab21c0b40ec1b" +"testdata/gds/issue-1353.gds" = "f0e9cbe2b0582d766b3e6e1737617865ca5e74dc525f7bc2872e09ffe6a82482" +"testdata/gds/issue_1835.gds" = "b2d78e8ed4f43e63738857c7aa81606c8ccc5ca0910e3d4e4569c058415d9171" +"testdata/gds/issue_893.gds" = "30a2bf58ea8ae5b36f6df67c5d5221bc257e916b985bc1e7a74cf19ba244eff8" +"testdata/gds/issue_893_au.gds" = "e33cbc5ae2a3f3e9779f6c1cbaf89762c33644c76836c96445e887820dd4eab6" +"testdata/gds/t166_au.gds.gz" = "bd5f0eda73cf13daae77d61eb2a2d9376f2752e66cd458307cfee0e4d5b5ea57" + +[sky130] +repo = "google/skywater-pdk-libs-sky130_fd_sc_hd" +revision = "ac7fb61f06e6470b94e8afdf7c25268f62fbd7b1" +license = "Apache-2.0" +license_sha256 = "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + +[sky130.layouts] +"cells/inv/sky130_fd_sc_hd__inv_1.gds" = "18a9cfa399957c7c7b5fc75175e03fc921d56ecc6b8de8fae34ba301bb7914c4" +"cells/inv/sky130_fd_sc_hd__inv_4.gds" = "1e0fb091a3e4583297e7124d68542e4f29ee6d1f711e80dcb41fc1ed8465c0ce" +"cells/nand2/sky130_fd_sc_hd__nand2_1.gds" = "4e7fa6255df901f59151791d6e8ff4c421b63c1ea3822d6d5d25b48221782a3a" +"cells/dfxtp/sky130_fd_sc_hd__dfxtp_1.gds" = "2b0eaaeedd5b7b2c359942098aee0cef69387108e02e10bc09940680e4cc7ec3" +"cells/tapvpwrvgnd/sky130_fd_sc_hd__tapvpwrvgnd_1.gds" = "642f8841345c7c09ec910fe7893ae803ab760606adfab65229e2aedb055005a5" +"cells/fill/sky130_fd_sc_hd__fill_1.gds" = "9f718b6252a027fc769c2e4298c182e1c4ee3b160fefcd6acf7d947d5c6c3453" + +[ihp] +repo = "IHP-GmbH/IHP-Open-PDK" +revision = "e73adb4045731ddec21634146d536380057f6a26" +license = "Apache-2.0" +license_sha256 = "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + +[ihp.layouts] +"ihp-sg13g2/libs.ref/sg13g2_pr/gds/sg13g2_pr.gds" = "3c640413b455354da11ec4db01c25f165489bcfd4cdec0f79559a9f78331ab42" +"ihp-sg13g2/libs.ref/sg13g2_stdcell/gds/sg13g2_stdcell.gds" = "ab9f2802c78935d4463093c24860f9d3f244057a839179fb19fc0243a18cf3f4" +"ihp-sg13g2/libs.ref/sg13g2_sram/gds/RM_IHPSG13_1P_1024x16_c2_bm_bist.gds" = "1ed0af3f9cc3a3455f4c0beb08394502a875dda3c54bc03123357e397b26e929" +"ihp-sg13g2/libs.qa/drc/devices/sg13g2_qacells.gds" = "21d8b2663ba15fa4c99752ad7a030f486043ba17950236bee8dc483e64c948b2" +"ihp-sg13g2/libs.qa/drc/special/sg13g2_stdcell_abutment.gds" = "e125bd8e6bf8976f41bf29fb761c28559064b50571289da98c148684676e1728" diff --git a/contrib/git/attributes b/contrib/git/attributes new file mode 100644 index 0000000..ed817cc --- /dev/null +++ b/contrib/git/attributes @@ -0,0 +1,5 @@ +# Add these lines to the layout repository's .gitattributes. +*.gds diff=pheon + +# For canonical textconv instead, replace the line above with: +# *.gds diff=pheon-text diff --git a/contrib/git/config b/contrib/git/config new file mode 100644 index 0000000..ca84adf --- /dev/null +++ b/contrib/git/config @@ -0,0 +1,10 @@ +# Include this file from Git config after installing pheon on PATH. +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false + +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true diff --git a/contrib/wheels/manylinux-image.txt b/contrib/wheels/manylinux-image.txt new file mode 100644 index 0000000..a8468dd --- /dev/null +++ b/contrib/wheels/manylinux-image.txt @@ -0,0 +1 @@ +quay.io/pypa/manylinux_2_28_x86_64@sha256:e0adf18521ad35192df733a6c62d35df7801957d24e0470238aacecfab2188a3 diff --git a/docs/arrow-schema.md b/docs/arrow-schema.md new file mode 100644 index 0000000..a0abd94 --- /dev/null +++ b/docs/arrow-schema.md @@ -0,0 +1,133 @@ +# Arrow layout reference + +This reference is for consumers of Pheon's Arrow readers. Each result is a +one-row `StructArray` representing a GDSII library. The +[Python guide](python.md) shows how to obtain and access it. + +Geometry fields named `layer` are **indices into the result's `layers` table**. +Reference `target` and cell `id` values index its `cell_names` table. Resolve IDs +using tables from the same result; a selected read can assign different IDs. + +`pheon_arrow_schema_version()` currently returns 1 for all layouts below. +Consumers of the [C API](native-api.md) must check both this application schema +version and the separate C API version. + +## Full library + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `meters_per_db_unit` | Float64 | Database unit size in meters. | +| `user_units_per_db_unit` | Float64 | User units per database unit. | +| `lib_name` | Utf8 | Library name. | +| `mod_time`, `acc_time` | FixedSizeList(Int16, 6) | Modification/access time: year, month, day, hour, minute, second. | +| `cells` | List(Cell) | Cell records. | +| `cell_names` | List(Utf8) | Names indexed by cell ID, including targets without cell definitions. | +| `layers` | List(UInt32) | Packed layer/type pairs indexed by layer ID. | + +### Cell + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into `cell_names`. | +| `file_offset` | UInt64 | Byte offset immediately after STRNAME, before the first element or ENDSTR. | +| `srefs`, `arefs` | List(SRef), List(ARef) | References without properties. | +| `sref_props`, `aref_props` | List(SRefProp), List(ARefProp) | References with properties. | +| `rect_batches` | List(RectBatch) | Optimized rectangular boundaries. | +| `boundary_batches` | List(BoundaryBatch) | Other boundaries without properties. | +| `boundary_props` | List(BoundaryProp) | Boundaries with properties. | +| `paths` | List(Path) | Paths. | +| `nodes`, `boxes` | List(BoxNode) | Node and box elements. | +| `texts` | List(Text) | Text labels. | + +## Hierarchy scan and selected reads + +Scans return the same metadata and `cell_names`, omit `layers`, and replace +`cells` with these records: + +| Field | Arrow type | Meaning | +| --- | --- | --- | +| `id` | UInt32 | Index into this result's `cell_names`. | +| `struct_start_offset` | UInt64 | Inclusive offset of BGNSTR. | +| `struct_end_offset` | UInt64 | Exclusive offset after ENDSTR. | +| `refs` | List(ScanRef) | References sorted by target ID, with properties omitted. | + +ScanRef has ARef's fields below. An SREF uses zero basis vectors and counts +`(1, 1)`. Geometry and text are skipped without decoding. + +Pass complete half-open cell ranges to `read_selected_library` in Rust or +`read_cells_bytes` in Python/C. The result uses the full library layout, with +cells in requested order. An empty selection returns metadata and empty lists. +Offsets always refer to the original uncompressed input. The Rust selected +reader returns the original input slice as its remainder. + +## Packed values + +| Value | Storage | Decoding | +| --- | --- | --- | +| Layer/type pair in `layers` | UInt32 | Layer in high 16 bits; datatype/texttype/boxtype/nodetype in low 16 bits. | +| Point or vector | UInt64 | Signed Int32 x in high 32 bits, signed Int32 y in low 32 bits. | +| Array counts | UInt32 | Columns in high 16 bits, rows in low 16 bits. | + +Layer halves preserve the original 16-bit patterns; reinterpret them as signed +Int16 where an integration uses signed GDS layer values. Array counts originate +from positive Int16 values. Flattened coordinate lists below use `[x, y, ...]` +in integer database units. + +## References + +| SRef field | Arrow type | Meaning | +| --- | --- | --- | +| `target` | UInt32 | Target cell ID. | +| `invert_y` | Boolean | Reflection flag. | +| `scale` | Float64 | Magnification. | +| `angle_rad` | Float64 | Rotation in radians. | +| `xy` | UInt64 | Packed origin. | + +ARef adds `xy0` and `xy1` (UInt64 packed column/row increments) and `counts` +(UInt32). SRefProp and ARefProp add `properties: List(Property)`. + +Array increments must be exact signed 32-bit integers. All Arrow readers reject +fractional or out-of-range endpoint/count division rather than rounding it. +One-placement AREFs can be normalized to SREFs. The review CLI retains original +endpoints and distinguishes SREF from AREF independently of this representation. + +## Geometry and text + +Every `layer` below is a UInt32 **table index**, not a packed pair. + +| Type | Fields beyond `layer` | +| --- | --- | +| RectBatch | `rects: List(Int32)`, flattened `[min_x, min_y, max_x, max_y]` rectangles. | +| BoundaryBatch | `vertices: List(Int32)`; `vertex_offsets: List(UInt32)` gives each boundary's start in points, not scalar coordinates. | +| BoundaryProp | `vertices: List(Int32)`; `properties: List(Property)`. | +| BoxNode | `xy: List(Int32)`; `properties: List(Property)`. | +| Path | Fields below. | +| Text | Fields below. | + +| Path field | Arrow type | Meaning | +| --- | --- | --- | +| `path_type` | Int16, nullable | Stored path type. | +| `extension_start`, `extension_end` | Int32, nullable | End extensions. | +| `width` | Int32, nullable | Null when WIDTH is absent. | +| `xy` | List(Int32) | Centerline coordinates. | +| `properties` | List(Property) | Element properties. | + +| Text field | Arrow type | Meaning | +| --- | --- | --- | +| `presentation_horiz`, `presentation_vert`, `presentation_font` | UInt8, nullable | Presentation attributes. | +| `path_type` | Int16, nullable | Stored path type. | +| `width` | Int32, nullable | Stored width. | +| `invert_y` | Boolean, nullable | Reflection flag. | +| `mag` | Float64, nullable | Magnification. | +| `angle_deg` | Float64, nullable | Rotation in degrees. | +| `xy` | UInt64 | Packed position. | +| `string` | Utf8 | Label text. | +| `properties` | List(Property) | Element properties. | + +Property contains `key: Int16` and `value: Utf8`. The reader preserves property +order and duplicates. Property-list fields are nullable in the schema; the +reader normally emits valid lists, including empty ones. + +Geometry batching and reference normalization mean this layout is not a verbatim +record stream; use the [CLI's record dump](cli.md#inspect-a-layout) when original +records matter. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..2d66ea6 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,151 @@ +# Command-line tools + +Use `pheon` to inspect a GDSII file, compare revisions, or trace cell references. +The CLI recognizes gzip-compressed GDSII by its file header. Reported byte offsets +refer to the decompressed layout. The CLI runs independently of Python. + +## Install + +Install the CLI [from crates.io](https://crates.io/crates/pheon) with Rust 1.88 +or newer. + +```sh +cargo install pheon --locked +pheon --help +``` + +From a source checkout, install with `cargo install --path . --locked --bin pheon` +or build with `cargo build --release --locked --bin pheon` and run +`target/release/pheon` (`pheon.exe` on Windows). + +## Inspect a layout + +```sh +pheon dump chip.gds +pheon dump chip.gds --records +pheon hierarchy chip.gds --format ascii --depth 4 +pheon hierarchy chip.gds --format mermaid --top CHIP +``` + +`dump` lists objects by cell. A prefix such as `3x` counts copies of the same +normalized object within a cell. Coordinates are in database units. Rectangles +show two corners, and longer vertex lists wrap across lines. References show +the target and any array dimensions before the coordinates. Add `--records` +to inspect the original record sequence, including timestamps. In color +output, the object dump highlights cell headings and layer numbers, and the +record dump highlights record names. + +`hierarchy` shows direct placement counts on references. `[subtree elided]` +marks a repeated cell whose children were skipped. Repeated leaves show +each parent's placement count without the marker. At the depth limit, +`[children omitted]` marks a cell with further descendants. Mermaid +output is diagram source for a Mermaid renderer. + +## Compare revisions + +Run `pheon diff before.gds after.gds` with your own files, or download these +small layouts from the pinned test corpus to try a comparison. +[small_rect.gds][small] · [big_rect.gds][big] + +```sh +pheon diff small_rect.gds big_rect.gds +``` + +The example exits with code 1 and shows one changed rectangle in `TOP`. + +```text +Different +Cells: +* TOP + 2/0: 1 rectangle changed +``` + +| Exit code | Meaning | +| --- | --- | +| 0 | Equal under the comparison rules and selected scope. | +| 1 | Differences found in a complete comparison. | +| 2 | Invalid input, a command error, or an incomplete comparison. | + +A code of 1 is an expected comparison result. A code of 2 can accompany definite +differences, but equality cannot be established. Read the +[comparison rules](comparison.md) before relying on results in automation, and +the [JSON reference](diff-reports.md) to consume reports programmatically. + +### Read the text report + +The cell list marks additions with `+`, removals with `-`, and modifications +with `*`. Modified cells summarize shape changes by layer and repeated +references by target. A single move can show its displacement in database units. +Layer names appear inline or in a legend. Object rows use `~` for likely +replacements and `+` or `-` for unmatched additions and removals. + +Use `--details` for individual objects, `--hierarchy ascii` for affected +parents, or `--format json` for machine-readable evidence. + +`diff` and `dump` color supported terminals by default. Set `NO_COLOR`, use +`TERM=dumb`, or pass `--color=never` for plain text. Pass `--color=always` when +piping into `less -R`. JSON and Mermaid diagram source stay plain. + +## Choose the comparison scope + +```sh +pheon diff before.gds after.gds --top CHIP --layer 1/0 +pheon diff before.gds after.gds --cell RESONATOR +``` + +| Option | Compared content | +| --- | --- | +| `--top NAME` | The named cell and its reachable descendants in each revision. | +| `--cell NAME` | One cell, including its references, without its descendants. | +| `--layer L/T\|NAME` | Objects on the selected layer/type pair, plus references leading to that layer. | + +Selectors combine: `--top CHIP --cell RESONATOR` selects that cell within CHIP's +subtree. A name absent from both revisions is an error; presence on only one +side is an addition or deletion. Layer selection omits cells irrelevant to that +layer, but library fields and metadata-version diagnostics remain relevant. +The complete input is parsed before scope selection, so malformed records +outside the scope still fail the command. + +Both `--layer` and `--show-layer` accept exact, case-sensitive names from the +embedded metadata in either revision. If a name refers to several numeric pairs, +Pheon lists the choices. A valid numeric pair always selects that pair directly. + +## Control report detail + +```sh +pheon diff before.gds after.gds --show-cell RESONATOR --show-layer WG +pheon diff before.gds after.gds --show-cell RESONATOR --details --max-changes 20 +pheon diff before.gds after.gds --hierarchy mermaid --depth 4 +``` + +`--show-cell` and `--show-layer` filter displayed changes without changing the +comparison, totals, or exit status. In text reports, `--show-layer` filters +object changes within modified cells and hides references without a numeric +layer. Added and removed cells appear as name-only entries outside the object +budget. Use `--show-cell` to filter those entries. + +`--max-changes N` limits displayed object occurrences after filtering. Text +defaults to 100 occurrences and shows a replacement only when both objects fit +within the limit. Grouped rows count each object toward the limit. When a +layer filter selects one side of a layer-only replacement, that side appears as +an addition or removal. The omission line lists changes hidden by filters or the +limit. JSON has no default object limit and records omissions in +`report_selection`. See the [JSON reference](diff-reports.md) for the selection +counters. + +`--details` shows individual object evidence inside modified cells in text +reports. Replacement rows include a nested `-` row for the old object and a +`+` row for the new object, with cell-local coordinates in database units, +retained fields, and properties. + +The diff hierarchy shows changed cells, affected ancestors, and modified +references from both revisions. Counts are direct placements, including +symbolic array counts. ASCII folds straight chains of unchanged edges and can +omit terminal edges already explained by the cell list. `--depth` counts the +original edges. For the full graph within the comparison scope, use +`--format json --hierarchy graph` without `--depth`. + +Next: configure [Git integration](git.md) or add [layer names](layer-names.md). + +[small]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/small_rect.gds +[big]: https://raw.githubusercontent.com/gdsfactory/gdsfactory/7407836dd5d872c704fc06f38562a2ffc88049dc/tests/gds/big_rect.gds diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 0000000..739fab3 --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,59 @@ +# Comparison rules + +Pheon compares GDSII objects using the `pheon-structural-v1` profile. Use it to +review exact layout changes while ignoring incidental ordering and timestamps. +The [CLI guide](cli.md) explains how to choose cells and layers. + +## What counts as equal + +Cells match by name and layers by numeric layer/type pair. By default, the +comparison includes the entire library. + +Pheon ignores library and cell timestamps, cell order, element order, and +property order. Boundary and box rings are normalized for their starting vertex +and winding direction. Other vertices remain significant. Repeated objects and +properties are compared as multisets, so duplicates still count. + +Other retained details compare exactly: paths, labels, reference transforms, +array endpoints and counts, properties, and flags. An explicitly stored default +is distinct from an omitted field. An SREF is distinct from a one-element AREF. +REAL8 encodings are retained, so differently encoded values can compare as +changed even when their decoded values agree. + +Library fields, including UNITS, compare separately. Coordinates are not +rescaled to a shared grid. Cell renames are additions and deletions; Pheon does +not infer identity from similar content. It does not perform geometric Boolean +operations, XOR, or tolerance-based matching. + +## Read change explanations + +Pheon first pairs unmatched polygons, boxes, and paths that differ only in +layer or type, with exactly one candidate in each revision. It then checks the +remaining objects for unique matches by kind, layer, and bounds, followed by +kind and layer. Reference matching uses the target cell. Ambiguous candidates +are reported as separate removals and additions. These explanations apply to +the current report and preserve the underlying addition and removal counts. + +Reported bounds are envelopes of stored coordinates in cell-local database +units. They exclude path width, rendered text size, and recursively transformed +reference geometry. Account for UNITS before interpreting a physical dimension +or movement. + +Recognized [layer-name annotations](layer-names.md) are removed from ordinary +object comparison. Their name mapping compares separately, so a layer rename +appears once. + +## Know when a result is incomplete + +Unsupported records, future metadata versions, unresolved references, and +hierarchy cycles prevent a complete comparison. Pheon reports known differences +but returns 2 rather than claiming equality. + +Scope can exclude a record known to belong to another layer. Unsupported +records whose layer is unknown can still make a layer-scoped comparison +incomplete. Library fields and future metadata diagnostics remain visible. +Malformed input fails before scope is applied. + +For automation, check completeness as well as differences. The +[JSON report reference](diff-reports.md) describes the status fields and retained +evidence. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..1d960d7 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,68 @@ +# Development + +The checkout selects stable Rust. The minimum supported compiler is 1.88. +Use the checked-in `Cargo.lock` and update dependencies explicitly with Cargo. + +## Check a change + +| Area | Start here | First check | +| --- | --- | --- | +| GDSII parsing and Arrow reads | [`src/record.rs`](../src/record.rs), [`src/library.rs`](../src/library.rs) | `cargo test --locked --test parser_errors --test selective_reads` | +| Comparisons and CLI | [`src/review.rs`](../src/review.rs), [`src/review/`](../src/review/), [`src/bin/pheon.rs`](../src/bin/pheon.rs) | `cargo test --locked --test review` | +| Python and CFFI | [`pheon/`](../pheon/), [`src/lib.rs`](../src/lib.rs) | `python -I tools/check_installed.py` after installation | + +Run `cargo test --locked` for the full Rust suite or `cargo test --release --locked` +for release builds. Set `RUSTUP_TOOLCHAIN=1.88.0` to check the minimum compiler. + +## Test the Python package + +Use Python 3.11 or newer and install into a virtual environment. + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install . +python -I tools/check_installed.py +``` + +To test a newly built shared library, set `PHEON_LIBRARY` to its absolute path +and run `python -m unittest discover -s tests/python` from the checkout. Clear +the override when checking the bundled library with `check_installed.py`. + +## Check upstream layouts + +The [GDSII corpus](../contrib/gds-corpus.toml) pins open-source fixtures and cell +libraries. Install Pheon in the virtual environment before running the corpus +check. + +```sh +cargo build --release --locked --bin pheon +python -I tools/check_gds_corpus.py --fetch +``` + +The checker uses `curl` to cache layouts and license files in `.cache/gds-corpus/` +and verifies their SHA-256 digests. Omit `--fetch` to rerun offline. It checks +full reads, scans, selected reads, and CLI self-comparisons. It writes diagnostics +to `.cache/gds-corpus/report.json` and exits with status 1 if a check fails. + +## Interfaces and reports + +The CFFI declarations live in [`pheon/_ffi.h`](../pheon/_ffi.h). Keep them aligned +with the exported native functions. The C API version and +[Arrow layout version](arrow-schema.md) are independent of package releases and +the [JSON report schema](diff-reports.md). + +Review tests cover comparison semantics, CLI behavior, symbolic array counts, +and versioned report goldens. To regenerate JSON goldens after deliberately +changing their contract: + +```sh +PHEON_UPDATE_REPORT_GOLDENS=1 cargo test --test review machine_reports_match_versioned_goldens +``` + +Inspect the resulting diff. Goldens normalize the producer version to `test`. +Use the [JSON Schema](diff-report-v1.schema.json) to check structure. Rust tests +verify links, totals, and agreement between the result and exit status. + +See [building releases](releases.md) for source archives, wheels, and standalone +CLI artifacts. diff --git a/docs/diff-report-v1.schema.json b/docs/diff-report-v1.schema.json new file mode 100644 index 0000000..4427d90 --- /dev/null +++ b/docs/diff-report-v1.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Pheon diff report version 1", + "description": "Structural comparison evidence, not a layout serialization. Unknown additional fields are permitted for compatible extensions.", + "type": "object", + "required": ["schema_version", "kind", "producer", "comparison_profile", "inputs", "scope", "coordinates", "status", "result", "exit_code", "summary", "report_selection", "layer_legend", "library_changes", "layer_name_changes", "cell_changes", "hierarchy", "diagnostics"], + "properties": { + "schema_version": {"const": 1}, + "kind": {"const": "diff"}, + "producer": { + "type": "object", "required": ["name", "version"], + "properties": {"name": {"const": "pheon"}, "version": {"type": "string"}} + }, + "comparison_profile": {"type": "string"}, + "inputs": { + "type": "object", "required": ["old", "new"], + "properties": {"old": {"$ref": "#/$defs/input"}, "new": {"$ref": "#/$defs/input"}} + }, + "scope": { + "type": "object", "required": ["top", "cell", "layer"], + "properties": {"top": {"type": ["string", "null"]}, "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}} + }, + "coordinates": { + "type": "object", "required": ["unit", "frame", "bounds"], + "properties": {"unit": {"const": "database_unit"}, "frame": {"const": "cell_local"}, "bounds": {"const": "coordinate_envelope"}} + }, + "status": {"enum": ["complete", "incomplete", "failed"]}, + "result": {"enum": ["equal", "different", "unknown"]}, + "exit_code": {"enum": [0, 1, 2]}, + "summary": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/summary"}]}, + "report_selection": {"$ref": "#/$defs/selection"}, + "layer_legend": { + "type": "array", "items": { + "type": "object", "required": ["layer", "name"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "name": {"type": "string"}} + } + }, + "library_changes": {"type": "array", "items": {"$ref": "#/$defs/libraryChange"}}, + "layer_name_changes": { + "type": "array", "items": { + "type": "object", "required": ["layer", "before", "after"], + "properties": {"layer": {"$ref": "#/$defs/layer"}, "before": {"type": ["string", "null"]}, "after": {"type": ["string", "null"]}} + } + }, + "cell_changes": {"type": "array", "items": {"$ref": "#/$defs/cellChange"}}, + "hierarchy": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/graph"}]}, + "diagnostics": {"type": "array", "items": {"$ref": "#/$defs/diagnostic"}} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "complete"}}}, + "then": { + "properties": {"result": {"enum": ["equal", "different"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "exact"}}}, "diagnostics": {"maxItems": 0}}, + "if": {"properties": {"result": {"const": "equal"}}}, + "then": {"properties": {"exit_code": {"const": 0}}}, + "else": {"properties": {"exit_code": {"const": 1}}} + }, + "else": {"properties": {"exit_code": {"const": 2}, "diagnostics": {"minItems": 1}}} + }, + { + "if": {"properties": {"status": {"const": "incomplete"}}}, + "then": {"properties": {"result": {"enum": ["different", "unknown"]}, "summary": {"$ref": "#/$defs/summary", "properties": {"counts": {"const": "observed"}}}}} + }, + { + "if": {"properties": {"status": {"const": "failed"}}}, + "then": {"properties": {"result": {"const": "unknown"}, "summary": {"type": "null"}, "library_changes": {"maxItems": 0}, "layer_name_changes": {"maxItems": 0}, "cell_changes": {"maxItems": 0}, "hierarchy": {"type": "null"}}} + } + ], + "$defs": { + "decimal": {"type": "string", "pattern": "^(0|[1-9][0-9]*)$"}, + "hex": {"type": "string", "pattern": "^0x([0-9a-f]{2})*$"}, + "uint16": {"type": "integer", "minimum": 0, "maximum": 65535}, + "int32": {"type": "integer", "minimum": -2147483648, "maximum": 2147483647}, + "layer": { + "type": "object", "required": ["number", "purpose"], + "properties": {"number": {"$ref": "#/$defs/uint16"}, "purpose": {"$ref": "#/$defs/uint16"}} + }, + "nullableLayer": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/layer"}]}, + "point": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"$ref": "#/$defs/int32"}}, + "fields": {"type": "array", "items": {"$ref": "#/$defs/field"}}, + "field": { + "type": "object", "required": ["tag", "name", "encoding", "value", "payload_hex"], + "properties": { + "tag": {"$ref": "#/$defs/uint16"}, "name": {"type": "string"}, + "encoding": {"enum": ["xy", "uint16", "int16", "int32", "bits", "real8", "ascii", "none", "bytes"]}, + "payload_hex": {"$ref": "#/$defs/hex"}, + "value": {"type": ["array", "string", "null"]} + }, + "allOf": [ + {"if": {"properties": {"encoding": {"const": "xy"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/point"}}}}}, + {"if": {"properties": {"encoding": {"enum": ["uint16", "bits"]}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/uint16"}}}}}, + {"if": {"properties": {"encoding": {"const": "int16"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": "integer", "minimum": -32768, "maximum": 32767}}}}}, + {"if": {"properties": {"encoding": {"const": "int32"}}}, "then": {"properties": {"value": {"type": "array", "items": {"$ref": "#/$defs/int32"}}}}}, + {"if": {"properties": {"encoding": {"const": "real8"}}}, "then": {"properties": {"value": {"type": "array", "items": {"type": ["number", "null"]}}}}}, + {"if": {"properties": {"encoding": {"const": "ascii"}}}, "then": {"properties": {"value": {"type": ["string", "null"]}}}}, + {"if": {"properties": {"encoding": {"enum": ["none", "bytes"]}}}, "then": {"properties": {"value": {"type": "null"}}}} + ] + }, + "input": { + "type": "object", "required": ["path", "path_encoded_hex", "present", "units"], + "properties": { + "path": {"type": ["string", "null"]}, + "path_encoded_hex": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/hex"}]}, + "present": {"type": ["boolean", "null"]}, + "units": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]} + } + }, + "summary": { + "type": "object", "required": ["counts", "changed_cells", "removed_objects", "added_objects", "layer_name_changes", "library_field_changes"], + "properties": { + "counts": {"enum": ["exact", "observed"]}, + "changed_cells": {"$ref": "#/$defs/decimal"}, "removed_objects": {"$ref": "#/$defs/decimal"}, "added_objects": {"$ref": "#/$defs/decimal"}, + "layer_name_changes": {"$ref": "#/$defs/decimal"}, "library_field_changes": {"$ref": "#/$defs/decimal"} + } + }, + "selection": { + "type": "object", "required": ["cell", "layer", "max_objects", "truncated", "filtered_cells", "filtered_objects", "limited_objects", "emitted_objects"], + "properties": { + "cell": {"type": ["string", "null"]}, "layer": {"$ref": "#/$defs/nullableLayer"}, + "max_objects": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "truncated": {"type": "boolean"}, + "filtered_cells": {"$ref": "#/$defs/decimal"}, "filtered_objects": {"$ref": "#/$defs/decimal"}, + "limited_objects": {"$ref": "#/$defs/decimal"}, "emitted_objects": {"$ref": "#/$defs/decimal"} + } + }, + "libraryChange": { + "oneOf": [ + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"const": "presence"}, "tag": {"type": "null"}, "before": {"type": "boolean"}, "after": {"type": "boolean"}} + }, + { + "type": "object", "required": ["field", "tag", "before", "after"], + "properties": {"field": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "before": {"$ref": "#/$defs/fields"}, "after": {"$ref": "#/$defs/fields"}} + } + ] + }, + "source": { + "type": "object", "required": ["side", "byte_offset", "end_offset"], + "properties": {"side": {"enum": ["old", "new"]}, "byte_offset": {"$ref": "#/$defs/decimal"}, "end_offset": {"$ref": "#/$defs/decimal"}} + }, + "property": { + "type": "object", "required": ["attribute", "value", "payload_hex"], + "properties": {"attribute": {"$ref": "#/$defs/uint16"}, "value": {"type": ["string", "null"]}, "payload_hex": {"$ref": "#/$defs/hex"}} + }, + "object": { + "type": "object", "required": ["id", "kind", "tag", "source", "layer", "bounds", "target", "origin", "width", "counts", "fields", "properties"], + "properties": { + "id": {"type": "string"}, "kind": {"type": "string"}, "tag": {"$ref": "#/$defs/uint16"}, "source": {"$ref": "#/$defs/source"}, + "layer": {"$ref": "#/$defs/nullableLayer"}, + "bounds": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 4, "maxItems": 4, "items": {"$ref": "#/$defs/int32"}}]}, + "target": {"type": ["string", "null"]}, "origin": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/point"}]}, + "width": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/int32"}]}, + "counts": {"oneOf": [{"type": "null"}, {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "integer", "minimum": 1, "maximum": 32767}}]}, + "fields": {"$ref": "#/$defs/fields"}, "properties": {"type": "array", "items": {"$ref": "#/$defs/property"}} + } + }, + "replacement": { + "type": "object", "required": ["removed", "added", "match_rule", "explanation", "field_changes", "properties_changed"], + "properties": { + "removed": {"type": "string"}, "added": {"type": "string"}, "match_rule": {"type": "string"}, "explanation": {"type": "string"}, "properties_changed": {"type": "boolean"}, + "field_changes": {"type": "array", "items": { + "type": "object", "required": ["tag", "before", "after"], + "properties": {"tag": {"$ref": "#/$defs/uint16"}, "before": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}, "after": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/field"}]}} + }} + } + }, + "cellSide": { + "oneOf": [{"type": "null"}, { + "type": "object", "required": ["byte_offset", "attributes"], + "properties": {"byte_offset": {"$ref": "#/$defs/decimal"}, "attributes": {"$ref": "#/$defs/fields"}} + }] + }, + "cellChange": { + "type": "object", "required": ["name", "before", "after", "attributes_changed", "removed", "added", "replacements"], + "properties": { + "name": {"type": "string"}, "before": {"$ref": "#/$defs/cellSide"}, "after": {"$ref": "#/$defs/cellSide"}, "attributes_changed": {"type": "boolean"}, + "removed": {"type": "array", "items": {"$ref": "#/$defs/object"}}, "added": {"type": "array", "items": {"$ref": "#/$defs/object"}}, + "replacements": {"type": "array", "items": {"$ref": "#/$defs/replacement"}} + } + }, + "diagnostic": { + "type": "object", "required": ["code", "severity", "message", "side", "byte_offset", "cell", "layer", "target"], + "properties": { + "code": {"type": "string"}, "severity": {"const": "error"}, "message": {"type": "string"}, "side": {"enum": ["old", "new", null]}, + "byte_offset": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/decimal"}]}, "cell": {"type": ["string", "null"]}, + "layer": {"$ref": "#/$defs/nullableLayer"}, "target": {"type": ["string", "null"]} + } + }, + "graph": { + "type": "object", "required": ["nodes", "edges"], + "properties": { + "nodes": {"type": "array", "items": { + "type": "object", "required": ["cell", "before", "after", "change"], + "properties": { + "cell": {"type": "string"}, "before": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, "after": {"enum": ["included", "outside_scope", "unresolved", "absent"]}, + "change": {"enum": ["direct", "affected", "unchanged"]} + } + }}, + "edges": {"type": "array", "items": { + "type": "object", "required": ["parent", "target", "before_count", "after_count", "reference_changed"], + "properties": {"parent": {"type": "string"}, "target": {"type": "string"}, "before_count": {"$ref": "#/$defs/decimal"}, "after_count": {"$ref": "#/$defs/decimal"}, "reference_changed": {"type": "boolean"}} + }} + } + } + } +} diff --git a/docs/diff-reports.md b/docs/diff-reports.md new file mode 100644 index 0000000..b71b0ab --- /dev/null +++ b/docs/diff-reports.md @@ -0,0 +1,131 @@ +# JSON diff reports + +Use JSON reports to automate layout review while retaining the evidence behind +a result. The [comparison rules](comparison.md) define equality; the +[version 1 JSON Schema](diff-report-v1.schema.json) defines every field. +A [complete example](https://mpxd.net/code/jan/pheon/src/branch/master/tests/golden/report-geometry.json) shows geometry changes. + +```sh +pheon diff before.gds after.gds --format json > report.json +pheon diff before.gds after.gds --format json --hierarchy graph +pheon diff before.gds after.gds --format json --top CHIP --layer 1/0 +``` + +`diff` writes one UTF-8 JSON document and a newline to stdout, including comparison +diagnostics. Output write failures go to stderr and return 2; interrupted output +is invalid. Other CLI commands retain text output. + +## Check the result first + +| `status` | `result` | Exit code | Meaning | +| --- | --- | --- | --- | +| `complete` | `equal` | 0 | Equal under the recorded profile and scope. | +| `complete` | `different` | 1 | Differences found. | +| `incomplete` | `different` | 2 | Definite differences, with comparison limitations. | +| `incomplete` | `unknown` | 2 | Equality cannot be established. | +| `failed` | `unknown` | 2 | No comparison result. | + +A code of 1 is an expected result, including when saving a report. Unsupported +records, future metadata, unresolved references, and cycles make a comparison +incomplete. Malformed input, I/O errors, invalid scopes, and command errors fail. + +`summary` covers the whole scoped comparison before display filters. Counts are +`exact` for complete results and `observed` for incomplete ones. Failed reports +have a null summary and empty change arrays. Replacement suggestions and affected +ancestors do not add to change counts. + +## Scope and evidence selection + +`scope` records `top`, `cell`, and numeric `layer` selectors; null means +unrestricted. The full input is parsed before scope selection. + +`report_selection` records display-only cell/layer filters and `max_objects`. +JSON is unbounded by default. Explicit `--max-changes N` limits total emitted +removals plus additions, with removals first within each cell. Selection counters +`filtered_cells`, `filtered_objects`, `limited_objects`, and `emitted_objects` +explain omissions. `truncated` is true when filtering or limits omit evidence. +Library changes, layer-name changes, diagnostics, summaries, and the optional +graph remain. Replacement links appear only if both objects are emitted. + +## Read values safely + +Aggregate counts, limits, and byte offsets are unsigned decimal **strings** to +avoid JSON number precision loss. Coordinates, numeric tags, layer/type pairs, +widths, and array dimensions are bounded JSON numbers. Null means absent or +unavailable; zero and empty arrays remain meaningful values. + +`inputs.old` and `inputs.new` record paths, presence, and UNITS. An absent revision +is distinct from an empty library. The Rust API supports absent Git revisions +with `Document::empty_like`. Failed reports may lack presence, units, or argument +context. Non-UTF-8 paths use null `path` and `path_encoded_hex`: `0x`-prefixed +platform-native Rust path bytes, not a portable filename encoding. + +## Interpret changes + +| Section | Evidence | +| --- | --- | +| `library_changes` | Typed before/after header fields, including unscaled UNITS. Presence changes use a null tag and boolean values. | +| `layer_name_changes` | Before/after names for numeric pairs `{ "number": 1, "purpose": 0 }`. | +| `layer_legend` | Unchanged names available in scope. Annotation labels do not also appear as object changes. | +| `cell_changes` | Name, nullable before/after cell records, attributes, removed/added objects, and replacement suggestions. A null side means absent; an existing empty cell still has a byte offset. | + +Objects retain kind, numeric tag, layer, target, bounds, source side and half-open +byte range, typed fields, and a sorted property multiset including duplicates. +Convenience fields include origin, width, and reference dimensions. SREF counts +are `[1, 1]`; kind and retained fields still distinguish a one-element AREF. + +Coordinates are cell-local database units. Bounds are stored-coordinate +envelopes, excluding stroke width, rendered text, and recursive target geometry. +Consider UNITS before deriving physical dimensions. + +Fields contain `tag`, display `name`, `encoding`, decoded `value`, and +`payload_hex`. Numeric tags are authoritative. XY values are lists of points; +other numeric values are arrays. Unsupported encodings have null values and +preserved bytes. Invalid UTF-8 strings/properties have null decoded values; +hex preserves their bytes. + +Evidence follows the comparison profile: boundary/box rings are canonicalized, +string padding is removed, and fields/properties are sorted. REAL8 encodings and +explicit defaults remain significant. For gzip inputs, source offsets locate +records in the decompressed layout. + +Replacements link report-local removed/added IDs with `match_rule`, +`field_changes`, and `properties_changed`. The `unique_layer_change` rule +matches a unique polygon, box, or path on each side with identical retained +fields and properties outside the layer and datatype fields. The other rules +are `unique_kind_layer_bounds`, `unique_kind_layer`, and `unique_kind_target`. +Ambiguous candidates stay separate. Replacement IDs refer only to objects in +the current report. Explanation text is for display only. + +## Diagnostics and hierarchy + +Diagnostics provide `code`, `severity`, `message`, input side, byte offset, cell, +layer, and target; unknown context is null. Use codes and context for automation. +Fatal parser locations currently appear only in the human message, with null +structured offsets. Do not parse message wording. + +Limitation codes are `unsupported_record`, `unsupported_metadata_version`, +`unresolved_reference`, `hierarchy_cycle`, and `scope_mismatch`. Failure codes are +`invalid_input`, `io_error`, `invalid_scope`, and `command_error`. + +`--hierarchy graph` includes the full scoped union graph. By default, `hierarchy` +is null. JSON output rejects `--depth` with exit code 2. Nodes are keyed by name, +with per-side states `included`, `outside_scope`, `unresolved`, or `absent`, and +change states `direct`, `affected`, or `unchanged`. +Edges record parent, target, before/after direct placement counts, and changed +reference records. Array multiplicities are symbolic, not expanded or recursively +accumulated. Shared cells occur once and removed edges remain. The graph covers +the full comparison scope regardless of display filters and object limits. + +## Versions + +`schema_version` is 1, `comparison_profile` is `pheon-structural-v1`, and +`producer.name` is `pheon`. Structure, equality rules, and producer version evolve +independently. Understand the profile before interpreting equality. Consumers +should tolerate additional optional fields and unknown codes/rules without +assuming equality. Removed fields or changed meanings require a schema change. + +Serialization is deterministic for identical inputs, paths, options, and producer +version. Sorted evidence retains duplicates. Reports have no generation timestamp +or required content fingerprint. See [development](development.md) for contract +tests and golden updates. diff --git a/docs/git.md b/docs/git.md new file mode 100644 index 0000000..2a571d0 --- /dev/null +++ b/docs/git.md @@ -0,0 +1,63 @@ +# Git integration + +Configure Git to show meaningful GDSII changes during review. Install the +[Pheon CLI](cli.md#install) first, then choose one of the following drivers. +Templates are also available in [contrib/git](../contrib/git/). + +## Object comparison + +Use the external driver for object-aware comparison and affected hierarchy. +Add this to your repository's `.gitattributes`: + +```gitattributes +*.gds diff=pheon +``` + +Add this to `.git/config`, or use the same section in your user Git config: + +```gitconfig +[diff "pheon"] + command = pheon git-diff --hierarchy ascii -- + binary = true + trustExitCode = false +``` + +Then run `git diff` as usual. The driver reads Git's temporary old and new files +and handles additions, deletions, and renames. The heading names the file and +indicates whether it was added or deleted. Keep the final `--` because Git appends +its path and revision arguments after it. Put Pheon options such as `--top CHIP` +or `--show-layer 1/0` before that marker. + +The external driver also accepts `--color=always` before `--` to keep colors +when its output goes through a pager that supports ANSI colors. + +For an existing `diff=layout` attribute, put the configuration under +`[diff "layout"]` instead. Attribute names only select a Git driver. + +The external driver returns 0 after rendering either an equal or different +comparison, and nonzero for invalid or incomplete comparisons. For automation +that needs equality status, run `pheon diff OLD NEW` directly; it returns +0, 1, or 2 as described in the [CLI guide](cli.md). + +## Normalized text comparison + +Use text conversion when you want Git to compare normalized text dumps. Set +`.gitattributes` to: + +```gitattributes +*.gds diff=pheon-text +``` + +Configure the driver: + +```gitconfig +[diff "pheon-text"] + textconv = pheon dump + binary = true + cachetextconv = true +``` + +Git can cache converted text. Invalidate that cache when upgrading to a tool or +comparison profile that changes the dump format, or disable `cachetextconv`. +These diffs are for human review; they are not patches that can be applied to +GDSII files. See the [comparison rules](comparison.md) for normalization details. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f4e876d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,19 @@ +# Pheon documentation + +Start with a guide for the task you want to perform. + +| Task | Guide | +| --- | --- | +| Inspect layouts, compare revisions, or view hierarchy | [Command-line tools](cli.md) | +| Load geometry or selected cells in Python | [Python](python.md) | +| Review GDSII files with Git | [Git integration](git.md) | +| Add readable names to numeric layers | [Layer names](layer-names.md) | +| Understand what a comparison considers equal | [Comparison rules](comparison.md) | + +For integrations, use the [Arrow layout reference](arrow-schema.md), +[Rust and C API guide](native-api.md), or [JSON report reference](diff-reports.md). + +For contributors and maintainers, see [development](development.md) and +[building releases](releases.md). + +Return to the [project introduction](../README.md). diff --git a/docs/layer-names.md b/docs/layer-names.md new file mode 100644 index 0000000..505b08d --- /dev/null +++ b/docs/layer-names.md @@ -0,0 +1,46 @@ +# Layer names + +Attach readable names to numeric GDSII layer/type pairs so Pheon can show a +legend in comparison reports. Names are annotations; geometry keeps its numeric +layer assignments. + +## Add or update names + +```sh +pheon name-layers chip.gds named.gds --cell CHIP --carrier 900/7 \ + --name 1:0:WG --name 2:0:METAL +``` + +Choose an existing cell and an otherwise unused `(layer, texttype)` pair in that +cell as the carrier. The carrier must contain no ordinary design objects. +The output path must be new; Pheon refuses to overwrite any existing file, +including the input. + +The output is uncompressed GDSII, including when the input is gzip-compressed. + +The command consolidates the legend into that cell. Existing mappings remain +unless you override them with `--name`. It creates no cells or references, and +copies other records, timestamps, unknown records, and trailing padding byte for +byte. Input and annotation validation finish before the output is created. + +## Annotation format + +A text label whose exact string is `PHEON_LAYER_NAMES_V1` identifies a carrier +pair in its cell. Every other text label on that pair in the same cell is a +mapping of the form: + +```text +layer:datatype:name +``` + +For example, `1:0:WG` names pair `(1, 0)` as WG. Numbers are unsigned 16-bit +decimals. The name must be nonempty UTF-8 and may contain colons; only the first +two colons separate fields. + +Mappings are collected across the library. Identical duplicates are accepted; +conflicting names for the same pair are errors. Recognized annotation labels +are excluded from ordinary object comparison. The name mapping compares +separately, while unchanged names remain available in the report legend. + +Pheon does not currently read LayoutEditor LayerNames metadata, extract layers, +or remap geometry. See the [CLI guide](cli.md) for layer selection during review. diff --git a/docs/native-api.md b/docs/native-api.md new file mode 100644 index 0000000..01ddcf4 --- /dev/null +++ b/docs/native-api.md @@ -0,0 +1,102 @@ +# Rust and C integration + +Use the Rust library or C API to read GDSII into Arrow without Python. For the +returned data layout, see the [Arrow reference](arrow-schema.md). + +## Rust + +Add [pheon](https://crates.io/crates/pheon) from your Rust project. + +```sh +cargo add pheon +``` + +The library readers accept GDSII byte slices. + +```rust +use pheon::library::read_library; + +let bytes = std::fs::read("chip.gds").expect("read layout"); +let (_remainder, layout) = read_library(&bytes).expect("parse layout"); +println!("{layout:?}"); +``` + +`read_library` reads geometry; `scan_library` returns metadata, references, and +cell ranges. Pass `(start, end)` ranges from a scan to `read_selected_library` +to load only those cells. Each range must span one complete cell. Readers return +`(remainder, StructArray)` on success. The selected reader returns the original +input as its remainder, since selection is random access. + +Layout-review APIs live in `pheon::review`, with layer-name helpers in +`pheon::review::names` and scope/display selectors in `pheon::review::selection`. + +`Comparison::between(&old, &new)` captures an owned result with revision metadata, +changes, and structured diagnostics. Use `outcome()` for completeness, equality, +and exit status. Its renderers work after the source documents and byte buffers +are dropped. `json_report` returns a JSON value, and `write_json_report` writes +object evidence incrementally to a writer. `Document::hierarchy` inspects a +single revision. A comparison captures reference counts and analyzes the union +graph and ancestors when a hierarchy is requested. + +`TextComparison::between` captures the data needed for text and hierarchy +reports. Both comparison types provide `render_text_with_ascii_hierarchy` +to render a combined report with redundant terminal edges omitted. Pass +`TextReportOptions { details: true, ..Default::default() }` to +`render_text_with_options` to show individual object evidence. +Use `TextReport::new(true)` with `write_text_report`, `write_canonical_dump`, +or `write_record_dump` to get plain text and style ranges for presentation. + +Use `RevisionPair::parse(&old_bytes, &new_bytes)` for an unscoped comparison +that can reuse matching cells. Its `compare` and `compare_text` methods return +the corresponding owned comparison. + +The `record`, `records`, and `basic` modules also provide low-level GDSII record +writing. There is no high-level writer for the Arrow layout. REAL8 writing is +exact for representable in-range `f64` values: magnitudes below `2^-260` +underflow to zero; magnitudes at least `2^252` and nonfinite values are rejected. + +Record writes check payload sizes before emitting bytes. Header packing returns +an `io::Result`, including an error when the total size exceeds the format limit. +Arrow readers return parse errors for invalid UTF-8 strings and timestamp years +that exceed the Int16 range after conversion from the 1900 epoch. + +## C API + +Build the shared library with `cargo build --release --locked`. The +[header](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h) defines the functions and Arrow C Data structures. +The library is `libpheon.so`, `libpheon.dylib`, or `pheon.dll` on Linux, macOS, +or Windows respectively. + +| Function | Input | Result | +| --- | --- | --- | +| `read_path` | NUL-terminated path | Full layout. | +| `read_bytes` | Byte pointer and size | Full layout. | +| `scan_bytes` | Byte pointer and size | Metadata, references, and cell ranges. | +| `read_cells_bytes` | Bytes and cell ranges | Full layout for selected cells. | + +`read_path` accepts UTF-8 paths on Windows and filesystem bytes on Unix. It reads +uncompressed GDSII; Python's `read_path` adds gzip handling. + +For `read_cells_bytes`, `ranges` points to alternating uint64 start/end offsets +and `range_count` is the number of cells, not the number of offsets. Ranges are +half-open, from BGNSTR through the end of ENDSTR. Range values must be aligned, +native-endian uint64 values. Input buffers must remain readable for the call. +Null buffer pointers are allowed only when their corresponding length is zero. + +### Status and ownership + +Each reader returns 0 on success and -1 on failure. Call `last_error_message()` +on the same thread after a failure. It returns a borrowed NUL-terminated message; +copy it before the next parser call or thread exit. It returns null after a +successful parser call. + +Supply valid, writable, initialized `ArrowArray` and `ArrowSchema` structures +without live allocations to overwrite. A failed call leaves them unchanged. +Successful outputs own their storage independently of the input buffer. Transfer +ownership to an Arrow importer, or call each non-null release callback exactly +once. Do not release structures again after an importer takes ownership. + +`pheon_abi_version()` and `pheon_arrow_schema_version()` both currently return 1. +Check both before using the API. The C calling/ownership contract and Arrow +layout have separate versions; neither is the package version or the +[JSON report schema](diff-reports.md). diff --git a/docs/python.md b/docs/python.md new file mode 100644 index 0000000..f02fffb --- /dev/null +++ b/docs/python.md @@ -0,0 +1,91 @@ +# Python + +Use `pheon.arrow` to load GDSII geometry into PyArrow, scan a hierarchy, or read +only selected cells. The package also provides integer packing. +Python 3.11 or newer is required. + +## Install + +Install Pheon from [PyPI](https://pypi.org/project/pheon/). + +```sh +python -m pip install pheon +``` + +The package installs PyArrow for layout reads. Existing `pheon[arrow]` +requirements still work. [Prebuilt wheels](https://pypi.org/project/pheon/#files) +bundle the native library. On platforms without a compatible wheel, pip builds +the library from source and needs Rust 1.88 or newer and a native linker. + +From a source checkout, install with `python -m pip install .`, then run Python +outside the checkout to import the installed package. + +## Read a layout + +```python +from pheon import arrow + +layout = arrow.read_path("chip.gds") +names = layout.field("cell_names")[0].as_py() +cells = layout.field("cells")[0].values +print([names[cell_id] for cell_id in cells.field("id").to_pylist()]) +``` + +`read_path` accepts strings and path-like objects, expands `~`, and recognizes +gzip by its magic bytes. `read_bytes` accepts an uncompressed, contiguous byte +buffer, such as `bytes`, `bytearray`, or a compatible memoryview. + +Every reader returns a one-row `pyarrow.StructArray`. The example uses each +cell's ID to look up its name. The name table can also contain reference targets +without cell definitions. Geometry is grouped by cell and element kind. The +returned arrays own their storage independently of the input buffer. +See the [Arrow reference](arrow-schema.md) for fields and encodings. + +## Scan, then read selected cells + +```python +from pathlib import Path +from pheon import arrow + +content = Path("chip.gds").read_bytes() # Uncompressed GDSII bytes. +scan = arrow.scan_bytes(content) +cells = scan.field("cells")[0].values +ranges = list(zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), +)) +selected = arrow.read_cells_bytes(content, ranges[:2]) +``` + +Scanning returns library metadata, cell names, references, and complete cell byte +ranges without decoding geometry or text. Selected reads return the full layout +schema for the requested cells, in the requested order. Use the selected result's +own name and layer tables: IDs may differ between reads. + +Ranges must have shape `(N, 2)` with nonnegative integer offsets. Each half-open +range must span exactly one complete cell, from BGNSTR through ENDSTR. An empty +list returns metadata without cells. For compressed files, decompress the bytes +before scanning or selecting ranges. + +## Pack integers + +```python +import pheon + +int2_bytes = pheon.pack_int2([1, 2, 3]) +int4_bytes = pheon.pack_int4([100, 200]) +``` + +These helpers return big-endian GDSII integer bytes. They accept NumPy arrays, +Python sequences, or scalars and preserve the caller's data, including for +noncontiguous or differently ordered arrays. Pheon currently has no high-level +Python layout writer. + +## Handle read errors + +Native parse failures raise `pheon.PheonError`, a subclass of `ValueError`. +Invalid buffer or range arguments raise `ValueError` or `TypeError`; file access +errors use Python's normal I/O exceptions. + +See [development](development.md) to test a locally built native library from +Python, or the [native API guide](native-api.md) to call the library directly. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..c36c7ba --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,95 @@ +# Building releases + +Build packages locally for distribution. The checkout holds Python build tools +in `.venv/`, release artifacts in `dist/release/`, and Linux wheel caches in +`.cache/wheels/`. Use your installed Rust toolchain and Cargo registry cache. + +## Start a release + +Use Python 3.11+ and Rust 1.88+. Run the commands from the repository root and +start with an empty `dist/release/` directory. + +```sh +python3 -m venv .venv +. .venv/bin/activate +export PIP_CACHE_DIR="$PWD/.cache/wheels/pip" +python -m pip install build twine +python -m build --sdist --outdir dist/release +``` + +Use `python -m build --wheel` to build a wheel for your own machine. For a +portable Linux wheel, use the container below. See [development](development.md) +for compiler checks. + +## Build Linux x86-64 wheels + +On x86-64 Linux, use rootless Podman and a Rust toolchain installed through +`rustup`. Build the portable wheel from the source archive. + +```sh +python -m pip install 'cibuildwheel==4.3.5' +python tools/build_manylinux.py dist/release/pheon-*.tar.gz +``` + +The helper pulls the [pinned image](../contrib/wheels/manylinux-image.txt) when +needed and reuses it from `.cache/wheels/podman/storage/`. It keeps Podman’s +storage runtime directory, download scratch space, and wheel caches under +`.cache/wheels/`. Podman still uses the normal system user runtime for host +services. To inspect this checkout’s images, use the same helper. + +```sh +python tools/build_manylinux.py podman image ls +``` + +cibuildwheel repairs the wheel and tests it on Python 3.11–3.14. The container +uses the host Rust toolchain and Cargo registry through read-only mounts. + +## Build Windows x86-64 packages from Linux + +Install MinGW-w64 with `x86_64-w64-mingw32-gcc` on `PATH`. From the project +environment, build the Windows wheel and the standalone CLI: + +```sh +rustup target add x86_64-pc-windows-gnu +PHEON_BUILD_TARGET=x86_64-pc-windows-gnu python -m build --wheel --outdir dist/release +cargo build --locked --release --target x86_64-pc-windows-gnu --bin pheon +``` + +The `py3-none-win_amd64` wheel lands in `dist/release/`. The executable is +`target/x86_64-pc-windows-gnu/release/pheon.exe`. Test the wheel and CLI on Windows +before publishing. + +## Validate artifacts + +Check the Python package metadata and verify that the packaged crate builds. + +```sh +python -m twine check --strict dist/release/*.tar.gz dist/release/*.whl +cargo publish --dry-run --locked +``` + +Wheel filenames use `py3-none` plus a platform tag, such as +`manylinux_2_28_x86_64` or `win_amd64`. Use repaired `manylinux_*` wheels for +Linux releases. + +To test a wheel manually, install it in a test environment on the target OS. +Clear `PHEON_LIBRARY` so the checks load the bundled library. Replace the +example path with the wheel you built. + +```sh +python -m pip install "path/to/pheon-wheel.whl" +python -I tools/check_installed.py +``` + +Build standalone CLI artifacts for each OS and architecture. Include the +executable, license, README, documentation, and Git templates. + +## Publish + +Upload the validated Python files you selected, then publish the crate from a +clean release checkout. + +```sh +python -m twine upload --repository pypi FILE ... +cargo publish --locked +``` diff --git a/klamath_rs_ext/__init__.py b/klamath_rs_ext/__init__.py deleted file mode 100644 index d042d92..0000000 --- a/klamath_rs_ext/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .basic import pack_int2 as pack_int2 -from .basic import pack_int4 as pack_int4 - - -__version__ = 0.2 - diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py deleted file mode 100644 index aabc246..0000000 --- a/klamath_rs_ext/basic.py +++ /dev/null @@ -1,92 +0,0 @@ -from collections.abc import Sequence -import ctypes -from pathlib import Path -from itertools import chain - -import numpy -from numpy.typing import NDArray - -from .klamath_rs_ext import lib, ffi - - -CONV_TABLE_i16 = { - numpy.float64: lib.f64_to_i16, - numpy.float32: lib.f32_to_i16, - numpy.int64: lib.i64_to_i16, - numpy.int32: lib.i32_to_i16, - numpy.int16: lib.i16_to_i16, - numpy.uint64: lib.u64_to_i16, - numpy.uint32: lib.u32_to_i16, - numpy.uint16: lib.u16_to_i16, - } - -CONV_TABLE_i32 = { - numpy.float64: lib.f64_to_i32, - numpy.float32: lib.f32_to_i32, - numpy.int64: lib.i64_to_i32, - numpy.int32: lib.i32_to_i32, - numpy.uint64: lib.u64_to_i32, - numpy.uint32: lib.u32_to_i32, - } - - - -def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.atleast_1d(data) - - for dtype in CONV_TABLE_i16.keys(): - if arr.dtype != dtype: - continue - - arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) - if arr is data: - arr = numpy.array(arr, copy=True) - - fn = CONV_TABLE_i16[dtype] - buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) - result = fn(buf, arr.size) - - if result != 0: - raise ValueError(f'Invalid value for conversion to Int2: {result}') - - i2arr = arr.view('>i2')[::arr.itemsize // 2] - return i2arr.tobytes() - - if arr.dtype == numpy.dtype('>i2'): - return arr.tobytes() - - if (arr > 32767).any() or (arr < -32768).any(): - raise Exception(f'int2 data out of range: {arr}') - - return arr.astype('>i2').tobytes() - - -def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: - arr = numpy.atleast_1d(data) - - for dtype in CONV_TABLE_i32.keys(): - if arr.dtype != dtype: - continue - - arr = numpy.require(arr, requirements=('C_CONTIGUOUS', 'ALIGNED', 'WRITEABLE', 'OWNDATA')) - if arr is data: - arr = numpy.array(arr, copy=True) - - fn = CONV_TABLE_i32[dtype] - buf = ffi.from_buffer(ffi.typeof(fn).args[0], arr, require_writable=True) - result = fn(buf, arr.size) - - if result != 0: - raise ValueError(f'Invalid value for conversion to Int4: {result}') - - i4arr = arr.view('>i4')[::arr.itemsize // 4] - return i4arr.tobytes() - - if arr.dtype == numpy.dtype('>i4'): - return arr.tobytes() - - if (arr > 2147483647).any() or (arr < -2147483648).any(): - raise Exception(f'int4 data out of range: {arr}') - - return arr.astype('>i4').tobytes() - diff --git a/pheon/__init__.py b/pheon/__init__.py new file mode 100644 index 0000000..535f808 --- /dev/null +++ b/pheon/__init__.py @@ -0,0 +1,10 @@ +"""Native GDSII helpers. Use pheon.arrow to read layouts into PyArrow.""" + +from importlib.metadata import version as _version + +from ._native import PheonError as PheonError +from .basic import pack_int2 as pack_int2 +from .basic import pack_int4 as pack_int4 + + +__version__ = _version("pheon") diff --git a/pheon/_ffi.h b/pheon/_ffi.h new file mode 100644 index 0000000..6bca31f --- /dev/null +++ b/pheon/_ffi.h @@ -0,0 +1,53 @@ +#ifndef PHEON_H +#define PHEON_H +#include +#include + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +uint32_t pheon_abi_version(void); +uint32_t pheon_arrow_schema_version(void); +int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema); +const char *last_error_message(void); + +double f64_to_i16(double *data, size_t size); +double f64_to_i32(double *data, size_t size); +float f32_to_i16(float *data, size_t size); +float f32_to_i32(float *data, size_t size); +int64_t i64_to_i16(int64_t *data, size_t size); +int64_t i64_to_i32(int64_t *data, size_t size); +int32_t i32_to_i16(int32_t *data, size_t size); +int32_t i32_to_i32(int32_t *data, size_t size); +int16_t i16_to_i16(int16_t *data, size_t size); +uint64_t u64_to_i16(uint64_t *data, size_t size); +uint64_t u64_to_i32(uint64_t *data, size_t size); +uint32_t u32_to_i16(uint32_t *data, size_t size); +uint32_t u32_to_i32(uint32_t *data, size_t size); +uint16_t u16_to_i16(uint16_t *data, size_t size); +#endif diff --git a/pheon/_native.py b/pheon/_native.py new file mode 100644 index 0000000..fc9dbd7 --- /dev/null +++ b/pheon/_native.py @@ -0,0 +1,55 @@ +"""Load a bundled or explicitly selected library through its stable C ABI.""" + +import os +from pathlib import Path +import sys + +from cffi import FFI + + +ffi = FFI() +header = Path(__file__).with_name("_ffi.h").read_text() +ffi.cdef( + "\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")) +) + +if sys.platform == "win32": + filename = "pheon.dll" +elif sys.platform == "darwin": + filename = "libpheon.dylib" +else: + filename = "libpheon.so" + +# An explicit override supports source-tree development without copying binaries +# into the package. Installed wheels load only their bundled native library. +library = ( + Path(os.environ["PHEON_LIBRARY"]).expanduser() + if "PHEON_LIBRARY" in os.environ + else Path(__file__).with_name(filename) +) +try: + lib = ffi.dlopen(str(library.resolve())) +except OSError as error: + raise ImportError( + f"Could not load Pheon library {library}: {error}. " + "Install a Pheon wheel or set PHEON_LIBRARY to a Cargo-built shared library." + ) from error + +ABI_VERSION = 1 +ARROW_SCHEMA_VERSION = 1 +try: + native_abi = lib.pheon_abi_version() + native_schema = lib.pheon_arrow_schema_version() +except AttributeError as error: + raise ImportError( + "The selected library does not expose the Pheon version API" + ) from error +if (native_abi, native_schema) != (ABI_VERSION, ARROW_SCHEMA_VERSION): + raise ImportError( + f"Incompatible Pheon library: C ABI {native_abi}, Arrow schema {native_schema}; " + f"expected C ABI {ABI_VERSION}, Arrow schema {ARROW_SCHEMA_VERSION}" + ) + + +class PheonError(ValueError): + """A native GDSII parse or selection error.""" diff --git a/pheon/arrow.py b/pheon/arrow.py new file mode 100644 index 0000000..af3a39a --- /dev/null +++ b/pheon/arrow.py @@ -0,0 +1,101 @@ +"""GDSII readers returning a one-row Arrow StructArray for the library. + +The returned arrays own their buffers independently of the input. +""" + +from __future__ import annotations + +from collections.abc import Sequence +import gzip +import os +from typing import Any, Callable + +import numpy +import pyarrow + +from ._native import PheonError, ffi, lib + + +def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray: + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + if call(array, schema) != 0: + error = lib.last_error_message() + message = ( + ffi.string(error).decode("utf-8", errors="replace") + if error != ffi.NULL + else "GDSII read failed" + ) + raise PheonError(message) + # PyArrow consumes the structures, setting their release callbacks to + # null. The finally block also covers failure during import. + return pyarrow.Array._import_from_c( + int(ffi.cast("uintptr_t", array)), int(ffi.cast("uintptr_t", schema)) + ) + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + +def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: + """Read a GDSII file, recognizing gzip by its magic bytes.""" + with open(os.path.expanduser(path), "rb") as stream: + compressed = stream.read(2) == b"\x1f\x8b" + stream.seek(0) + if compressed: + with gzip.GzipFile(fileobj=stream) as source: + return read_bytes(source.read()) + return read_bytes(stream.read()) + + +def _read_buffer( + buffer: Any, call: Callable[[Any, int, Any, Any], int] +) -> pyarrow.StructArray: + with memoryview(buffer) as view: + if not view.c_contiguous: + raise ValueError("GDSII input must be a contiguous buffer") + with ffi.from_buffer("const uint8_t[]", view) as data: + return _export(lambda array, schema: call(data, view.nbytes, array, schema)) + + +def read_bytes(buffer: Any) -> pyarrow.StructArray: + """Read a complete GDSII library from a contiguous borrowed byte buffer.""" + return _read_buffer(buffer, lib.read_bytes) + + +def scan_bytes(buffer: Any) -> pyarrow.StructArray: + """Scan metadata, cell byte ranges and references without decoding geometry.""" + return _read_buffer(buffer, lib.scan_bytes) + + +def read_cells_bytes( + buffer: Any, ranges: Sequence[Sequence[int]] | numpy.ndarray +) -> pyarrow.StructArray: + """Read complete cells in the requested order from half-open byte ranges. + + ``ranges`` has shape (N, 2) and unsigned 64-bit integer offsets. Signed + integer inputs are accepted after checking that every offset is nonnegative. + An empty sequence requests metadata with no cells. + """ + values = numpy.asarray(ranges) + if values.shape == (0,): + values = numpy.empty((0, 2), dtype=numpy.uint64) + if values.ndim != 2 or values.shape[1] != 2: + raise ValueError("Cell ranges must have shape (N, 2)") + if values.dtype.kind not in "iu": + raise TypeError("Cell offsets must be integers") + if values.dtype.kind == "i" and numpy.any(values < 0): + raise ValueError("Cell offsets must be nonnegative") + values = numpy.require( + values, dtype=numpy.uint64, requirements=("C_CONTIGUOUS", "ALIGNED") + ) + with ffi.from_buffer("const uint64_t[]", values) as offsets: + return _read_buffer( + buffer, + lambda data, size, array, schema: lib.read_cells_bytes( + data, size, offsets, len(values), array, schema + ), + ) diff --git a/pheon/basic.py b/pheon/basic.py new file mode 100644 index 0000000..51100aa --- /dev/null +++ b/pheon/basic.py @@ -0,0 +1,70 @@ +from collections.abc import Sequence +from typing import Any + +import numpy +from numpy.typing import NDArray + +from ._native import lib, ffi + + +CONV_TABLE_i16 = { + numpy.float64: lib.f64_to_i16, + numpy.float32: lib.f32_to_i16, + numpy.int64: lib.i64_to_i16, + numpy.int32: lib.i32_to_i16, + numpy.int16: lib.i16_to_i16, + numpy.uint64: lib.u64_to_i16, + numpy.uint32: lib.u32_to_i16, + numpy.uint16: lib.u16_to_i16, +} + +CONV_TABLE_i32 = { + numpy.float64: lib.f64_to_i32, + numpy.float32: lib.f32_to_i32, + numpy.int64: lib.i64_to_i32, + numpy.int32: lib.i32_to_i32, + numpy.uint64: lib.u64_to_i32, + numpy.uint32: lib.u32_to_i32, +} + + +def _pack_int( + data: NDArray[numpy.integer] | Sequence[int] | int, + width: int, + converters: dict[type[numpy.generic], Any], +) -> bytes: + arr = numpy.atleast_1d(data) + target_dtype = numpy.dtype(f">i{width}") + + convert = converters.get(arr.dtype.type) if arr.dtype.isnative else None + if convert is not None: + arr = numpy.require( + arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") + ) + if arr is data: + arr = numpy.array(arr, copy=True) + + with ffi.from_buffer( + ffi.typeof(convert).args[0], arr, require_writable=True + ) as buf: + result = convert(buf, arr.size) + if result != 0: + raise ValueError(f"Invalid value for conversion to Int{width}: {result}") + + return arr.reshape(-1).view(target_dtype)[:: arr.itemsize // width].tobytes() + + if arr.dtype == target_dtype: + return arr.tobytes() + + limit = 1 << (width * 8 - 1) + if (arr > limit - 1).any() or (arr < -limit).any(): + raise ValueError(f"int{width} data out of range: {arr}") + return arr.astype(target_dtype).tobytes() + + +def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 2, CONV_TABLE_i16) + + +def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: + return _pack_int(data, 4, CONV_TABLE_i32) diff --git a/klamath_rs_ext/py.typed b/pheon/py.typed similarity index 100% rename from klamath_rs_ext/py.typed rename to pheon/py.typed diff --git a/pyproject.toml b/pyproject.toml index 06aa1af..aaa9512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,16 @@ [build-system] -requires = ["maturin>1.0,<2.0"] -build-backend = "maturin" +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" [project] -name = "klamath_rs_ext" -description = "Compiled extensions for klamath GDS library" -#readme = "README.md" -#license = { file = "LICENSE.md" } +name = "pheon" +description = "Native GDSII parsing, Arrow interchange, and structural layout review" +readme = "README.md" +license = "GPL-3.0-only" +license-files = ["LICENSE.md"] authors = [ { name="Jan Petykiewicz", email="jan@mpxd.net" }, ] -homepage = "https://mpxd.net/code/jan/klamath-rs" -repository = "https://mpxd.net/code/jan/klamath-rs" classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 4 - Beta", @@ -20,18 +19,26 @@ classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: Manufacturing", "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", ] requires-python = ">=3.11" -#include = [ -# "LICENSE.md" -# ] dynamic = ["version"] dependencies = [ "cffi", + "numpy", + "pyarrow>=14", ] -[tool.maturin] -bindings = "cffi" +[project.optional-dependencies] +arrow = [] # Compatibility with existing requirements for pheon[arrow]. + +[project.urls] +Homepage = "https://mpxd.net/code/jan/pheon" +Repository = "https://mpxd.net/code/jan/pheon" + +[tool.setuptools] +packages = ["pheon"] + +[tool.setuptools.package-data] +pheon = ["py.typed", "_ffi.h"] diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..292fe49 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7a22f81 --- /dev/null +++ b/setup.py @@ -0,0 +1,79 @@ +"""Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tomllib + +from setuptools import Distribution, setup +from setuptools.command.build_py import build_py +from wheel.bdist_wheel import bdist_wheel + + +ROOT = Path(__file__).resolve().parent +VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] +BUILD_TARGET = os.environ.get("PHEON_BUILD_TARGET") +WINDOWS_TARGET = "x86_64-pc-windows-gnu" + +if BUILD_TARGET is not None and BUILD_TARGET != WINDOWS_TARGET: + raise ValueError(f"PHEON_BUILD_TARGET must be {WINDOWS_TARGET}, got {BUILD_TARGET!r}") + + +if not (ROOT / "Cargo.lock").is_file(): + raise FileNotFoundError("Source builds require the supplied Cargo.lock") + + +class CargoBuild(build_py): + def run(self): + super().run() + command = ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"] + if BUILD_TARGET: + command.extend(["--target", BUILD_TARGET]) + result = subprocess.run( + command, + cwd=ROOT, capture_output=True, text=True, + ) + sys.stderr.write(result.stderr) + library = None + for line in result.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + print(line, file=sys.stderr) + continue + if message.get("reason") == "compiler-message": + sys.stderr.write(message["message"].get("rendered") or "") + if (message.get("reason") == "compiler-artifact" + and message["target"]["name"] == "pheon" + and "cdylib" in message["target"]["crate_types"]): + for filename in message["filenames"]: + if Path(filename).suffix in (".so", ".dylib", ".dll"): + library = Path(filename) + result.check_returncode() + if library is None or (BUILD_TARGET and library.name != "pheon.dll"): + raise RuntimeError("Cargo did not produce the expected Pheon shared library") + shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) + + +class NativeDistribution(Distribution): + def has_ext_modules(self): + return True + + +class NativeWheel(bdist_wheel): + def get_tag(self): + # CFFI ABI mode needs a platform tag and no CPython extension ABI. + if BUILD_TARGET == WINDOWS_TARGET: + return "py3", "none", "win_amd64" + _, _, platform = super().get_tag() + return "py3", "none", platform + + +setup( + version=VERSION, + distclass=NativeDistribution, + cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel}, + options={"build": {"build_base": f"build/{BUILD_TARGET}"}} if BUILD_TARGET else {}, +) diff --git a/src/basic.rs b/src/basic.rs index 1019be1..e4de7a0 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -1,7 +1,6 @@ /// /// Functionality for parsing and writing basic data types /// -use byteorder::{ByteOrder, BigEndian}; use std::io; use std::fmt; @@ -15,15 +14,15 @@ pub enum ErrType { Failed(String), } -pub fn fail(input: &[u8], msg: String) -> IResult { +pub fn fail(input: &[u8], msg: String) -> IResult<'_, O> { Err((input, ErrType::Failed(msg))) } -pub fn incomplete(input: &[u8], size: Option) -> IResult { +pub fn incomplete(input: &[u8], size: Option) -> IResult<'_, O> { Err((input, ErrType::Incomplete(size))) } -pub fn take_bytes>(input: &[u8], count: CC) -> IResult<&[u8]> { +pub fn take_bytes>(input: &[u8], count: CC) -> IResult<'_, &[u8]> { let cc = count.into(); if input.len() < cc { incomplete(input, Some(cc)) @@ -37,22 +36,26 @@ pub fn take_bytes>(input: &[u8], count: CC) -> IResult<&[u8]> { /* * Parse functions */ -pub fn parse_u16(input: &[u8]) -> IResult { - let (input, buf) = take_bytes(input, 2_usize)?; - let val = BigEndian::read_u16(&buf); - Ok((input, val)) +fn take_array(input: &[u8]) -> IResult<'_, [u8; N]> { + match input.split_first_chunk::() { + Some((bytes, remaining)) => Ok((remaining, *bytes)), + None => incomplete(input, Some(N)), + } } -pub fn parse_int2(input: &[u8]) -> IResult { - let (input, buf) = take_bytes(input, 2_usize)?; - let val = BigEndian::read_i16(&buf); - Ok((input, val)) +pub fn parse_u16(input: &[u8]) -> IResult<'_, u16> { + let (input, bytes) = take_array(input)?; + Ok((input, u16::from_be_bytes(bytes))) } -pub fn parse_int4(input: &[u8]) -> IResult { - let (input, buf) = take_bytes(input, 4_usize)?; - let val = BigEndian::read_i32(&buf); - Ok((input, val)) +pub fn parse_int2(input: &[u8]) -> IResult<'_, i16> { + let (input, bytes) = take_array(input)?; + Ok((input, i16::from_be_bytes(bytes))) +} + +pub fn parse_int4(input: &[u8]) -> IResult<'_, i32> { + let (input, bytes) = take_array(input)?; + Ok((input, i32::from_be_bytes(bytes))) } /// Convert GDS REAL8 to IEEE float64 @@ -67,41 +70,62 @@ pub fn decode_real8(int: u64) -> f64 { mant * 2_f64.powi(exp2) } -pub fn parse_real8(input: &[u8]) -> IResult { - let (input, buf) = take_bytes(input, 8_usize)?; - let data = BigEndian::read_u64(&buf); +pub fn parse_real8(input: &[u8]) -> IResult<'_, f64> { + let (input, bytes) = take_array(input)?; + let data = u64::from_be_bytes(bytes); Ok((input, decode_real8(data))) } -pub fn parse_datetime(input: &[u8]) -> IResult<[i16; 6]> { +pub fn parse_datetime(input: &[u8]) -> IResult<'_, [i16; 6]> { let mut buf = [0_i16; 6]; - let mut input = input; - for ii in 0..6 { - (input, buf[ii]) = parse_int2(input)?; + let (mut remaining, year) = parse_int2(input)?; + let Some(year) = year.checked_add(1900) else { + return fail( + input, + "Timestamp year exceeds the Int16 range after adding 1900".to_string(), + ); + }; + buf[0] = year; + for bb in &mut buf[1..] { + (remaining, *bb) = parse_int2(remaining)?; } - buf[0] += 1900; // Year is from 1900 - Ok((input, buf)) + Ok((remaining, buf)) +} + +pub(crate) fn decode_utf8(bytes: Vec) -> Result { + String::from_utf8(bytes) + .map_err(|error| ErrType::Failed(format!("Invalid UTF-8 string: {}", error.utf8_error()))) +} + +pub(crate) fn decode_name(mut bytes: &[u8]) -> Result<&str, String> { + while let Some(unpadded) = bytes.strip_suffix(&[0]) { + bytes = unpadded; + } + if bytes.contains(&0) { + return Err("name contains an embedded NUL".to_string()); + } + std::str::from_utf8(bytes).map_err(|error| format!("Invalid UTF-8 string: {error}")) } -pub fn parse_bitarray(input: &[u8]) -> IResult<[bool; 16]> { +pub fn parse_bitarray(input: &[u8]) -> IResult<'_, [bool; 16]> { let mut bits = [false; 16]; let (input, val) = parse_int2(input)?; - for ii in 0..16 { - bits[ii] = ((val >> (16 - 1 - ii)) & 0x01) == 1; + for (ii, bit) in bits.iter_mut().enumerate() { + *bit = ((val >> (16 - 1 - ii)) & 0x01) == 1; } Ok((input, bits)) } -pub fn parse_ascii(input: &[u8], length: u16) -> IResult> { - let length = length as usize; +pub fn parse_ascii(input: &[u8], length: u16) -> IResult<'_, Vec> { let (input, data) = take_bytes(input, length)?; - let last = data[length - 1]; - let true_length = if last == 0 { length - 1 } else { length }; - let vec = data[..true_length].to_vec(); - Ok((input, vec)) + Ok((input, unpad_ascii(data).to_vec())) +} + +pub(crate) fn unpad_ascii(data: &[u8]) -> &[u8] { + data.strip_suffix(&[0]).unwrap_or(data) } @@ -111,27 +135,28 @@ pub fn parse_ascii(input: &[u8], length: u16) -> IResult> { pub fn bitarray2int(bits: &[bool; 16]) -> u16 { let mut int: u16 = 0; - for ii in 0..16 { - int |= (bits[ii] as u16) << (16 - 1 - ii); + for (ii, bit) in bits.iter().enumerate() { + int |= (*bit as u16) << (16 - 1 - ii); } int } pub fn pack_bitarray(buf: &mut [u8], bits: &[bool; 16]) { - BigEndian::write_u16(buf, bitarray2int(bits)); + buf[..2].copy_from_slice(&bitarray2int(bits).to_be_bytes()); } pub fn pack_int2(buf: &mut [u8], int: i16) { - BigEndian::write_i16(buf, int); + buf[..2].copy_from_slice(&int.to_be_bytes()); } pub fn pack_int4(buf: &mut [u8], int: i32) { - BigEndian::write_i32(buf, int); + buf[..4].copy_from_slice(&int.to_be_bytes()); } pub fn pack_real8(buf: &mut [u8], fnum: f64) -> Result<(), FloatTooBigError> { - BigEndian::write_u64(buf, encode_real8(fnum)?); + let bytes = encode_real8(fnum)?.to_be_bytes(); + buf[..8].copy_from_slice(&bytes); Ok(()) } @@ -167,86 +192,42 @@ impl fmt::Display for FloatTooBigError { } } -/// Convert from float64 to GDS REAL8 representation. +/// Convert float64 to a normalized GDS REAL8 representation. +/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0. +/// Nonfinite values and magnitudes at or above 2^252 return an error. pub fn encode_real8(fnum: f64) -> Result { - // Split the ieee float bitfields let ieee = fnum.to_bits(); + let ieee_exp = ((ieee >> 52) & 0x7ff) as i32; + if ieee_exp == 0x7ff { + return Err(FloatTooBigError { float_value: fnum }); + } + + let exp2 = ieee_exp - 1023; + if exp2 < -260 { + // This also handles both signed zeros and every IEEE subnormal. + return Ok(0); + } + + let gds_exp = exp2.div_euclid(4) + 65; + if gds_exp > 0x7f { + return Err(FloatTooBigError { float_value: fnum }); + } + + // The 53-bit IEEE significand needs only 0..3 left shifts to become + // a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8. + let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52); + let mantissa = significand << exp2.rem_euclid(4); let sign = ieee & 0x8000_0000_0000_0000; - let ieee_exp = (ieee >> 52) as i32 & 0x7ff; - let ieee_mant = ieee & 0x000f_ffff_ffff_ffff; - - let subnorm = (ieee_exp == 0) & (ieee_mant != 0); - if (ieee_exp == 0) & (ieee_mant == 0) { - return Ok(0) - } - - // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) - // IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022) - // GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64) - // = (gds_mant / 2^56) * 2^(4 * gds_exp - 256) - - // Convert exponent. - let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII) - let mut exp16 = exp2 / 4; - let rest = exp2 % 4; - - // Compensate for exponent coarseness - let comp = rest != 0; - let mut shift; - if comp { - exp16 += 1; - shift = 4 - rest; - } else { - shift = rest; - } - shift -= 3; // account for gds bit position - - // add leading one - let mut gds_mant_unshifted = ieee_mant; - if !subnorm { - gds_mant_unshifted += 0x10_0000_0000_0000; - } - - let mut gds_mant = if shift > 0 { - gds_mant_unshifted >> shift - } else { - gds_mant_unshifted << -shift - }; - - // add gds exponent bias - let mut gds_exp = exp16 + 64; - - if gds_exp < -14 { - // number is too small - return Ok(0) - } - - let neg_biased = gds_exp < 0; - if neg_biased { - gds_mant >>= gds_exp * 4; - gds_exp = 0; - } - - let too_big = (gds_exp > 0x7f) & !subnorm; - if too_big { - return Err(FloatTooBigError{float_value: fnum}); - } - - let gds_exp_bits = (gds_exp as u64) << 56; - - let real8 = sign | gds_exp_bits | gds_mant; - Ok(real8) + Ok(sign | ((gds_exp as u64) << 56) | mantissa) } - #[cfg(test)] mod tests { #[test] fn test_parse_bitarray() { use crate::basic::parse_bitarray; - //assert!(parse_bitarray(b"59") == 13625); assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]); assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]); diff --git a/src/bin/pheon.rs b/src/bin/pheon.rs new file mode 100644 index 0000000..72db205 --- /dev/null +++ b/src/bin/pheon.rs @@ -0,0 +1,568 @@ +#[path = "pheon/color.rs"] +mod color; + +use flate2::read::MultiGzDecoder; +use pheon::review::report::{ReportInputs, ReportOptions, failure_report}; +use pheon::review::{ + Comparison, Diagnostic, Document, Layer, Outcome, RevisionPair, Side, TextComparison, + TextReportOptions, TextStyle, + names::{parse_layer, parse_layer_name, resolve_layer_name, write_layer_names}, + selection::{DisplayFilter, Scope}, + write_record_dump, +}; +use std::{ + collections::BTreeMap, + ffi::{OsStr, OsString}, + fmt::Write as _, + fs, + io::{self, Read, Write}, + path::PathBuf, + process::ExitCode, + thread, +}; +use termcolor::ColorChoice; + +const HELP: &str = "Usage: + pheon dump FILE [--records] [--color auto|always|never] + pheon diff OLD NEW [--top NAME] [--cell NAME] [--layer L/T|NAME]\n [--show-cell NAME] [--show-layer L/T|NAME] [--hierarchy ascii|mermaid|none|graph]\n [--details] [--max-changes N] [--depth N] [--format text|json] + [--color auto|always|never] + pheon hierarchy FILE [--format ascii|mermaid] [--depth N] [--top NAME]\n pheon git-diff [OPTIONS] -- PATH OLD OLD_OID OLD_MODE NEW NEW_OID NEW_MODE + pheon name-layers INPUT OUTPUT --cell NAME --carrier L/T --name L:T:NAME ... + +Structural comparison: ignore timestamps and element order, normalize polygon start/winding, +preserve duplicates, compare all retained fields exactly. No geometric XOR or tolerance. +Gzip-compressed GDSII input is recognized by its header. +Exit status: 0 equal/success, 1 different, 2 invalid/unsupported/incomplete. +"; + +fn read_input(path: &OsStr) -> Result, Diagnostic> { + let io_error = + |error: io::Error| Diagnostic::new("io_error", format!("{path:?}: {error}"), None); + let mut file = fs::File::open(path).map_err(io_error)?; + let mut bytes = Vec::new(); + (&mut file) + .take(2) + .read_to_end(&mut bytes) + .map_err(io_error)?; + if bytes == b"\x1f\x8b" { + let mut decoded = Vec::new(); + MultiGzDecoder::new(bytes.as_slice().chain(file)) + .read_to_end(&mut decoded) + .map_err(|error| { + Diagnostic::new( + "invalid_input", + format!("{path:?}: gzip decompression failed: {error}"), + None, + ) + })?; + return Ok(decoded); + } + file.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + +fn value(args: &[OsString], index: &mut usize) -> Result { + *index += 1; + args.get(*index) + .and_then(|s| s.to_str()) + .map(str::to_owned) + .ok_or_else(|| "missing UTF-8 option value".to_string()) +} + +fn name_layers(args: &[OsString]) -> Result { + if args.len() < 2 { + return Err(HELP.to_string()); + } + let (mut cell, mut carrier) = (None, None); + let mut names = BTreeMap::new(); + let mut i = 2; + while i < args.len() { + match args[i].to_str() { + Some("--cell") => cell = Some(value(args, &mut i)?), + Some("--carrier") => carrier = Some(parse_layer(&value(args, &mut i)?)?), + Some("--name") => { + let (pair, name) = parse_layer_name(&value(args, &mut i)?)?; + if names.get(&pair).is_some_and(|old| old != &name) { + return Err("conflicting --name arguments".to_string()); + } + names.insert(pair, name); + } + _ => { + return Err( + "name-layers requires --cell, --carrier and --name arguments".to_string(), + ); + } + } + i += 1; + } + if names.is_empty() { + return Err("provide at least one --name".to_string()); + } + let input = read_input(&args[0]).map_err(|error| error.message)?; + let output = write_layer_names( + &input, + &cell.ok_or("provide --cell")?, + carrier.ok_or("provide --carrier")?, + &names, + )?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[1]) + .map_err(|e| format!("{:?}: {e}", args[1]))?; + if let Err(e) = file.write_all(&output).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(&args[1]); + return Err(e.to_string()); + } + Ok(0) +} + +fn stdout(output: &str) -> Result<(), String> { + io::stdout() + .lock() + .write_all(output.as_bytes()) + .map_err(|e| e.to_string()) +} + +fn json_stdout( + write: impl FnOnce(&mut io::BufWriter>) -> serde_json::Result<()>, +) -> Result<(), String> { + let mut output = io::BufWriter::new(io::stdout().lock()); + write(&mut output).map_err(|e| e.to_string())?; + output + .write_all(b"\n") + .and_then(|()| output.flush()) + .map_err(|e| e.to_string()) +} + +fn parse_revision(bytes: Option<&[u8]>, side: Side) -> Result>, Diagnostic> { + bytes + .map(|bytes| { + let mut document = Document::parse(bytes) + .map_err(|message| Diagnostic::new("invalid_input", message, Some(side)))?; + // Comparison uses the parsed cells and their source offsets. + document.records = Vec::new(); + Ok(document) + }) + .transpose() +} + +#[derive(Default)] +struct Selectors { + scope: Scope, + filter: DisplayFilter, + layer_name: Option, + show_layer_name: Option, +} + +fn layer_argument(value: String) -> (Option, Option) { + match parse_layer(&value) { + Ok(pair) => (Some(pair), None), + Err(_) => (None, Some(value)), + } +} + +impl Selectors { + fn resolve_layers( + &mut self, + resolve: impl Fn(&str) -> Result, + ) -> Result<(), Diagnostic> { + for (flag, code, name, pair) in [ + ( + "--layer", + "invalid_scope", + &self.layer_name, + &mut self.scope.layer, + ), + ( + "--show-layer", + "command_error", + &self.show_layer_name, + &mut self.filter.layer, + ), + ] { + if let Some(name) = name { + *pair = Some( + resolve(name) + .map_err(|error| Diagnostic::new(code, format!("{flag} {error}"), None))?, + ); + } + } + Ok(()) + } +} + +fn prepare_comparison( + old_path: Option<&OsStr>, + new_path: Option<&OsStr>, + selectors: &mut Selectors, + compare: impl FnOnce(&Document<'_>, &Document<'_>) -> T, + compare_pair: impl FnOnce(RevisionPair<'_>) -> T, +) -> Result { + let read = |path: Option<&OsStr>, side| { + path.map(|path| { + read_input(path).map_err(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic + }) + }) + .transpose() + }; + let old_bytes = read(old_path, Side::Old)?; + let new_bytes = read(new_path, Side::New)?; + const PARALLEL_PARSE_MIN_BYTES: usize = 2 * 1024 * 1024; + let large_pair = old_bytes + .as_deref() + .zip(new_bytes.as_deref()) + .filter(|(old, new)| old.len().min(new.len()) >= PARALLEL_PARSE_MIN_BYTES); + if selectors.scope == Scope::default() + && selectors.layer_name.is_none() + && let Some((old, new)) = large_pair + { + let pair = RevisionPair::parse(old, new)?; + selectors.resolve_layers(|name| pair.resolve_layer_name(name))?; + return Ok(compare_pair(pair)); + } + let parse_old = || parse_revision(old_bytes.as_deref(), Side::Old); + let parse_new = || parse_revision(new_bytes.as_deref(), Side::New); + let parallel = if large_pair.is_some() + && thread::available_parallelism().is_ok_and(|count| count.get() > 1) + { + thread::scope(|scope| { + let worker = thread::Builder::new().spawn_scoped(scope, parse_old).ok()?; + let new = parse_new(); + Some((worker.join().unwrap(), new)) + }) + } else { + None + }; + // Report an old-side parse error first if both inputs are malformed. + let (old, new) = match parallel { + Some((old, new)) => (old?, new?), + None => (parse_old()?, parse_new()?), + }; + let (old, new) = match (old, new) { + (Some(old), Some(new)) => (old, new), + (None, Some(new)) => (new.empty_like(), new), + (Some(old), None) => { + let new = old.empty_like(); + (old, new) + } + (None, None) => { + return Err(Diagnostic::new( + "invalid_input", + "both Git revisions are absent", + None, + )); + } + }; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&old.layer_names, &new.layer_names]))?; + let scope = &selectors.scope; + if scope == &Scope::default() { + return Ok(compare(&old, &new)); + } + scope + .validate_pair(&old, &new) + .map_err(|message| Diagnostic::new("invalid_scope", message, None))?; + let old = old + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::Old)))?; + let new = new + .selected(scope, true) + .map_err(|message| Diagnostic::new("invalid_scope", message, Some(Side::New)))?; + Ok(compare(&old, &new)) +} + +fn json_diff( + paths: &[OsString], + selectors: &mut Selectors, + max_objects: Option, + hierarchy: bool, +) -> Result { + let inputs = ReportInputs { + old: Some(PathBuf::from(&paths[0])), + new: Some(PathBuf::from(&paths[1])), + }; + let comparison = prepare_comparison( + Some(&paths[0]), + Some(&paths[1]), + selectors, + Comparison::between, + |pair| pair.compare(), + ); + let options = ReportOptions { + filter: selectors.filter.clone(), + max_objects, + hierarchy, + }; + let (result, outcome) = match comparison { + Ok(diff) => ( + json_stdout(|output| diff.write_json_report(output, &inputs, &options)), + diff.outcome(), + ), + Err(error) => { + let report = failure_report(&inputs, &selectors.scope, &options, &error); + ( + json_stdout(|output| serde_json::to_writer_pretty(output, &report)), + Outcome::Failed, + ) + } + }; + if let Err(error) = result { + // Once output starts, never append a second report on a write failure. + eprintln!("pheon: {error}"); + return Ok(2); + } + Ok(outcome.exit_code()) +} + +fn run(args: &[OsString]) -> Result { + if args.len() == 1 && matches!(args[0].to_str(), Some("--version" | "-V")) { + stdout(&format!("pheon {}\n", env!("CARGO_PKG_VERSION")))?; + return Ok(0); + } + if args.is_empty() || args[0] == "--help" || args[0] == "-h" { + stdout(HELP)?; + return Ok(0); + } + let command = args[0].to_str().ok_or("invalid command")?; + if command == "name-layers" { + return name_layers(&args[1..]); + } + let expected = match command { + "dump" | "hierarchy" => 1, + "diff" => 2, + "git-diff" => 7, + _ => return Err(HELP.to_string()), + }; + let compare = command == "diff" || command == "git-diff"; + let color_supported = compare || command == "dump"; + let mut color_choice = ColorChoice::Auto; + let mut selectors = Selectors::default(); + let mut raw = false; + let mut format = if compare { + "none".to_string() + } else { + "ascii".to_string() + }; + let mut max_changes = 100; + let mut details = false; + let mut explicit_max = None; + let mut output_format = "text".to_string(); + let mut depth = 8; + let mut explicit_depth = false; + let mut paths = Vec::new(); + let mut literal = false; + let mut i = 1; + while i < args.len() { + if literal { + paths.push(args[i].clone()); + i += 1; + continue; + } + match args[i].to_str() { + Some("--") => literal = true, + Some("--help" | "-h") => { + stdout(HELP)?; + return Ok(0); + } + Some("--records") if command == "dump" => raw = true, + Some("--color") if color_supported => { + color_choice = color::parse_choice(&value(args, &mut i)?)?; + } + Some(s) if color_supported && s.starts_with("--color=") => { + color_choice = color::parse_choice(&s["--color=".len()..])?; + } + Some("--top") => selectors.scope.top = Some(value(args, &mut i)?), + Some("--cell") => selectors.scope.cell = Some(value(args, &mut i)?), + Some("--layer") => { + (selectors.scope.layer, selectors.layer_name) = + layer_argument(value(args, &mut i)?); + } + Some("--show-cell") if compare => selectors.filter.cell = Some(value(args, &mut i)?), + Some("--show-layer") if compare => { + (selectors.filter.layer, selectors.show_layer_name) = + layer_argument(value(args, &mut i)?); + } + Some("--details") if compare => details = true, + Some("--hierarchy") if compare => format = value(args, &mut i)?, + Some("--format") if command == "hierarchy" => format = value(args, &mut i)?, + Some("--format") if command == "diff" => output_format = value(args, &mut i)?, + Some("--max-changes") if compare => { + max_changes = value(args, &mut i)? + .parse() + .map_err(|_| "invalid change limit")?; + explicit_max = Some(max_changes); + } + Some("--depth") if command != "dump" => { + depth = value(args, &mut i)?.parse().map_err(|_| "invalid depth")?; + explicit_depth = true; + } + Some(s) if s.starts_with('-') => return Err(format!("unknown option {s:?}")), + _ => paths.push(args[i].clone()), + } + i += 1; + } + if paths.len() != expected && !(command == "git-diff" && paths.len() == 9) { + return Err(HELP.to_string()); + } + if !matches!(output_format.as_str(), "text" | "json") { + return Err("diff format must be text or json".to_string()); + } + if output_format == "json" { + if !matches!(format.as_str(), "none" | "graph") || explicit_depth { + return Err( + "JSON accepts --hierarchy graph|none; --depth applies to text output".to_string(), + ); + } + return json_diff(&paths, &mut selectors, explicit_max, format == "graph"); + } + if !matches!(format.as_str(), "ascii" | "mermaid" | "none") + || (command == "hierarchy" && format == "none") + { + return Err("format must be ascii or mermaid (diff also accepts none)".to_string()); + } + let read = |path: &OsString| read_input(path).map_err(|error| error.message); + let printer = color::Printer::new(color_choice); + if raw { + if selectors.scope != Scope::default() || selectors.layer_name.is_some() { + return Err("record dumps do not accept object scope selectors".to_string()); + } + let mut report = printer.report(); + write_record_dump(&mut report, &read(&paths[0])?)?; + printer.print(&report).map_err(|error| error.to_string())?; + return Ok(0); + } + if !compare { + let bytes = read(&paths[0])?; + let doc = Document::parse(&bytes)?; + selectors + .resolve_layers(|name| resolve_layer_name(name, [&doc.layer_names])) + .map_err(|error| error.message)?; + let doc = doc.selected(&selectors.scope, false)?; + if command == "dump" { + let mut report = printer.report(); + doc.write_canonical_dump(&mut report); + printer.print(&report).map_err(|error| error.to_string())?; + } else { + stdout(&doc.hierarchy(format == "mermaid", depth))?; + for diagnostic in &doc.diagnostics { + eprintln!("INCOMPLETE: {}", diagnostic.message); + } + } + return Ok(if doc.diagnostics.is_empty() { 0 } else { 2 }); + } + let git = command == "git-diff"; + let old_missing = git && (paths[3] == "000000" || paths[3] == "."); + let new_missing = git && (paths[6] == "000000" || paths[6] == "."); + let diff = prepare_comparison( + (!old_missing).then_some(paths[if git { 1 } else { 0 }].as_os_str()), + (!new_missing).then_some(paths[if git { 4 } else { 1 }].as_os_str()), + &mut selectors, + TextComparison::between, + |pair| pair.compare_text(), + ) + .map_err(|error| error.message)?; + let mut output = printer.report(); + if git { + let action = match (old_missing, new_missing) { + (true, false) => " added", + (false, true) => " deleted", + _ => "", + }; + if paths.len() == 9 { + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?} -> {:?}{action}", paths[0], paths[7]), + ); + } else { + output.write_styled( + TextStyle::Heading, + format_args!("layout {:?}{action}", paths[0]), + ); + } + writeln!(output).unwrap(); + if !old_missing && !new_missing && paths[3] != paths[6] { + output.write_styled( + TextStyle::Heading, + format_args!("mode {:?} -> {:?}", paths[3], paths[6]), + ); + writeln!(output).unwrap(); + } + } + let show_library_presence = !old_missing && !new_missing; + diff.write_text_report( + &mut output, + max_changes, + &selectors.filter, + show_library_presence, + TextReportOptions { + details, + ascii_depth: (format == "ascii").then_some(depth), + }, + ); + if format == "mermaid" { + output.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + writeln!(output, "\n```mermaid\n{}```", diff.hierarchy(true, depth)).unwrap(); + } + printer.print(&output).map_err(|error| error.to_string())?; + // Git's external driver expects successful rendering for either equality + // or differences. Actual comparison/input failures remain nonzero. + Ok(if git && diff.outcome().exit_code() == 1 { + 0 + } else { + diff.outcome().exit_code() + }) +} + +fn wants_json(args: &[OsString]) -> bool { + if args.first().is_none_or(|arg| arg != "diff") { + return false; + } + let mut format = None; + let mut i = 1; + while i < args.len() { + match args[i].to_str() { + Some("--") => break, + Some("--format") => { + format = args.get(i + 1).and_then(|v| v.to_str()); + i += 1; + } + Some( + "--top" | "--cell" | "--layer" | "--show-cell" | "--show-layer" | "--hierarchy" + | "--max-changes" | "--depth" | "--color", + ) => i += 1, + _ => (), + } + i += 1; + } + format == Some("json") +} + +fn main() -> ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let json = wants_json(&args); + match run(&args) { + Ok(status) => ExitCode::from(status), + Err(message) => { + if json { + let report = failure_report( + &ReportInputs::default(), + &Scope::default(), + &ReportOptions::default(), + &Diagnostic::new("command_error", message, None), + ); + if let Err(error) = + json_stdout(|output| serde_json::to_writer_pretty(output, &report)) + { + eprintln!("pheon: {error}"); + } + } else { + eprintln!("pheon: {message}"); + } + ExitCode::from(2) + } + } +} diff --git a/src/bin/pheon/color.rs b/src/bin/pheon/color.rs new file mode 100644 index 0000000..82fb933 --- /dev/null +++ b/src/bin/pheon/color.rs @@ -0,0 +1,57 @@ +use pheon::review::{TextReport, TextStyle}; +use std::io::{self, IsTerminal, Write}; +use termcolor::{BufferedStandardStream, Color, ColorChoice, ColorSpec, WriteColor}; + +pub(super) fn parse_choice(value: &str) -> Result { + match value { + "auto" => Ok(ColorChoice::Auto), + "always" => Ok(ColorChoice::Always), + "never" => Ok(ColorChoice::Never), + _ => Err("color must be auto, always, or never".to_string()), + } +} + +pub(super) struct Printer(Option); + +impl Printer { + pub fn new(choice: ColorChoice) -> Self { + if choice == ColorChoice::Never + || (choice == ColorChoice::Auto && !io::stdout().is_terminal()) + { + return Self(None); + } + let stream = BufferedStandardStream::stdout(choice); + Self(stream.supports_color().then_some(stream)) + } + + pub fn report(&self) -> TextReport { + TextReport::new(self.0.is_some()) + } + + pub fn print(self, report: &TextReport) -> io::Result<()> { + let text = report.as_str().as_bytes(); + let Some(mut out) = self.0 else { + return io::stdout().lock().write_all(text); + }; + let mut end = 0; + for (range, style) in report.styles() { + out.write_all(&text[end..range.start])?; + let (color, bold) = match style { + TextStyle::Heading => (Color::Cyan, true), + TextStyle::Layer => (Color::Cyan, false), + TextStyle::Added => (Color::Green, false), + TextStyle::Removed => (Color::Red, false), + TextStyle::Changed => (Color::Yellow, false), + TextStyle::Success => (Color::Green, true), + TextStyle::Warning => (Color::Yellow, true), + TextStyle::Error => (Color::Red, true), + }; + out.set_color(ColorSpec::new().set_fg(Some(color)).set_bold(bold))?; + let written = out.write_all(&text[range.clone()]); + written.and(out.reset())?; + end = range.end; + } + out.write_all(&text[end..])?; + out.flush() + } +} diff --git a/src/elements.rs b/src/elements.rs index b711da2..cfee79c 100644 --- a/src/elements.rs +++ b/src/elements.rs @@ -3,530 +3,913 @@ /// structure references) and associated properties. /// -use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF, +use crate::records::{ DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE, - LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, + LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING, ArrayPlacement, STRANS, MAG, ANGLE, PROPATTR, PROPVALUE, ENDEL, BGNEXTN, ENDEXTN, SNAME, }; use crate::records; -use crate::record::{RecordHeader, Record}; -use crate::basic::{OResult, IResult, fail}; +use crate::record::{Record, RecordFrame}; +use crate::basic::{ErrType, IResult, decode_name, decode_utf8, fail}; +use std::string::String; use std::collections::HashMap; -use std::io::Write; + +use arrow_array::builder::{ + StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder, + Int32Builder, Int16Builder, UInt64Builder, UInt32Builder, UInt8Builder, Float64Builder, + }; + + +pub(crate) type DListBuilder = ListBuilder>; +pub(crate) type FListBuilder = FixedSizeListBuilder>; + +struct PropertyRecord { + key: i16, + value: String, +} + +struct BoundaryRecord { + layer_id: u32, + vertices: Vec, + properties: Vec, +} + +struct BoundaryBatchRecord { + layer_id: u32, + vertices: Vec, + vertex_offsets: Vec, +} + +struct RectBatchRecord { + layer_id: u32, + rects: Vec, +} + +struct SRefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + properties: Vec, +} + +struct ARefRecord { + target_id: u32, + invert_y: bool, + scale: f64, + angle_rad: f64, + xy: u64, + xy0: u64, + xy1: u64, + counts: u32, + properties: Vec, +} + +#[derive(Clone)] +pub(crate) struct ScanRefRecord { + pub(crate) target_id: u32, + pub(crate) invert_y: bool, + pub(crate) scale: f64, + pub(crate) angle_rad: f64, + pub(crate) xy: u64, + pub(crate) xy0: u64, + pub(crate) xy1: u64, + pub(crate) counts: u32, +} + +enum RefRecord { + SRef(SRefRecord), + ARef(ARefRecord), +} + + +const UNIT_COUNTS: u32 = (1_u32 << 16) | 1_u32; +const ZERO_XY: u64 = 0; + + +fn layer_id_for( + layer: i16, + dtype: i16, + layers: &mut HashMap, + ) -> u32 { + let layer32 = ((layer as u16 as u32) << 16) | (dtype as u16 as u32); + let next_id = layers.len(); + *layers.entry(layer32).or_insert(next_id.try_into().unwrap()) +} + + +fn insert_layer( + layer: i16, + dtype: i16, + layers: &mut HashMap, + struct_builder: &mut StructBuilder, + field_index: usize, + ) { + let id = layer_id_for(layer, dtype, layers); + let layer_builder = struct_builder.field_builder::(field_index).unwrap(); + layer_builder.append_value(id); +} + + +pub(crate) fn read_elements<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + names: &mut HashMap, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let mut input = input; + let mut boundaries = Vec::::new(); + let mut srefs = Vec::::new(); + let mut arefs = Vec::::new(); + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, record.tag, names)?; + match ref_record { + RefRecord::SRef(sref) => srefs.push(sref), + RefRecord::ARef(aref) => arefs.push(aref), + } + (next_input, ()) + }, + records::RTAG_BOUNDARY => { + let (next_input, boundary) = read_boundary(input, layers)?; + boundaries.push(boundary); + (next_input, ()) + }, + records::RTAG_PATH => read_path(input, cell_builder, layers)?, + records::RTAG_NODE | records::RTAG_BOX => { + read_boxnode(input, cell_builder, record.tag, layers)? + } + records::RTAG_TEXT => read_text(input, cell_builder, layers)?, + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; + } + _ => (remaining, ()), + }; + }; + + let mut sref_plain = Vec::::with_capacity(srefs.len() + arefs.len() / 4); + let mut sref_props = Vec::::with_capacity(srefs.len() / 16 + arefs.len() / 64); + for sref in srefs { + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } + + let mut aref_plain = Vec::::with_capacity(arefs.len()); + let mut aref_props = Vec::::with_capacity(arefs.len() / 16); + for aref in arefs { + if aref.counts == UNIT_COUNTS { + let sref = aref_to_sref(aref); + if sref.properties.is_empty() { + sref_plain.push(sref); + } else { + sref_props.push(sref); + } + } else if aref.properties.is_empty() { + aref_plain.push(aref); + } else { + aref_props.push(aref); + } + } + sref_plain.sort_by_key(|sref| sref.target_id); + aref_plain.sort_by_key(|aref| aref.target_id); + + let mut boundary_plain = Vec::::with_capacity(boundaries.len()); + let mut boundary_props = Vec::::with_capacity(boundaries.len() / 16); + for boundary in boundaries { + if boundary.properties.is_empty() { + boundary_plain.push(boundary); + } else { + boundary_props.push(boundary); + } + } + boundary_plain.sort_by_key(|boundary| boundary.layer_id); + + let mut rect_batches = Vec::::new(); + let mut boundary_batches = Vec::::new(); + let mut ii = 0; + while ii < boundary_plain.len() { + let start = ii; + let layer_id = boundary_plain[ii].layer_id; + while ii < boundary_plain.len() && boundary_plain[ii].layer_id == layer_id { + ii += 1; + } + + let group = &boundary_plain[start..ii]; + let rect_capacity = group.len() * 4; + let vertex_capacity = group.iter().map(|boundary| boundary.vertices.len()).sum(); + let mut rect_batch = RectBatchRecord { + layer_id, + rects: Vec::with_capacity(rect_capacity), + }; + let mut boundary_batch = BoundaryBatchRecord { + layer_id, + vertices: Vec::with_capacity(vertex_capacity), + vertex_offsets: Vec::with_capacity(group.len()), + }; + + let mut next_offset: u32 = 0; + for boundary in group { + if let Some(rect) = XY::rectangle(boundary.vertices.as_chunks::<2>().0) { + rect_batch.rects.extend_from_slice(&rect); + } else { + boundary_batch.vertex_offsets.push(next_offset); + boundary_batch.vertices.extend_from_slice(&boundary.vertices); + next_offset += (boundary.vertices.len() / 2) as u32; + } + } + if !rect_batch.rects.is_empty() { + rect_batches.push(rect_batch); + } + if !boundary_batch.vertex_offsets.is_empty() { + boundary_batches.push(boundary_batch); + } + } + + let sref_builder = cell_builder.field_builder::(2).unwrap(); + for sref in sref_plain { + append_sref(sref_builder, sref); + } + sref_builder.append(true); + + let aref_builder = cell_builder.field_builder::(3).unwrap(); + for aref in aref_plain { + append_aref(aref_builder, aref); + } + aref_builder.append(true); + + let sref_props_builder = cell_builder.field_builder::(4).unwrap(); + for sref in sref_props { + append_sref_prop(sref_props_builder, sref); + } + sref_props_builder.append(true); + + let aref_props_builder = cell_builder.field_builder::(5).unwrap(); + for aref in aref_props { + append_aref_prop(aref_props_builder, aref); + } + aref_props_builder.append(true); + + let rect_batches_builder = cell_builder.field_builder::(6).unwrap(); + for rect_batch in rect_batches { + append_rect_batch(rect_batches_builder, rect_batch); + } + rect_batches_builder.append(true); + + let boundary_batches_builder = cell_builder.field_builder::(7).unwrap(); + for boundary_batch in boundary_batches { + append_boundary_batch(boundary_batches_builder, boundary_batch); + } + boundary_batches_builder.append(true); + + let boundary_props_builder = cell_builder.field_builder::(8).unwrap(); + for boundary in boundary_props { + append_boundary_prop(boundary_props_builder, boundary); + } + boundary_props_builder.append(true); + + let paths_builder = cell_builder.field_builder::(9).unwrap(); + paths_builder.append(true); + let nodes_builder = cell_builder.field_builder::(10).unwrap(); + nodes_builder.append(true); + let boxes_builder = cell_builder.field_builder::(11).unwrap(); + boxes_builder.append(true); + let texts_builder = cell_builder.field_builder::(12).unwrap(); + texts_builder.append(true); + + Ok((input, ())) +} + + +pub(crate) fn scan_elements<'a>( + input: &'a [u8], + names: &mut HashMap, + ) -> IResult<'a, Vec> { + let mut input = input; + let mut refs = Vec::::new(); + let input = loop { + let (remaining, record) = RecordFrame::read(input)?; + (input, ()) = match record.tag { + records::RTAG_SREF | records::RTAG_AREF => { + let (next_input, ref_record) = read_ref(input, record.tag, names)?; + refs.push(scan_ref_from_record(ref_record)); + (next_input, ()) + }, + records::RTAG_BOUNDARY + | records::RTAG_PATH + | records::RTAG_NODE + | records::RTAG_BOX + | records::RTAG_TEXT => skip_element(input)?, + records::RTAG_ENDSTR => { + record.decode::()?; + break remaining; + } + _ => (remaining, ()), + }; + }; + refs.sort_by_key(|scan_ref| scan_ref.target_id); + Ok((input, refs)) +} + + +fn append_sref( + sref_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_struct_builder = sref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + sref_struct_builder.append(true); +} + + +fn scan_ref_from_record(ref_record: RefRecord) -> ScanRefRecord { + match ref_record { + RefRecord::SRef(sref) => ScanRefRecord { + target_id: sref.target_id, + invert_y: sref.invert_y, + scale: sref.scale, + angle_rad: sref.angle_rad, + xy: sref.xy, + xy0: ZERO_XY, + xy1: ZERO_XY, + counts: UNIT_COUNTS, + }, + RefRecord::ARef(aref) => ScanRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + xy0: aref.xy0, + xy1: aref.xy1, + counts: aref.counts, + }, + } +} + + +fn skip_element<'a>(input: &'a [u8]) -> IResult<'a, ()> { + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == ENDEL::tag() { + record.decode::()?; + return Ok((remaining, ())); + } + input = remaining; + } +} + + +fn append_aref( + aref_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_struct_builder = aref_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_struct_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_struct_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_struct_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_struct_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_struct_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_struct_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_struct_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_struct_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + aref_struct_builder.append(true); +} + + +fn append_property(props_builder: &mut DListBuilder, property: PropertyRecord) { + let prop_builder = props_builder.values().as_any_mut().downcast_mut::().unwrap(); + let key_builder = prop_builder.field_builder::(0).unwrap(); + key_builder.append_value(property.key); + + let val_builder = prop_builder.field_builder::(1).unwrap(); + val_builder.append_value(property.value); + + prop_builder.append(true); +} + + +fn append_properties( + props_builder: &mut DListBuilder, + properties: Vec, + ) { + for prop in properties { + append_property(props_builder, prop); + } + props_builder.append(true); +} + + +fn append_sref_prop( + sref_props_builder: &mut DListBuilder, + sref: SRefRecord, + ) { + let sref_prop_builder = sref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = sref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(sref.target_id); + let invert_builder = sref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(sref.invert_y); + let scale_builder = sref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(sref.scale); + let angle_builder = sref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(sref.angle_rad); + let xy_builder = sref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(sref.xy); + + let props_builder = sref_prop_builder.field_builder::(5).unwrap(); + append_properties(props_builder, sref.properties); + + sref_prop_builder.append(true); +} + + +fn append_aref_prop( + aref_props_builder: &mut DListBuilder, + aref: ARefRecord, + ) { + let aref_prop_builder = aref_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let target_builder = aref_prop_builder.field_builder::(0).unwrap(); + target_builder.append_value(aref.target_id); + let invert_builder = aref_prop_builder.field_builder::(1).unwrap(); + invert_builder.append_value(aref.invert_y); + let scale_builder = aref_prop_builder.field_builder::(2).unwrap(); + scale_builder.append_value(aref.scale); + let angle_builder = aref_prop_builder.field_builder::(3).unwrap(); + angle_builder.append_value(aref.angle_rad); + let xy_builder = aref_prop_builder.field_builder::(4).unwrap(); + xy_builder.append_value(aref.xy); + let xy0_builder = aref_prop_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(aref.xy0); + let xy1_builder = aref_prop_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(aref.xy1); + let counts_builder = aref_prop_builder.field_builder::(7).unwrap(); + counts_builder.append_value(aref.counts); + + let props_builder = aref_prop_builder.field_builder::(8).unwrap(); + append_properties(props_builder, aref.properties); + + aref_prop_builder.append(true); +} + + +fn aref_to_sref(aref: ARefRecord) -> SRefRecord { + SRefRecord { + target_id: aref.target_id, + invert_y: aref.invert_y, + scale: aref.scale, + angle_rad: aref.angle_rad, + xy: aref.xy, + properties: aref.properties, + } +} + + +fn read_boundary<'a>( + input: &'a [u8], + layers: &mut HashMap, + ) -> IResult<'a, BoundaryRecord> { + let (input, _) = records::BOUNDARY::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + let layer_id = layer_id_for(layer, dtype, layers); + + let (input, xy_iter) = XY::read(input)?; + let mut vertices: Vec = xy_iter.collect(); + vertices.truncate(vertices.len().saturating_sub(2)); + + let (input, properties) = read_properties_vec(input)?; + + Ok((input, BoundaryRecord { layer_id, vertices, properties })) +} + + +fn append_boundary_batch( + boundary_batches_builder: &mut DListBuilder, + boundary_batch: BoundaryBatchRecord, + ) { + let boundary_batch_builder = boundary_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary_batch.layer_id); + + let vertices_builder = boundary_batch_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary_batch.vertices); + vertices_builder.append(true); + + let offsets_builder = boundary_batch_builder.field_builder::(2).unwrap(); + let offset_builder = offsets_builder.values().as_any_mut().downcast_mut::().unwrap(); + offset_builder.append_slice(&boundary_batch.vertex_offsets); + offsets_builder.append(true); + + boundary_batch_builder.append(true); +} + + +fn append_rect_batch( + rect_batches_builder: &mut DListBuilder, + rect_batch: RectBatchRecord, + ) { + let rect_batch_builder = rect_batches_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = rect_batch_builder.field_builder::(0).unwrap(); + layer_builder.append_value(rect_batch.layer_id); + + let rects_builder = rect_batch_builder.field_builder::(1).unwrap(); + let rect_builder = rects_builder.values().as_any_mut().downcast_mut::().unwrap(); + rect_builder.append_slice(&rect_batch.rects); + rects_builder.append(true); + + rect_batch_builder.append(true); +} + + +fn append_boundary_prop( + boundary_props_builder: &mut DListBuilder, + boundary: BoundaryRecord, + ) { + let boundary_prop_builder = boundary_props_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let layer_builder = boundary_prop_builder.field_builder::(0).unwrap(); + layer_builder.append_value(boundary.layer_id); + + let vertices_builder = boundary_prop_builder.field_builder::(1).unwrap(); + let vertex_builder = vertices_builder.values().as_any_mut().downcast_mut::().unwrap(); + vertex_builder.append_slice(&boundary.vertices); + vertices_builder.append(true); + + let props_builder = boundary_prop_builder.field_builder::(2).unwrap(); + append_properties(props_builder, boundary.properties); + + boundary_prop_builder.append(true); +} + + +fn read_path<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let paths_builder = cell_builder.field_builder::(9).unwrap(); + let path_builder = paths_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = records::PATH::read(input)?; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = DATATYPE::read(input)?; + insert_layer(layer, dtype, layers, path_builder, 0); + + let mut path_type = None; + let mut width = None; + let mut bgn_ext = None; + let mut end_ext = None; + + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), + records::RTAG_BGNEXTN => bgn_ext = Some(record.decode::()?), + records::RTAG_ENDEXTN => end_ext = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; + } + let path_type_builder = path_builder.field_builder::(1).unwrap(); + path_type_builder.append_option(path_type); + let ext0_builder = path_builder.field_builder::(2).unwrap(); + ext0_builder.append_option(bgn_ext); + let ext1_builder = path_builder.field_builder::(3).unwrap(); + ext1_builder.append_option(end_ext); + let width_builder = path_builder.field_builder::(4).unwrap(); + width_builder.append_option(width); + + let xys_builder = path_builder.field_builder::(5).unwrap(); + let xy_iter = record.decode::()?; + for xy in xy_iter { + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = path_builder.field_builder::(6).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + path_builder.append(true); + Ok((input, ())) +} + +fn read_boxnode<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + tag: u16, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let field_num = match tag { + records::RTAG_NODE => 10, + records::RTAG_BOX => 11, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let boxnodes_builder = cell_builder.field_builder::(field_num).unwrap(); + let boxnode_builder = boxnodes_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let (input, _) = match tag { + records::RTAG_NODE => records::NODE::read(input)?, + records::RTAG_BOX => records::BOX::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = match tag { + records::RTAG_NODE => NODETYPE::read(input)?, + records::RTAG_BOX => BOXTYPE::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + insert_layer(layer, dtype, layers, boxnode_builder, 0); + + let xys_builder = boxnode_builder.field_builder::(1).unwrap(); + let xy_builder = xys_builder.values().as_any_mut().downcast_mut::().unwrap(); + let (input, xy_iter) = XY::read(input)?; + for xy in xy_iter { + xy_builder.append_value(xy); + } + xys_builder.append(true); + + let props_builder = boxnode_builder.field_builder::(2).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + boxnode_builder.append(true); + Ok((input, ())) +} + +fn read_text<'a>( + input: &'a [u8], + cell_builder: &mut StructBuilder, + layers: &mut HashMap, + ) -> IResult<'a, ()> { + let texts_builder = cell_builder.field_builder::(12).unwrap(); + let text_builder = texts_builder.values().as_any_mut().downcast_mut::().unwrap(); + + let mut path_type = None; + let mut pres_hori = None; + let mut pres_vert = None; + let mut pres_font = None; + let mut invert_y = None; + let mut width = None; + let mut mag = None; + let mut angle_deg = None; + + let (input, layer) = LAYER::skip_and_read(input)?; + let (input, dtype) = TEXTTYPE::read(input)?; + insert_layer(layer, dtype, layers, text_builder, 0); + + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { + // TODO warn if repeat tags? + records::RTAG_PRESENTATION => { + let presentation = record.decode::()?; + pres_hori = Some(presentation[14] as u8 * 2 + presentation[15] as u8); + pres_vert = Some(presentation[12] as u8 * 2 + presentation[13] as u8); + pres_font = Some(presentation[10] as u8 * 2 + presentation[11] as u8); + } + records::RTAG_PATHTYPE => path_type = Some(record.decode::()?), + records::RTAG_WIDTH => width = Some(record.decode::()?), + records::RTAG_STRANS => { + let strans = record.decode::()?; + invert_y = Some(strans[0]); + } + records::RTAG_MAG => mag = Some(record.decode::()?), + records::RTAG_ANGLE => angle_deg = Some(record.decode::()?), + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; + } + + let pres_hori_builder = text_builder.field_builder::(1).unwrap(); + pres_hori_builder.append_option(pres_hori); + let pres_vert_builder = text_builder.field_builder::(2).unwrap(); + pres_vert_builder.append_option(pres_vert); + let pres_font_builder = text_builder.field_builder::(3).unwrap(); + pres_font_builder.append_option(pres_font); + let path_type_builder = text_builder.field_builder::(4).unwrap(); + path_type_builder.append_option(path_type); + let width_builder = text_builder.field_builder::(5).unwrap(); + width_builder.append_option(width); + let inv_builder = text_builder.field_builder::(6).unwrap(); + inv_builder.append_option(invert_y); + let mag_builder = text_builder.field_builder::(7).unwrap(); + mag_builder.append_option(mag); + let angle_builder = text_builder.field_builder::(8).unwrap(); + angle_builder.append_option(angle_deg); + + let mut xy_iter = record.decode::()?; + let (Some(xx), Some(yy)) = (xy_iter.next(), xy_iter.next()) else { + return fail(record.data, "TEXT needs an XY coordinate pair".to_string()); + }; + let xy = pack_point([xx, yy]); + let xy_builder = text_builder.field_builder::(9).unwrap(); + xy_builder.append_value(xy); + + let (input, string_bytes) = STRING::read(input)?; + let string = decode_utf8(string_bytes).map_err(|error| (input, error))?; + let string_builder = text_builder.field_builder::(10).unwrap(); + string_builder.append_value(string); + + let props_builder = text_builder.field_builder::(11).unwrap(); + let (input, ()) = read_properties(input, props_builder)?; + + text_builder.append(true); + Ok((input, ())) +} + + +fn pack_point([x, y]: [i32; 2]) -> u64 { + ((x as u32 as u64) << 32) | (y as u32 as u64) +} + + +fn read_ref<'a>( + input: &'a [u8], + tag: u16, + names: &mut HashMap, + ) -> IResult<'a, RefRecord> { + let (input, _) = match tag { + records::RTAG_SREF => records::SREF::read(input)?, + records::RTAG_AREF => records::AREF::read(input)?, + _ => return fail(input, format!("Unexpected tag {:04x}", tag)), + }; + + let is_aref = tag == records::RTAG_AREF; + let mut invert_y = false; + let mut scale = 1.0; + let mut angle_rad = 0.0; + let mut colrow = None; + + let (input, struct_name_bytes) = SNAME::skip_and_read(input)?; + let struct_name = decode_name(&struct_name_bytes) + .map_err(|error| (input, ErrType::Failed(error)))?; + let next_id = names.len(); + let id = names.entry(struct_name.to_owned()).or_insert(next_id.try_into().unwrap()); + + let (mut input, mut record) = RecordFrame::read(input)?; + while record.tag != records::RTAG_XY { + match record.tag { + records::RTAG_STRANS => { + let strans = record.decode::()?; + invert_y = strans[0]; + } + records::RTAG_MAG => scale = record.decode::()?, + records::RTAG_ANGLE => { + angle_rad = record.decode::()?.to_radians(); + } + records::RTAG_COLROW => { + if !is_aref { + return fail(record.data, "Got a COLROW record inside an SREF".to_string()); + } + COLROW::check_size(record.data.len() as u16) + .map_err(|error| (record.data, ErrType::Failed(error)))?; + colrow = Some(record.data); + } + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), + } + (input, record) = RecordFrame::read(input)?; + } + + let (input, properties) = read_properties_vec(input)?; + + if is_aref { + let Some(colrow) = colrow else { + return fail(input, "AREF without COLROW before XY".to_string()); + }; + let placement = ArrayPlacement::decode(colrow, record.data)?; + let [columns, rows] = placement.counts; + let [origin, column_end, row_end] = placement.points; + + let pitch = |end: i32, origin: i32, count: u16| { + let delta = i64::from(end) - i64::from(origin); + let divisor = i64::from(count); + if delta % divisor != 0 { + return Err((input, crate::basic::ErrType::Failed( + "AREF pitch is not an integer database-unit increment; the Arrow schema cannot represent it exactly".to_string() + ))); + } + let value = delta / divisor; + i32::try_from(value).map_err(|_| (input, crate::basic::ErrType::Failed( + "AREF pitch exceeds signed 32-bit coordinate range".to_string() + ))) + }; + let axis_pitch = |end: [i32; 2], count| -> Result<_, _> { + Ok(pack_point([ + pitch(end[0], origin[0], count)?, + pitch(end[1], origin[1], count)?, + ])) + }; + let counts = (u32::from(columns) << 16) | u32::from(rows); + Ok((input, RefRecord::ARef(ARefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy: pack_point(origin), + xy0: axis_pitch(column_end, columns)?, + xy1: axis_pitch(row_end, rows)?, + counts, + properties, + }))) + } else { + if record.data.len() != 8 { + return fail(record.data, "SREF needs exactly one XY coordinate pair".to_string()); + } + let origin = XY::points(record.data).next().unwrap(); + Ok((input, RefRecord::SRef(SRefRecord { + target_id: *id, + invert_y, + scale, + angle_rad, + xy: pack_point(origin), + properties, + }))) + } +} /// /// Read element properties. /// -/// Assumes PROPATTR records have unique values. +/// Preserves record order and duplicate property attributes. /// Stops reading after consuming ENDEL record. /// -/// Args: -/// stream: Stream to read from. -/// -/// Returns: -/// propattr: -> propvalue mapping -/// -pub fn read_properties(input: &[u8]) -> IResult>> { - let mut properties = HashMap::new(); +fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> { + let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?; + props_builder.append(true); + Ok((input, ())) +} - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != ENDEL::tag() { - if header.tag == PROPATTR::tag() { - let result = PROPATTR::read_data(input, header.data_size)?; - input = result.0; - let key = result.1; - let result = PROPVALUE::read(input)?; - input = result.0; - let value = result.1; - assert!(!properties.contains_key(&key), "Duplicate property key: {}", key); - properties.insert(key, value); + +fn walk_properties<'a>( + input: &'a [u8], + mut on_property: impl FnMut(PropertyRecord), + ) -> IResult<'a, ()> { + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { + records::RTAG_ENDEL => { + record.decode::()?; + return Ok((remaining, ())); + } + records::RTAG_PROPATTR => { + let key = record.decode::()?; + let value_bytes; + (input, value_bytes) = PROPVALUE::read(remaining)?; + let value = decode_utf8(value_bytes).map_err(|error| (input, error))?; + on_property(PropertyRecord { key, value }); + } + _ if record.data.is_empty() => input = remaining, + _ => return fail(record.data, format!("Unexpected tag {:04x}", record.tag)), } - (input, header) = RecordHeader::read(input)?; } +} + + +fn read_properties_vec<'a>(input: &'a [u8]) -> IResult<'a, Vec> { + let mut properties = Vec::new(); + let (input, ()) = walk_properties(input, |property| properties.push(property))?; Ok((input, properties)) } - - -/// -/// Write element properties. -/// -/// This is does _not_ write the ENDEL record. -/// -/// Args: -/// stream: Stream to write to. -/// -pub fn write_properties(ww: &mut W, properties: &HashMap::>) -> OResult { - let mut size = 0; - for (key, value) in properties { - size += PROPATTR::write(ww, key)?; - size += PROPVALUE::write(ww, value)?; - } - Ok(size) -} - -pub trait Element { - /// - /// Read from a stream to construct this object. - /// Consumes up to (and including) the ENDEL record. - /// - fn read(input: &[u8]) -> IResult where Self: Sized; - - /// - /// Write this element to a stream. - /// Finishes with an ENDEL record. - /// - fn write(&self, ww: &mut W) -> OResult; -} - -/// -/// Datastructure representing -/// an instance of a structure (SREF / structure reference) or -/// an array of instances (AREF / array reference). -/// Type is determined by the presence of the `colrow` tuple. -/// -/// Transforms are applied to each individual instance (_not_ -/// to the instance's origin location || array vectors). -/// -#[derive(Debug, Clone)] -pub struct Reference { - /// Name of the structure being referenced. - struct_name: Vec, - /// Whether to mirror the pattern (negate y-values / flip across x-axis). Default false. - invert_y: bool, - /// Scaling factor (default 1) - mag: f64, - /// Rotation (degrees counterclockwise) - angle_deg: f64, - - /// (For SREF) Location in the parent structure corresponding to the instance's origin (0, 0). - /// (For AREF) 3 locations: - /// [`offset`, - /// `offset + col_basis_vector * colrow[0]`, - /// `offset + row_basis_vector * colrow[1]`] - /// which define the first instance's offset and the array's basis vectors. - /// Note that many GDS implementations only support manhattan basis vectors, and some - /// assume a certain axis mapping (e.g. x->columns, y->rows) and "reinterpret" the - /// basis vectors to match it. - xy: Vec, - - /// Number of columns and rows (AREF) || None (SREF) - colrow: Option<(i16, i16)>, - /// Properties associated with this reference. - properties: HashMap::>, -} - -impl Element for Reference { - fn read(input: &[u8]) -> IResult { - let mut invert_y = false; - let mut mag = 1.0; - let mut angle_deg = 0.0; - let mut colrow = None; - let (input, struct_name) = SNAME::skip_and_read(input)?; - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_STRANS => { - let result = STRANS::read_data(input, header.data_size)?; - input = result.0; - invert_y = result.1[0]; - }, - records::RTAG_MAG => - {(input, mag) = MAG::read_data(input, header.data_size)?;}, - records::RTAG_ANGLE => - {(input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, - records::RTAG_COLROW => { - let result = COLROW::read_data(input, header.data_size)?; - input = result.0; - colrow = Some((result.1[0], result.1[1])); - }, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - let (input, properties) = read_properties(input)?; - Ok((input, Reference{ - struct_name: struct_name, - xy: xy, - properties: properties, - colrow: colrow, - invert_y: invert_y, - mag: mag, - angle_deg: angle_deg - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += match self.colrow { - None => SREF::write(ww, &())?, - Some(_) => AREF::write(ww, &())?, - }; - - size += SNAME::write(ww, &self.struct_name)?; - if self.angle_deg != 0.0 || self.mag != 1.0 || self.invert_y { - let strans = { - let mut arr = [false; 16]; - arr[0] = self.invert_y; - arr - }; - size += STRANS::write(ww, &strans)?; - if self.mag != 1.0 { - size += MAG::write(ww, &self.mag)?; - } - if self.angle_deg != 0.0 { - size += ANGLE::write(ww, &self.angle_deg)?; - } - } - - if let Some(cr) = self.colrow { - size += COLROW::write(ww, &vec!{cr.0, cr.1})?; - } - - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - -impl Reference { - pub fn check(&self) { - if self.colrow.is_some() { - assert!(self.xy.len() != 6, "colrow is Some, so expected size-6 xy. Got {:?}", self.xy); - } else { - assert!(self.xy.len() != 2, "Expected size-2 xy. Got {:?}", self.xy); - } - } -} - - -/// -/// Datastructure representing a Boundary element. -/// -#[derive(Debug, Clone)] -pub struct Boundary { - /// (layer, data_type) tuple - layer: (i16, i16), - /// Ordered vertices of the shape. First and last points should be identical. Order x0, y0, x1,... - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Boundary { - fn read(input: &[u8]) -> IResult { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = DATATYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Boundary{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += BOUNDARY::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += DATATYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Path element. -/// -/// If `path_type < 4`, `extension` values are not written. -/// During read, `exension` defaults to (0, 0) even if unused. -/// -#[derive(Debug, Clone)] -pub struct Path { - /// (layer, data_type) tuple - layer: (i16, i16), - /// End-cap type (0: flush, 1: circle, 2: square, 4: custom) - path_type: i16, - /// Path width - width: i32, - /// Extension when using path_type=4. Ignored otherwise. - extension: (i32, i32), - /// Path centerline coordinates. [x0, y0, x1, y1,...] - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Path { - fn read(input: &[u8]) -> IResult { - let mut path_type = 0; - let mut width = 0; - let mut bgn_ext = 0; - let mut end_ext = 0; - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = DATATYPE::read(input)?; - - let (mut input, mut header) = RecordHeader::read(&input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PATHTYPE => - {(input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, - records::RTAG_WIDTH => - {(input, width) = WIDTH::read_data(input, header.data_size)?;}, - records::RTAG_BGNEXTN => - {(input, bgn_ext) = BGNEXTN::read_data(input, header.data_size)?;}, - records::RTAG_ENDEXTN => - {(input, end_ext) = ENDEXTN::read_data(input, header.data_size)?;}, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - }; - (input, header) = RecordHeader::read(&input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - let (input, properties) = read_properties(input)?; - Ok((input, Path{ - layer: (layer, dtype), - xy: xy, - properties: properties, - extension: (bgn_ext, end_ext), - path_type: path_type, - width: width, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += PATH::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += DATATYPE::write(ww, &self.layer.1)?; - if self.path_type != 0 { - size += PATHTYPE::write(ww, &self.path_type)?; - } - if self.width != 0 { - size += WIDTH::write(ww, &self.width)?; - } - - if self.path_type < 4 { - let (bgn_ext, end_ext) = self.extension; - if bgn_ext != 0 { - size += BGNEXTN::write(ww, &bgn_ext)?; - } - if end_ext != 0 { - size += ENDEXTN::write(ww, &end_ext)?; - } - } - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Box element. Rarely used. -/// -#[derive(Debug, Clone)] -pub struct GDSBox { - /// (layer, box_type) tuple - layer: (i16, i16), - /// Box coordinates (5 pairs) - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for GDSBox { - fn read(input: &[u8]) -> IResult { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = BOXTYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, GDSBox{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += BOX::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += BOXTYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - - -/// -/// Datastructure representing a Node element. Rarely used. -/// -#[derive(Debug, Clone)] -pub struct Node { - /// (layer, box_type) tuple - layer: (i16, i16), - /// 1-50 pairs of coordinates. - xy: Vec, - /// Properties for the element. - properties: HashMap::>, -} - -impl Element for Node { - fn read(input: &[u8]) -> IResult { - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = NODETYPE::read(input)?; - let (input, xy) = XY::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Node{ - layer: (layer, dtype), - xy: xy, - properties: properties, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += NODE::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += NODETYPE::write(ww, &self.layer.1)?; - size += XY::write(ww, &self.xy)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} - -/// -/// Datastructure representing a text label. -/// -#[derive(Debug, Clone)] -pub struct Text { - /// (layer, node_type) tuple - layer: (i16, i16), - - /// Bit array. Default all zeros. - /// bits 0-1: 00 left/01 center/10 right - /// bits 2-3: 00 top/01 middle/10 bottom - /// bits 4-5: font number - presentation: [bool; 16], - - /// Default 0 - path_type: i16, - /// Default 0 - width: i32, - /// Vertical inversion. Default false. - invert_y: bool, - /// Scaling factor. Default 1. - mag: f64, - /// Rotation (ccw). Default 0. - angle_deg: f64, - /// Position (1 pair only) - xy: Vec, - /// Text content - string: Vec, - /// Properties for the element. - properties: HashMap::> -} - -impl Element for Text { - fn read(input: &[u8]) -> IResult { - let mut path_type = 0; - let mut presentation = [false; 16]; - let mut invert_y = false; - let mut width = 0; - let mut mag = 1.0; - let mut angle_deg = 0.0; - let (input, layer) = LAYER::skip_and_read(input)?; - let (input, dtype) = TEXTTYPE::read(input)?; - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_XY { - match header.tag { - records::RTAG_PRESENTATION => - {(input, presentation) = PRESENTATION::read_data(input, header.data_size)?;}, - records::RTAG_PATHTYPE => - {(input, path_type) = PATHTYPE::read_data(input, header.data_size)?;}, - records::RTAG_WIDTH => - {(input, width) = WIDTH::read_data(input, header.data_size)?;}, - records::RTAG_STRANS => { - let result = STRANS::read_data(input, header.data_size)?; - input = result.0; - invert_y = result.1[0]; - }, - records::RTAG_MAG => - {(input, mag) = MAG::read_data(input, header.data_size)?;}, - records::RTAG_ANGLE => - {(input, angle_deg) = ANGLE::read_data(input, header.data_size)?;}, - _ => - return fail(input, format!("Unexpected tag {:04x}", header.tag)), - } - (input, header) = RecordHeader::read(input)?; - } - let (input, xy) = XY::read_data(input, header.data_size)?; - - let (input, string) = STRING::read(input)?; - let (input, properties) = read_properties(input)?; - Ok((input, Text{ - layer: (layer, dtype), - xy: xy, - properties: properties, - string: string, - presentation: presentation, - path_type: path_type, - width: width, - invert_y: invert_y, - mag: mag, - angle_deg: angle_deg, - })) - } - - fn write(&self, ww: &mut W) -> OResult { - let mut size = 0; - size += TEXT::write(ww, &())?; - size += LAYER::write(ww, &self.layer.0)?; - size += TEXTTYPE::write(ww, &self.layer.1)?; - if self.presentation.iter().any(|&x| x) { - size += PRESENTATION::write(ww, &self.presentation)?; - } - if self.path_type != 0 { - size += PATHTYPE::write(ww, &self.path_type)?; - } - if self.width != 0 { - size += WIDTH::write(ww, &self.width)? - } - if self.angle_deg != 0.0 || self.mag != 1.0 || self.invert_y { - let strans = { - let mut arr = [false; 16]; - arr[0] = self.invert_y; - arr - }; - size += STRANS::write(ww, &strans)?; - if self.mag != 1.0 { - size += MAG::write(ww, &self.mag)?; - } - if self.angle_deg != 0.0 { - size += ANGLE::write(ww, &self.angle_deg)?; - } - } - size += XY::write(ww, &self.xy)?; - size += STRING::write(ww, &self.string)?; - size += write_properties(ww, &self.properties)?; - size += ENDEL::write(ww, &())?; - Ok(size) - } -} diff --git a/src/lib.rs b/src/lib.rs index cf8365c..c31b907 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,86 +1,307 @@ -extern crate byteorder; - pub mod basic; pub mod record; pub mod records; -pub mod elements; +mod elements; pub mod library; +pub mod review; +use crate::library::{read_library, read_selected_library, scan_library}; -use byteorder::{ByteOrder, BigEndian}; +use std::any::Any; +use std::cell::RefCell; use std::mem::size_of; +use std::ffi::{CStr, CString, c_char}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::ptr; +#[cfg(unix)] +use std::os::unix::ffi::OsStringExt; +use std::str; +use std::fs; +use std::slice; + +use arrow_array::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, StructArray}; -macro_rules! impl_i16be { - ( $tt:ty, $arr:ident, $size:ident ) => { - { - let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; - for xx in sl.iter_mut() { - if *xx < i16::MIN as $tt { return *xx } - if *xx > i16::MAX as $tt { return *xx } +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i16(&mut buf, *xx as i16); - *xx = <$tt>::from_le_bytes(buf); - } - 0 as $tt - } +/// Stable C API version. Increment for incompatible function/ownership changes. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_abi_version() -> u32 { + 1 +} + +/// Layout of the full-read and hierarchy-scan Arrow payloads. +#[unsafe(no_mangle)] +pub extern "C" fn pheon_arrow_schema_version() -> u32 { + 1 +} + + +fn clear_last_error() { + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = None); +} + + +fn set_last_error(message: String) -> i32 { + let sanitized = message.replace('\0', "\\0"); + LAST_ERROR.with(|last_error| *last_error.borrow_mut() = Some(CString::new(sanitized).unwrap())); + -1 +} + + +fn describe_panic(payload: Box) -> String { + if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unexpected panic".to_string() } } -macro_rules! impl_i32be { - ( $tt:ty, $arr:ident, $size:ident ) => { - { - let sl = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; - for xx in sl.iter_mut() { - if *xx < i32::MIN as $tt { return *xx } - if *xx > i32::MAX as $tt { return *xx } +fn describe_parse_error(input: &[u8], (remaining, reason): (&[u8], crate::basic::ErrType)) -> String { + // Selected-cell errors may contain a bounded range rather than a file suffix. + let offset = remaining.as_ptr() as usize - input.as_ptr() as usize; + let message = match reason { + crate::basic::ErrType::Failed(message) => message, + crate::basic::ErrType::Incomplete(Some(size)) => format!("Unexpected end of input; need {size} bytes"), + crate::basic::ErrType::Incomplete(None) => "Unexpected end of input".to_string(), + }; + format!("at byte {offset}: {message}") +} - let mut buf = [0; size_of::<$tt>()]; - BigEndian::write_i32(&mut buf, *xx as i32); - *xx = <$tt>::from_le_bytes(buf); - } - 0 as $tt - } + +fn export_struct_array( + op_name: &str, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + build: impl FnOnce() -> Result, + ) -> i32 { + clear_last_error(); + if arr.is_null() || schema.is_null() { + return set_last_error(format!("{op_name}: Arrow output pointer was null")); + } + match catch_unwind(AssertUnwindSafe(build)) { + Ok(Ok(struct_arr)) => match to_ffi(&struct_arr.to_data()) { + Ok((ffi_arr, ffi_schema)) => { + unsafe { + *arr = ffi_arr; + *schema = ffi_schema; + } + 0 + }, + Err(err) => set_last_error(format!("{op_name}: failed to export Arrow data: {err}")), + }, + Ok(Err(message)) => set_last_error(format!("{op_name}: {message}")), + Err(payload) => set_last_error(format!("{op_name}: {}", describe_panic(payload))), } } -#[no_mangle] -pub extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_i16be!(f64, arr, size) } -#[no_mangle] -pub extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_i32be!(f64, arr, size) } +fn pathbuf_from_c(cpath: *const c_char) -> Result { + if cpath.is_null() { + return Err("path pointer was null".to_string()); + } -#[no_mangle] -pub extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_i16be!(f32, arr, size) } -#[no_mangle] -pub extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_i32be!(f32, arr, size) } - -#[no_mangle] -pub extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_i16be!(u64, arr, size) } -#[no_mangle] -pub extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_i32be!(u64, arr, size) } - -#[no_mangle] -pub extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_i16be!(i64, arr, size) } -#[no_mangle] -pub extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_i32be!(i64, arr, size) } - -#[no_mangle] -pub extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_i16be!(u32, arr, size) } -#[no_mangle] -pub extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_i32be!(u32, arr, size) } - -#[no_mangle] -pub extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_i16be!(i32, arr, size) } -#[no_mangle] -pub extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_i32be!(i32, arr, size) } + let cstr = unsafe { CStr::from_ptr(cpath) }; + #[cfg(unix)] + { + return Ok(PathBuf::from(std::ffi::OsString::from_vec(cstr.to_bytes().to_vec()))); + } + #[cfg(windows)] + { + let ustr = str::from_utf8(cstr.to_bytes()) + .map_err(|_| "Non-UTF8 paths are not supported".to_string())?; + return Ok(PathBuf::from(ustr)); + } + #[allow(unreachable_code)] + Err("Unsupported OS".to_string()) +} -#[no_mangle] -pub extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_i16be!(u16, arr, size) } +/// Return this thread's last parser error, or null if its last call succeeded. +/// The returned string remains valid until this thread's next parser call or exit. +#[unsafe(no_mangle)] +pub extern "C" fn last_error_message() -> *const c_char { + LAST_ERROR.with(|last_error| last_error.borrow() + .as_ref() + .map_or(ptr::null(), |message| message.as_ptr())) +} -#[no_mangle] -pub extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_i16be!(i16, arr, size) } +/// Read a GDSII file and export its Arrow data. Return 0 on success or -1 on error. +/// +/// # Safety +/// `cpath` must point to a null-terminated path. Non-null output pointers must +/// point to initialized, writable Arrow FFI structures. Null pointers are rejected. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_path( + cpath: *const c_char, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_path", arr, schema, || { + let path = pathbuf_from_c(cpath)?; + let input = fs::read(Path::new(&path)) + .map_err(|err| format!("file read failed for {}: {err}", path.display()))?; + let (_input, struct_arr) = read_library(&input) + .map_err(|err| format!("read failed {}", describe_parse_error(&input, err)))?; + Ok(struct_arr) + }) +} + + +unsafe fn input_slice<'a, T>(data: *const T, size: usize, name: &str) -> Result<&'a [T], String> { + if size == 0 { + return Ok(&[]); + } + if data.is_null() { + return Err(format!("{name} pointer was null")); + } + if !data.is_aligned() { + return Err(format!("{name} pointer was misaligned")); + } + if size.checked_mul(size_of::()).is_none_or(|bytes| bytes > isize::MAX as usize) { + return Err(format!("{name} length is too large")); + } + Ok(unsafe { slice::from_raw_parts(data, size) }) +} + +/// Read a borrowed GDSII buffer, exporting Arrow data that owns its storage. +/// +/// # Safety +/// `data` must cover `size` readable bytes (null is allowed for zero bytes). +/// Outputs must be initialized Arrow FFI structures, as for `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, +) -> i32 { + export_struct_array("read_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = read_library(input) + .map_err(|err| format!("read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Scan cell ranges and hierarchy references. Return 0 on success or -1 on error. +/// +/// # Safety +/// Non-null `data` must point to `size` readable bytes. Null is allowed when +/// `size` is zero. Output pointers have the same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scan_bytes( + data: *const u8, + size: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("scan_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let (_input, struct_arr) = scan_library(input) + .map_err(|err| format!("scan failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +/// Read the cells in half-open byte ranges. Return 0 on success or -1 on error. +/// +/// # Safety +/// `data` has the same requirements as `scan_bytes`. Non-null `ranges` must +/// point to `range_count * 2` readable, aligned u64 values, alternating start +/// and end offsets. Null is allowed for zero ranges. Output pointers have the +/// same requirements as `read_path`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read_cells_bytes( + data: *const u8, + size: usize, + ranges: *const u64, + range_count: usize, + arr: *mut FFI_ArrowArray, + schema: *mut FFI_ArrowSchema, + ) -> i32 { + export_struct_array("read_cells_bytes", arr, schema, || { + let input = unsafe { input_slice(data, size, "data") }?; + let range_size = range_count.checked_mul(2) + .ok_or_else(|| "range count is too large".to_string())?; + let flat_ranges = unsafe { input_slice(ranges, range_size, "ranges") }?; + let cell_ranges: Vec<(u64, u64)> = flat_ranges + .as_chunks::<2>() + .0 + .iter() + .map(|&[start, end]| (start, end)) + .collect(); + let (_input, struct_arr) = read_selected_library(input, &cell_ranges) + .map_err(|err| format!("selected-cell read failed {}", describe_parse_error(input, err)))?; + Ok(struct_arr) + }) +} + + +macro_rules! impl_intbe { + ($source:ty, $target:ty, $arr:ident, $size:ident) => {{ + let values = unsafe { std::slice::from_raw_parts_mut($arr, $size) }; + for value in values { + // All source types represent the target bounds exactly in f64. + let number = *value as f64; + if !number.is_finite() + || number < <$target>::MIN as f64 + || number > <$target>::MAX as f64 + { + return *value; + } + let mut slot = [0; size_of::<$source>()]; + slot[..size_of::<$target>()].copy_from_slice(&(*value as $target).to_be_bytes()); + *value = <$source>::from_ne_bytes(slot); + } + 0 as $source + }}; +} + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i16(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f64_to_i32(arr: *mut f64, size: usize) -> f64 { impl_intbe!(f64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i16(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn f32_to_i32(arr: *mut f32, size: usize) -> f32 { impl_intbe!(f32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i16(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u64_to_i32(arr: *mut u64, size: usize) -> u64 { impl_intbe!(u64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i16(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i64_to_i32(arr: *mut i64, size: usize) -> i64 { impl_intbe!(i64, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i16(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u32_to_i32(arr: *mut u32, size: usize) -> u32 { impl_intbe!(u32, i32, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i16(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i16, arr, size) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i32_to_i32(arr: *mut i32, size: usize) -> i32 { impl_intbe!(i32, i32, arr, size) } + + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn u16_to_i16(arr: *mut u16, size: usize) -> u16 { impl_intbe!(u16, i16, arr, size) } + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn i16_to_i16(arr: *mut i16, size: usize) -> i16 { impl_intbe!(i16, i16, arr, size) } diff --git a/src/library.rs b/src/library.rs index e5b5758..2e0ac81 100644 --- a/src/library.rs +++ b/src/library.rs @@ -3,17 +3,24 @@ /// use std::io::Write; + +use crate::record::{Record, RecordFrame}; +use crate::records; +use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord}; +use crate::basic::{ErrType, IResult, OResult, decode_name, fail}; + +use std::string::String; use std::collections::HashMap; +use std::sync::Arc; -pub use crate::record; -pub use crate::record::{RecordHeader, Record}; -pub use crate::records; -pub use crate::elements; -pub use crate::elements::{Element}; -pub use crate::basic::{IResult, OResult, take_bytes, fail}; - - -const DEFAULT_DATE: [i16; 6] = [1900, 0, 0, 0, 0, 0]; +use arrow_schema::{DataType, Field, Fields}; +use arrow_array::{ + StructArray, + builder::{ + BooleanBuilder, Float64Builder, Int16Builder, StringBuilder, StructBuilder, + UInt32Builder, UInt64Builder, + }, +}; /// @@ -44,8 +51,8 @@ impl FileHeader { mod_time: [0, 1, 1, 0, 0, 0], acc_time: [0, 1, 1, 0, 0, 0], name: name.to_owned(), - user_units_per_db_unit: user_units_per_db_unit, - meters_per_db_unit: meters_per_db_unit, + user_units_per_db_unit, + meters_per_db_unit, } } @@ -57,16 +64,16 @@ impl FileHeader { /// Returns: /// FileHeader object /// - pub fn read(input: &[u8]) -> IResult { + pub fn read(input: &[u8]) -> IResult<'_, Self> { let (input, _version) = records::HEADER::read(input)?; let (input, [mod_time, acc_time]) = records::BGNLIB::read(input)?; let (input, name) = records::LIBNAME::skip_and_read(input)?; let (input, (uu, dbu)) = records::UNITS::skip_and_read(input)?; Ok((input, FileHeader{ - mod_time: mod_time, - acc_time: acc_time, - name: name, + mod_time, + acc_time, + name, user_units_per_db_unit: uu, meters_per_db_unit: dbu, })) @@ -91,303 +98,508 @@ impl FileHeader { } -/// -/// Scan through a GDS file, building a table of -/// {b'structure_name': byte_offset}. -/// The intent of this function is to enable random access -/// and/or partial (structure-by-structure) reads. -/// -/// Args: -/// input: Seekable input to read from. Should be positioned -/// before the first structure record, but possibly -/// already past the file header. -/// -pub fn scan_structs(input: &[u8]) -> IResult, usize>> { +fn make_name_list_type() -> DataType { + DataType::List(Arc::new( + Field::new_list_field(DataType::Utf8, false) + )) +} + + +fn make_time_type() -> DataType { + DataType::FixedSizeList(Arc::new( + Field::new_list_field(DataType::Int16, false), + ), + 6, + ) +} + + +fn make_top_level_builder(cells_field: Field, extra_fields: Vec) -> StructBuilder { + let mut fields = vec![ + Field::new("meters_per_db_unit", DataType::Float64, false), + Field::new("user_units_per_db_unit", DataType::Float64, false), + Field::new("lib_name", DataType::Utf8, false), + Field::new("mod_time", make_time_type(), false), + Field::new("acc_time", make_time_type(), false), + cells_field, + Field::new("cell_names", make_name_list_type(), false), + ]; + fields.extend(extra_fields); + StructBuilder::from_fields(fields, 0) +} + + +fn reference_type(is_array: bool, properties: Option<&DataType>) -> DataType { + let mut fields = vec![ + Field::new("target", DataType::UInt32, false), + Field::new("invert_y", DataType::Boolean, false), + Field::new("scale", DataType::Float64, false), + Field::new("angle_rad", DataType::Float64, false), + Field::new("xy", DataType::UInt64, false), + ]; + if is_array { + fields.extend([ + Field::new("xy0", DataType::UInt64, false), + Field::new("xy1", DataType::UInt64, false), + Field::new("counts", DataType::UInt32, false), + ]); + } + if let Some(properties) = properties { + fields.push(Field::new("properties", properties.clone(), true)); + } + DataType::Struct(Fields::from(fields)) +} + + +fn make_library_builder() -> StructBuilder { + let property_t = DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Int16, false), + Field::new("value", DataType::Utf8, false), + ])); + + let property_list_t = DataType::List(Arc::new( + Field::new_list_field(property_t, false) + )); + + let sref_struct_t = reference_type(false, None); + let aref_struct_t = reference_type(true, None); + let sref_prop_struct_t = reference_type(false, Some(&property_list_t)); + let aref_prop_struct_t = reference_type(true, Some(&property_list_t)); + + let text_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("presentation_horiz", DataType::UInt8, true), + Field::new("presentation_vert", DataType::UInt8, true), + Field::new("presentation_font", DataType::UInt8, true), + Field::new("path_type", DataType::Int16, true), + Field::new("width", DataType::Int32, true), + Field::new("invert_y", DataType::Boolean, true), + Field::new("mag", DataType::Float64, true), + Field::new("angle_deg", DataType::Float64, true), + Field::new("xy", DataType::UInt64, false), + Field::new("string", DataType::Utf8, false), + Field::new("properties", property_list_t.clone(), true), + ])); + + + let coords_t = DataType::List(Arc::new( + Field::new_list_field(DataType::Int32, false) + )); + + let boundary_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new( + "vertex_offsets", + DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false))), + false, + ), + ])); + + let rect_batch_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("rects", coords_t.clone(), false), + ])); + + let boundary_prop_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("vertices", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let path_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("path_type", DataType::Int16, true), + Field::new("extension_start", DataType::Int32, true), + Field::new("extension_end", DataType::Int32, true), + Field::new("width", DataType::Int32, true), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let boxnode_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("layer", DataType::UInt32, false), + Field::new("xy", coords_t.clone(), false), + Field::new("properties", property_list_t.clone(), true), + ])); + + let sref_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_struct_t, false) + )); + + let aref_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_struct_t, false) + )); + + let sref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(sref_prop_struct_t, false) + )); + + let aref_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(aref_prop_struct_t, false) + )); + + let text_list_t = DataType::List(Arc::new( + Field::new_list_field(text_struct_t, false) + )); + + let boundary_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_batch_struct_t, false) + )); + + let rect_batch_list_t = DataType::List(Arc::new( + Field::new_list_field(rect_batch_struct_t, false) + )); + + let boundary_prop_list_t = DataType::List(Arc::new( + Field::new_list_field(boundary_prop_struct_t, false) + )); + + let path_list_t = DataType::List(Arc::new( + Field::new_list_field(path_struct_t, false) + )); + + let boxnode_list_t = DataType::List(Arc::new( + Field::new_list_field(boxnode_struct_t, false) + )); + + let layer_list_t = DataType::List(Arc::new( + Field::new_list_field(DataType::UInt32, false) + )); + + let cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("file_offset", DataType::UInt64, false), + Field::new("srefs", sref_list_t, false), + Field::new("arefs", aref_list_t, false), + Field::new("sref_props", sref_prop_list_t, false), + Field::new("aref_props", aref_prop_list_t, false), + Field::new("rect_batches", rect_batch_list_t, false), + Field::new("boundary_batches", boundary_batch_list_t, false), + Field::new("boundary_props", boundary_prop_list_t, false), + Field::new("paths", path_list_t, false), + Field::new("nodes", boxnode_list_t.clone(), true), + Field::new("boxes", boxnode_list_t.clone(), true), + Field::new("texts", text_list_t, false), + ])); + + let cells_list_t = DataType::List(Arc::new( + Field::new_list_field(cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", cells_list_t, false), + vec![ + Field::new("layers", layer_list_t, false), + ], + ) +} + + +fn make_scan_builder() -> StructBuilder { + let scan_ref_struct_t = reference_type(true, None); + + let scan_ref_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_ref_struct_t, false) + )); + + let scan_cell_struct_t = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("struct_start_offset", DataType::UInt64, false), + Field::new("struct_end_offset", DataType::UInt64, false), + Field::new("refs", scan_ref_list_t, false), + ])); + + let scan_cells_list_t = DataType::List(Arc::new( + Field::new_list_field(scan_cell_struct_t, false) + )); + + make_top_level_builder( + Field::new("cells", scan_cells_list_t, false), + Vec::new(), + ) +} + + +fn append_file_header( + lib_builder: &mut StructBuilder, + header: &FileHeader, + ) -> Result<(), ErrType> { + let name = decode_name(&header.name).map_err(ErrType::Failed)?; + let dbu_builder = lib_builder.field_builder::(0).unwrap(); + dbu_builder.append_value(header.meters_per_db_unit); + let uu_builder = lib_builder.field_builder::(1).unwrap(); + uu_builder.append_value(header.user_units_per_db_unit); + let libname_builder = lib_builder.field_builder::(2).unwrap(); + libname_builder.append_value(name); + let mtl_builder = lib_builder.field_builder::(3).unwrap(); + let mt_builder = mtl_builder.values().as_any_mut().downcast_mut::().unwrap(); + mt_builder.append_values(&header.mod_time, &[true; 6]); + mtl_builder.append(true); + let atl_builder = lib_builder.field_builder::(4).unwrap(); + let at_builder = atl_builder.values().as_any_mut().downcast_mut::().unwrap(); + at_builder.append_values(&header.acc_time, &[true; 6]); + atl_builder.append(true); + Ok(()) +} + + +fn append_names( + lib_builder: &mut StructBuilder, + field_index: usize, + names: HashMap, + ) { + // Interned IDs are consecutive, so a vector directly represents ID order. + let mut ordered_names = vec![String::new(); names.len()]; + for (name, id) in names { + ordered_names[id as usize] = name; + } + let names_builder = lib_builder.field_builder::(field_index).unwrap(); + let name_builder = names_builder.values().as_any_mut().downcast_mut::().unwrap(); + for name in ordered_names { + name_builder.append_value(name); + } + names_builder.append(true); +} + + +fn append_layers( + lib_builder: &mut StructBuilder, + layers: HashMap, + ) { + let mut ordered_layers = vec![0; layers.len()]; + for (layer, id) in layers { + ordered_layers[id as usize] = layer; + } + let layers_builder = lib_builder.field_builder::(7).unwrap(); + let layer_builder = layers_builder.values().as_any_mut().downcast_mut::().unwrap(); + layer_builder.append_slice(&ordered_layers); + layers_builder.append(true); +} + + +fn append_scan_ref( + refs_builder: &mut DListBuilder, + scan_ref: ScanRefRecord, + ) { + let ref_builder = refs_builder.values().as_any_mut().downcast_mut::().unwrap(); + let target_builder = ref_builder.field_builder::(0).unwrap(); + target_builder.append_value(scan_ref.target_id); + let invert_builder = ref_builder.field_builder::(1).unwrap(); + invert_builder.append_value(scan_ref.invert_y); + let scale_builder = ref_builder.field_builder::(2).unwrap(); + scale_builder.append_value(scan_ref.scale); + let angle_builder = ref_builder.field_builder::(3).unwrap(); + angle_builder.append_value(scan_ref.angle_rad); + let xy_builder = ref_builder.field_builder::(4).unwrap(); + xy_builder.append_value(scan_ref.xy); + let xy0_builder = ref_builder.field_builder::(5).unwrap(); + xy0_builder.append_value(scan_ref.xy0); + let xy1_builder = ref_builder.field_builder::(6).unwrap(); + xy1_builder.append_value(scan_ref.xy1); + let counts_builder = ref_builder.field_builder::(7).unwrap(); + counts_builder.append_value(scan_ref.counts); + ref_builder.append(true); +} + + +fn intern_name(names: &mut HashMap, name: String) -> u32 { + let next_id = names.len(); + *names.entry(name).or_insert(next_id.try_into().unwrap()) +} + + +fn read_cell_header<'a>(begin: &RecordFrame<'a>, input: &'a [u8]) -> IResult<'a, String> { + if begin.tag != records::RTAG_BGNSTR { + return fail( + begin.data, + format!( + "Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", + begin.tag, + records::RTAG_BGNSTR + ), + ); + } + begin.decode::()?; + let (input, name_bytes) = records::STRNAME::read(input)?; + let name = decode_name(&name_bytes).map_err(|error| (input, ErrType::Failed(error)))?; + Ok((input, name.to_owned())) +} + + +fn append_full_cell<'a>( + cells_builder: &mut DListBuilder, + names: &mut HashMap, + layers: &mut HashMap, + struct_input: &'a [u8], + id: u32, + file_offset: u64, + ) -> IResult<'a, ()> { + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let offset_builder = cell_builder.field_builder::(1).unwrap(); + offset_builder.append_value(file_offset); + + let (next_input, _) = read_elements(struct_input, cell_builder, names, layers)?; + cell_builder.append(true); + Ok((next_input, ())) +} + + +fn walk_structs<'a, F>( + input: &'a [u8], + base_offset: usize, + names: &mut HashMap, + mut on_struct: F, + ) -> IResult<'a, ()> +where + F: FnMut(&'a [u8], usize, u32, &mut HashMap) -> IResult<'a, ()>, +{ let input_size = input.len(); - let mut positions = HashMap::new(); - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDLIB { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_BGNSTR { - let name; - (input, name) = records::STRNAME::read(input)?; - if positions.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - let position = input_size - input.len(); - positions.insert(name, position); - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, positions)) -} - - -#[derive(Debug, Clone)] -pub struct Cell { - name: Vec, - - boundaries: Vec, - paths: Vec, - nodes: Vec, - boxes: Vec, - texts: Vec, - refs: Vec, -} - -impl Cell { - /// Build an empty cell - pub fn new(name: Vec) -> Self { - Cell{ - name: name, - boundaries: Vec::new(), - paths: Vec::new(), - nodes: Vec::new(), - boxes: Vec::new(), - texts: Vec::new(), - refs: Vec::new(), - } - } - - /// Skip to the next structure and attempt to read it. - /// - /// Args: - /// input: Seekable input to read from. - /// - /// Returns: - /// (name, elements) if a structure was found. - /// None if no structure was found before the end of the library. - /// - pub fn read(input: &[u8]) -> IResult> { - let (input, success) = records::BGNSTR::skip_past(input)?; - if !success { - return Ok((input, None)) - } - - let (input, name) = records::STRNAME::read(input)?; - let mut cell = Cell::new(name); - let (input, _) = cell.read_elements(input)?; - Ok((input, Some(cell))) - } - - /// Read elements from the input until an ENDSTR - /// record is encountered. The ENDSTR record is also - /// consumed. - /// - /// Args: - /// input: Seekable input to read from. - /// - /// Returns: - /// List of element objects. - /// - pub fn read_elements<'a>(&mut self, input: &'a [u8]) -> IResult<'a, ()> { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - match header.tag { - records::RTAG_BOUNDARY => { - let boundary; - (input, _) = records::BOUNDARY::read(input)?; - (input, boundary) = elements::Boundary::read(input)?; - self.boundaries.push(boundary); - }, - records::RTAG_PATH => { - let path; - (input, _) = records::PATH::read(input)?; - (input, path) = elements::Path::read(input)?; - self.paths.push(path); - }, - records::RTAG_NODE => { - let node; - (input, _) = records::NODE::read(input)?; - (input, node) = elements::Node::read(input)?; - self.nodes.push(node); - }, - records::RTAG_BOX => { - let gds_box; - (input, _) = records::BOX::read(input)?; - (input, gds_box) = elements::GDSBox::read(input)?; - self.boxes.push(gds_box); - }, - records::RTAG_TEXT => { - let txt; - (input, _) = records::TEXT::read(input)?; - (input, txt) = elements::Text::read(input)?; - self.texts.push(txt); - }, - records::RTAG_SREF => { - let sref; - (input, _) = records::SREF::read(input)?; - (input, sref) = elements::Reference::read(input)?; - self.refs.push(sref); - }, - records::RTAG_AREF => { - let aref; - (input, _) = records::AREF::read(input)?; - (input, aref) = elements::Reference::read(input)?; - self.refs.push(aref); - }, - _ => { - // don't care, skip - (input, _) = take_bytes(input, header.data_size)?; - } - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, ())) - } - - /// - /// Write a structure to the provided input. - /// - /// Args: - /// name: Structure name (ascii-encoded). - /// elements: List of Elements containing the geometry and text in this struct. - /// cre_time: Creation time (optional). - /// mod_time: Modification time (optional). - /// - /// Return: - /// Number of bytes written - /// - pub fn write( - &self, - ww: &mut W, - cre_time: Option<[i16; 6]>, - mod_time: Option<[i16; 6]>, - ) -> OResult { - let mut size = 0; - size += records::BGNSTR::write(ww, &[cre_time.unwrap_or(DEFAULT_DATE), - mod_time.unwrap_or(DEFAULT_DATE)])?; - size += records::STRNAME::write(ww, &self.name)?; - size += self.write_elements(ww)?; - size += records::ENDSTR::write(ww, &())?; - Ok(size) - } - - pub fn write_elements(&self, ww: &mut W) -> OResult { - let mut size = 0; - for boundary in &self.boundaries { - size += boundary.write(ww)?; - } - for path in &self.paths { - size += path.write(ww)?; - } - for node in &self.nodes { - size += node.write(ww)?; - } - for gds_box in &self.boxes { - size += gds_box.write(ww)?; - } - for text in &self.texts { - size += text.write(ww)?; - } - for reference in &self.refs { - size += reference.write(ww)?; - } - Ok(size) - } -} - - -/* -/// -/// Scan through a GDS file, building a table of instance counts -/// `{b'structure_name': {b'ref_name': count}}`. -/// -/// This is intended to provide a fast overview of the file's -/// contents without performing a full read of all elements. -/// -/// Args: -/// input: Seekable input to read from. Should be positioned -/// before the first structure record, but possibly -/// already past the file header. -/// -pub fn scan_hierarchy(input: &[u8]) -> IResult, HashMap::, u32>>> { - let mut structures = HashMap::new(); - - let mut ref_name = None; - let mut ref_count = None; - let (mut input, mut header) = RecordHeader::read(input)?; - let mut cur_structure = HashMap::new(); - while header.tag != records::RTAG_ENDLIB { - match header.tag { + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + match record.tag { records::RTAG_BGNSTR => { - (input, _) = take_bytes(input, header.data_size)?; - let result = records::STRNAME::read(input)?; - input = result.0; - let name = result.1; - if structures.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - let mut cur_structure = HashMap::new(); - structures.insert(name, cur_structure); - ref_name = None; - ref_count = None; - }, - records::RTAG_SNAME => { - let result = records::SNAME::read_data(input, header.data_size)?; - input = result.0; - ref_name = Some(result.1); - }, - records::RTAG_COLROW => { - let result = records::COLROW::read_data(input, header.data_size)?; - input = result.0; - let (col, row) = (result.1[0], result.1[1]); - ref_count = Some((col * row) as u32); - }, - records::RTAG_ENDEL => { - *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); - }, - _ => { - (input, _) = take_bytes(input, header.data_size)?; - }, - } - (input, header) = RecordHeader::read(input)?; - } - Ok((input, structures)) -} -*/ - -/* -pub fn count_ref(input: &[u8]) -> IResult, u32))> { - let (input, found_struc) = records::BGNSTR.skip_past(input)?; - if !found_struc { - return Ok((input, None)) - } - let mut cur_structure = HashMap::new(); - let (input, name) = records::STRNAME::read(input)?; - if structures.contains_key(&name) { - return fail(input, format!("Duplicate structure name: {:?}", name)); - } - - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != records::RTAG_ENDSTR { - let mut ref_name = None; - let mut ref_count = None; - while header.tag != records::RTAG_ENDEL { - match header.tag { - records::RTAG_SNAME => { - let result = records::SNAME::read_data(input1, header.data_size)?; - input1 = result.0; - ref_name = Some(result.1); - }, - records::RTAG_COLROW => { - let result = records::COLROW::read_data(input1, header.data_size)?; - input1 = result.0; - let (col, row) = (result.1[0], result.1[1]); - ref_count = Some((col * row) as u32); - }, - _ => { - (input1, _) = take_bytes(input1, header.data_size)?; - }, + let record_start = base_offset + input_size - input.len(); + let (after_name, name) = read_cell_header(&record, remaining)?; + let id = intern_name(names, name); + (input, ()) = on_struct(after_name, record_start, id, names)?; } - (input1, header) = RecordHeader::read(input1)?; + records::RTAG_ENDLIB => { + record.decode::()?; + return Ok((remaining, ())); + } + _ => input = remaining, } - // got ENDEL, update count for this reference - *cur_structure.entry(ref_name.unwrap()).or_insert(0) += ref_count.unwrap_or(1); - (input1, header) = RecordHeader::read(input1)?; } - structures.insert(name, cur_structure); - (input, header) = RecordHeader::read(input1)?; } -*/ + + +pub fn read_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_library_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, _record_start, id, names| { + let position = base_offset + walk_input_size - struct_input.len(); + append_full_cell( + cells_builder, + names, + &mut layers, + struct_input, + id, + position.try_into().unwrap(), + ) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Scan cell byte ranges and references without decoding geometry. +pub fn scan_library(input: &[u8]) -> IResult<'_, StructArray> { + let full_input_size = input.len(); + let mut lib_builder = make_scan_builder(); + + let (input, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (input, error))?; + let base_offset = full_input_size - input.len(); + + let mut names = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + let walk_input_size = input.len(); + let (input, ()) = walk_structs(input, base_offset, &mut names, |struct_input, record_start, id, names| { + let (after_cell, refs) = scan_elements(struct_input, names)?; + let struct_end = base_offset + walk_input_size - after_cell.len(); + let cell_builder = cells_builder.values().as_any_mut().downcast_mut::().unwrap(); + let id_builder = cell_builder.field_builder::(0).unwrap(); + id_builder.append_value(id); + + let start_builder = cell_builder.field_builder::(1).unwrap(); + start_builder.append_value(record_start.try_into().unwrap()); + + let end_builder = cell_builder.field_builder::(2).unwrap(); + end_builder.append_value(struct_end.try_into().unwrap()); + + let refs_builder = cell_builder.field_builder::(3).unwrap(); + for scan_ref in refs { + append_scan_ref(refs_builder, scan_ref); + } + refs_builder.append(true); + cell_builder.append(true); + Ok((after_cell, ())) + })?; + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} + + +/// Read exact half-open cell ranges returned by `scan_library`, in caller order. +/// Each range includes BGNSTR through ENDSTR. Cell and layer IDs are local to +/// this result. This random-access operation returns the original input slice +/// as its remainder; it does not advance a sequential parsing cursor. +pub fn read_selected_library<'a>(input: &'a [u8], ranges: &[(u64, u64)]) -> IResult<'a, StructArray> { + let mut lib_builder = make_library_builder(); + let (header_end, header) = FileHeader::read(input)?; + append_file_header(&mut lib_builder, &header).map_err(|error| (header_end, error))?; + + let mut names = HashMap::::new(); + let mut layers = HashMap::::new(); + let cells_builder = lib_builder.field_builder::(5).unwrap(); + + for (start_u64, end_u64) in ranges { + if start_u64 >= end_u64 || *end_u64 > input.len() as u64 { + return fail(input, format!("Cell byte range out of bounds: [{start_u64}, {end_u64})")); + } + let start = usize::try_from(*start_u64).unwrap(); + let end = usize::try_from(*end_u64).unwrap(); + + let (struct_input, begin) = RecordFrame::read(&input[start..end])?; + let (struct_input, name) = read_cell_header(&begin, struct_input)?; + let file_offset = end - struct_input.len(); + + let id = intern_name(&mut names, name); + let (remaining_struct, _) = append_full_cell( + cells_builder, + &mut names, + &mut layers, + struct_input, + id, + file_offset.try_into().unwrap(), + )?; + if !remaining_struct.is_empty() { + return fail(remaining_struct, "Cell byte range extends past ENDSTR".to_string()); + } + } + cells_builder.append(true); + + append_names(&mut lib_builder, 6, names); + append_layers(&mut lib_builder, layers); + + lib_builder.append(true); + let lib = lib_builder.finish(); + Ok((input, lib)) +} diff --git a/src/record.rs b/src/record.rs index 41921c4..5ad0762 100644 --- a/src/record.rs +++ b/src/record.rs @@ -1,57 +1,91 @@ /// /// Generic record-level read/write functionality. /// -use std::io::Write; -use std::convert::TryInto; -use byteorder::{ByteOrder, BigEndian}; +use std::io::{self, Write}; -use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)] -use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)] -use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; +use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; +use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; +use crate::basic::{ErrType, IResult, OResult, fail, incomplete, take_bytes}; use crate::records; -//#[no_mangle] -//pub extern "C" fn write_record_header( - - #[repr(C)] +#[derive(Debug, Clone)] pub struct RecordHeader { pub tag: u16, pub data_size: u16, } impl RecordHeader { - pub fn read(input: &[u8]) -> IResult { - let (input, size) = parse_u16(input)?; - let (input, tag) = parse_u16(input)?; - Ok((input, RecordHeader{tag:tag, data_size:size - 4})) + pub fn read(input: &[u8]) -> IResult<'_, RecordHeader> { + let Some((&[size_hi, size_lo, tag_hi, tag_lo], payload)) = input.split_first_chunk::<4>() else { + return incomplete(input, Some(4)); + }; + let size = u16::from_be_bytes([size_hi, size_lo]); + let Some(data_size) = size.checked_sub(4) else { + return fail(input, format!("Record size {size} is smaller than its header")); + }; + let tag = u16::from_be_bytes([tag_hi, tag_lo]); + Ok((payload, RecordHeader { tag, data_size })) } - pub fn pack_into(&self) -> [u8; 4] { - assert!(self.data_size < 0xffff - 4, "Record too big!"); - let vals = [self.data_size, self.tag]; - let mut buf = [0x77; 4]; - BigEndian::write_u16_into(&vals, &mut buf); - buf + pub fn pack_into(&self) -> io::Result<[u8; 4]> { + let size = self.data_size.checked_add(4) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + let [size_hi, size_lo] = size.to_be_bytes(); + let [tag_hi, tag_lo] = self.tag.to_be_bytes(); + Ok([size_hi, size_lo, tag_hi, tag_lo]) } pub fn write(&self, ww: &mut W) -> OResult { - let bytes = self.pack_into(); - ww.write(&bytes) + let bytes = self.pack_into()?; + ww.write_all(&bytes)?; + Ok(bytes.len()) } } +/// A complete record borrowing its payload from the input. +pub(crate) struct RecordFrame<'a> { + pub tag: u16, + pub data: &'a [u8], +} + +impl<'a> RecordFrame<'a> { + pub fn read(input: &'a [u8]) -> IResult<'a, Self> { + let (_, header) = RecordHeader::read(input)?; + let (remaining, data) = read_payload(input, header.data_size)?; + Ok((remaining, Self { tag: header.tag, data })) + } + + /// Decode the payload after the caller has matched the record tag. + pub fn decode + ?Sized, D: RecordData>( + &self, + ) -> Result, (&'a [u8], ErrType)> { + let (_, value) = R::read_data(self.data, self.data.len() as u16)?; + Ok(value) + } +} + +fn read_payload(input: &[u8], size: u16) -> IResult<'_, &[u8]> { + if !size.is_multiple_of(2) { + return fail(input, "odd record length".to_string()); + } + let Some((record, remaining)) = input.split_at_checked(4 + usize::from(size)) else { + return fail(input, "truncated record".to_string()); + }; + Ok((remaining, &record[4..])) +} + + pub trait RecordData { - type BareData; + type BareData<'a>; type InData : ?Sized; type ByteData : AsRef<[u8]>; - fn read(input: &[u8], size: u16) -> IResult; + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>; fn pack_into(buf: &mut [u8], data: &Self::InData); - //fn size(data: &Self::BareData) -> u16; fn pack(data: &Self::InData) -> Self::ByteData; } @@ -71,22 +105,19 @@ pub trait Record { } } - fn read_header(input: &[u8]) -> IResult { - RecordHeader::read(input) - } - fn write_header(ww: &mut W, data_size: u16) -> OResult { - RecordHeader{tag: Self::tag(), data_size: data_size}.write(ww) + Self::check_size(data_size) + .map_err(|message| io::Error::new(io::ErrorKind::InvalidInput, message))?; + RecordHeader{tag: Self::tag(), data_size}.write(ww) } - fn read_data(input: &[u8], size: u16) -> IResult { + fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> { + if let Err(message) = Self::check_size(size) { + return fail(input, message); + } RData::read(input, size) } - fn pack_data(buf: &mut [u8], data: &RData::InData) { - RData::pack_into(buf, data) - } - /// /// Skip to the end of the next occurence of this record. /// @@ -94,31 +125,33 @@ pub trait Record { /// True if the record was encountered and skipped. /// False if the end of the library was reached. /// - fn skip_past(input: &[u8]) -> IResult { + fn skip_past(input: &[u8]) -> IResult<'_, bool> { let original_input = input; - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - if header.tag == records::RTAG_ENDLIB { - return Ok((original_input, false)) + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, true)); } - (input, header) = RecordHeader::read(input)?; + if record.tag == records::RTAG_ENDLIB { + return Ok((original_input, false)); + } + input = remaining; } - (input, _) = take_bytes(input, header.data_size)?; - Ok((input, true)) } - fn skip_and_read(input: &[u8]) -> IResult { - let (mut input, mut header) = RecordHeader::read(input)?; - while header.tag != Self::tag() { - (input, _) = take_bytes(input, header.data_size)?; - (input, header) = RecordHeader::read(input)?; + fn skip_and_read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let mut input = input; + loop { + let (remaining, record) = RecordFrame::read(input)?; + if record.tag == Self::tag() { + return Ok((remaining, record.decode::()?)); + } + input = remaining; } - let (input, data) = Self::read_data(input, header.data_size)?; - Ok((input, data)) } - fn expect_header(input: &[u8]) -> IResult { + fn expect_header(input: &[u8]) -> IResult<'_, u16> { let (input, header) = RecordHeader::read(input)?; if header.tag != Self::tag() { fail(input, format!("Unexpected record! Got tag 0x{:04x}, expected 0x{:04x}", header.tag, Self::tag())) @@ -127,32 +160,34 @@ pub trait Record { } } - fn read(input: &[u8]) -> IResult { - let (input, size) = Self::expect_header(input)?; - Self::check_size(size).unwrap(); - let (input, data) = Self::read_data(input, size)?; - Ok((input, data)) + fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> { + let (_, size) = Self::expect_header(input)?; + let (remaining, data) = read_payload(input, size)?; + let (_, value) = Self::read_data(data, size)?; + Ok((remaining, value)) } fn write(ww: &mut W, data: &RData::InData) -> OResult { let packed_data = RData::pack(data); let data_bytes = packed_data.as_ref(); - let len: u16 = data_bytes.len().try_into().expect("Record longer than max size (u16)!"); - let mut size = 0; - size += Self::write_header(ww, len)?; - size += ww.write(data_bytes)?; - Ok(size) + let len = u16::try_from(data_bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Record longer than max size (u16)"))?; + Self::write_header(ww, len)?; + ww.write_all(data_bytes)?; + Ok(usize::from(len) + 4) } } pub struct BitArray; impl RecordData for BitArray { - type BareData = [bool; 16]; + type BareData<'a> = [bool; 16]; type InData = [bool; 16]; type ByteData = [u8; 2]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 2); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_bitarray(input) } @@ -170,12 +205,14 @@ impl RecordData for BitArray { pub struct Int2; impl RecordData for Int2 { - type BareData = i16; + type BareData<'a> = i16; type InData = i16; type ByteData = [u8; 2]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 2); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 2 { + return fail(input, format!("Expected record size {}, got {size}", 2)); + } parse_int2(input) } @@ -192,12 +229,14 @@ impl RecordData for Int2 { pub struct Int4; impl RecordData for Int4 { - type BareData = i32; + type BareData<'a> = i32; type InData = i32; type ByteData = [u8; 4]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 4); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 4 { + return fail(input, format!("Expected record size {}, got {size}", 4)); + } parse_int4(input) } @@ -215,71 +254,109 @@ impl RecordData for Int4 { pub struct Int2Array; impl RecordData for Int2Array { - type BareData = Vec; + type BareData<'a> = Int2ArrayReader<'a>; type InData = [i16]; type ByteData = Vec; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size % 2 == 0, "Record must contain an integer quantity of integers"); - let mut buf = Vec::with_capacity(size as usize / 2); - let mut input = input; - for ii in 0..buf.len() { - (input, buf[ii]) = parse_int2(input)?; + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if !size.is_multiple_of(2) { + return fail(input, format!("Record size must be divisible by 2, got {size}")); } - Ok((input, buf)) + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int2ArrayReader{bytes})) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - BigEndian::write_i16_into(&data, buf) + assert_eq!(buf.len(), data.len() * 2); + for (bytes, value) in buf.as_chunks_mut::<2>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } } fn pack(data: &Self::InData) -> Self::ByteData { - let mut buf = Vec::with_capacity(data.len() * 2); + let mut buf = vec![0; data.len() * 2]; Self::pack_into(&mut buf, data); buf } } +pub struct Int2ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int2ArrayReader<'_> { + type Item = i16; + + fn next(&mut self) -> Option { + if self.bytes.len() < 2 { + None + } else { + let (remaining, val) = parse_int2(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + pub struct Int4Array; impl RecordData for Int4Array { - type BareData = Vec; + type BareData<'a> = Int4ArrayReader<'a>; type InData = [i32]; type ByteData = Vec; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size % 4 == 0, "Record must contain an integer quantity of integers"); - let mut buf = Vec::with_capacity(size as usize / 4); - let mut input = input; - for ii in 0..buf.len() { - (input, buf[ii]) = parse_int4(input)?; + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if !size.is_multiple_of(4) { + return fail(input, format!("Record size must be divisible by 4, got {size}")); } - Ok((input, buf)) + let (input, bytes) = take_bytes(input, size)?; + Ok((input, Int4ArrayReader{bytes})) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - BigEndian::write_i32_into(&data, buf) + assert_eq!(buf.len(), data.len() * 4); + for (bytes, value) in buf.as_chunks_mut::<4>().0.iter_mut().zip(data) { + *bytes = value.to_be_bytes(); + } } fn pack(data: &Self::InData) -> Self::ByteData { - let mut buf = Vec::with_capacity(data.len() * 4); + let mut buf = vec![0; data.len() * 4]; Self::pack_into(&mut buf, data); buf } } +pub struct Int4ArrayReader<'a> { + bytes: &'a [u8], +} +impl Iterator for Int4ArrayReader<'_> { + type Item = i32; + + fn next(&mut self) -> Option { + if self.bytes.len() < 4 { + None + } else { + let (remaining, val) = parse_int4(self.bytes).unwrap(); + self.bytes = remaining; + Some(val) + } + } +} + pub struct Real8; impl RecordData for Real8 { - type BareData = f64; + type BareData<'a> = f64; type InData = f64; type ByteData = [u8; 8]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 8); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 8 { + return fail(input, format!("Expected record size {}, got {size}", 8)); + } parse_real8(input) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(buf, *data).expect(&format!("Float {0} too big for Real8", data)) + pack_real8(buf, *data).unwrap_or_else(|_| panic!("Float {0} too big for Real8", data)) } fn pack(data: &Self::InData) -> Self::ByteData { @@ -291,20 +368,22 @@ impl RecordData for Real8 { pub struct Real8Pair; impl RecordData for Real8Pair { - type BareData = (f64, f64); + type BareData<'a> = (f64, f64); type InData = (f64, f64); type ByteData = [u8; 2 * 8]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 2 * 8); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 2 * 8 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 8)); + } let (input, data0) = parse_real8(input)?; let (input, data1) = parse_real8(input)?; Ok((input, (data0, data1))) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_real8(&mut buf[8 * 0..], data.0).expect(&format!("Float.0 {0} too big for Real8", data.0)); - pack_real8(&mut buf[8 * 1..], data.1).expect(&format!("Float.1 {0} too big for Real8", data.1)); + pack_real8(buf, data.0).unwrap_or_else(|_| panic!("Float.0 {0} too big for Real8", data.0)); + pack_real8(&mut buf[8 * 1..], data.1).unwrap_or_else(|_| panic!("Float.1 {0} too big for Real8", data.1)); } fn pack(data: &Self::InData) -> Self::ByteData { @@ -313,19 +392,15 @@ impl RecordData for Real8Pair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct ASCII; impl RecordData for ASCII { - type BareData = Vec; + type BareData<'a> = Vec; type InData = [u8]; type ByteData = Vec; - fn read(input: &[u8], size: u16) -> IResult { + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { parse_ascii(input, size) } @@ -334,7 +409,7 @@ impl RecordData for ASCII { } fn pack(data: &Self::InData) -> Self::ByteData { - let mut buf = Vec::with_capacity(data.len() * 4); + let mut buf = vec![0; data.len() + data.len() % 2]; Self::pack_into(&mut buf, data); buf } @@ -343,19 +418,21 @@ impl RecordData for ASCII { pub struct DateTimePair; impl RecordData for DateTimePair { - type BareData = [[i16; 6]; 2]; + type BareData<'a> = [[i16; 6]; 2]; type InData = [[i16; 6]; 2]; type ByteData = [u8; 2 * 6 * 2]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 2 * 6 * 2); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 2 * 6 * 2 { + return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2)); + } let (input, data0) = parse_datetime(input)?; let (input, data1) = parse_datetime(input)?; Ok((input, [data0, data1])) } fn pack_into(buf: &mut [u8], data: &Self::InData) { - pack_datetime(&mut buf[6 * 2 * 0..], &data[0]); + pack_datetime(buf, &data[0]); pack_datetime(&mut buf[6 * 2 * 1..], &data[1]); } @@ -365,20 +442,18 @@ impl RecordData for DateTimePair { buf } - //fn write(ww: &mut W, data: &Self::BareData) -> OResult { - // let mut buf = [u8; 2 * 6 * 2]; - // Self::pack_into(&mut buf, data) - //} } pub struct Empty; impl RecordData for Empty { - type BareData = (); + type BareData<'a> = (); type InData = (); type ByteData = [u8; 0]; - fn read(input: &[u8], size: u16) -> IResult { - assert!(size == 0); + fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> { + if size != 0 { + return fail(input, format!("Expected record size {}, got {size}", 0)); + } Ok((input, ())) } @@ -388,6 +463,4 @@ impl RecordData for Empty { fn pack(_data: &Self::InData) -> Self::ByteData { [] } - //fn write(ww: &mut W, data: &Self::BareData) { - //} } diff --git a/src/records.rs b/src/records.rs index 4fe21fa..3f6ba58 100644 --- a/src/records.rs +++ b/src/records.rs @@ -2,9 +2,11 @@ /// Record type and tag definitions /// -use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty}; - -//use std::io::Write; +use crate::basic::{ErrType, IResult, fail}; +use crate::record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, +}; // record tags pub const RTAG_HEADER: u16 = 0x0002; @@ -77,42 +79,66 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300; pub const RTAG_SPACERERROR: u16 = 0x4400; pub const RTAG_CONTACT: u16 = 0x4500; -/* -// data types -pub const DATA_TYPE_NONE: u16 = 0x00; -pub const DATA_TYPE_BIT: u16 = 0x01; -pub const DATA_TYPE_INT16: u16 = 0x02; -pub const DATA_TYPE_INT32: u16 = 0x03; -pub const DATA_TYPE_REAL32: u16 = 0x04; -pub const DATA_TYPE_REAL64: u16 = 0x05; -pub const DATA_TYPE_STR: u16 = 0x06; -pub const MAX_DATA_SIZE: usize = 8; -/// Returns the size of the given data type in bytes. -pub fn data_size(t: u16) -> Option { - match t { - x if x == DATA_TYPE_NONE => 0, - x if x == DATA_TYPE_BIT => 2, - x if x == DATA_TYPE_INT16 => 2, - x if x == DATA_TYPE_INT32 => 4, - x if x == DATA_TYPE_REAL32 => 4, - x if x == DATA_TYPE_REAL64 => 8, - _ => 0 - } -*/ +// Define fixed-size records once for typed decoding and raw-record validation. +macro_rules! fixed_size_records { + ($($(#[$attr:meta])* $name:ident => $tag:ident, $data:ty, $size:expr;)+) => { + $( + $(#[$attr])* + pub struct $name; + impl Record<$data> for $name { + fn tag() -> u16 { $tag } + fn expected_size() -> Option { Some($size) } + } + )+ -pub struct HEADER; -impl Record for HEADER { - fn tag() -> u16 { RTAG_HEADER } - fn expected_size() -> Option { Some(2) } + pub(crate) fn check_payload_size(tag: u16, size: u16) -> Result<(), String> { + match tag { + $($tag => $name::check_size(size),)+ + RTAG_UNITS => UNITS::check_size(size), + RTAG_XY => XY::check_size(size), + _ => Ok(()), + } + } + }; } -//impl Record for HEADER; -pub struct BGNLIB; -impl Record for BGNLIB { - fn tag() -> u16 { RTAG_BGNLIB } - fn expected_size() -> Option { Some(2 * 6) } +fixed_size_records! { + HEADER => RTAG_HEADER, Int2, 2; + BGNLIB => RTAG_BGNLIB, DateTimePair, 2 * 2 * 6; + ENDLIB => RTAG_ENDLIB, Empty, 0; + BGNSTR => RTAG_BGNSTR, DateTimePair, 2 * 6 * 2; + ENDSTR => RTAG_ENDSTR, Empty, 0; + BOUNDARY => RTAG_BOUNDARY, Empty, 0; + PATH => RTAG_PATH, Empty, 0; + SREF => RTAG_SREF, Empty, 0; + AREF => RTAG_AREF, Empty, 0; + TEXT => RTAG_TEXT, Empty, 0; + LAYER => RTAG_LAYER, Int2, 2; + DATATYPE => RTAG_DATATYPE, Int2, 2; + WIDTH => RTAG_WIDTH, Int4, 4; + ENDEL => RTAG_ENDEL, Empty, 0; + COLROW => RTAG_COLROW, Int2Array, 4; + NODE => RTAG_NODE, Empty, 0; + TEXTTYPE => RTAG_TEXTTYPE, Int2, 2; + PRESENTATION => RTAG_PRESENTATION, BitArray, 2; + STRANS => RTAG_STRANS, BitArray, 2; + MAG => RTAG_MAG, Real8, 8; + ANGLE => RTAG_ANGLE, Real8, 8; + PATHTYPE => RTAG_PATHTYPE, Int2, 2; + GENERATIONS => RTAG_GENERATIONS, Int2, 2; + ELFLAGS => RTAG_ELFLAGS, BitArray, 2; + NODETYPE => RTAG_NODETYPE, Int2, 2; + PROPATTR => RTAG_PROPATTR, Int2, 2; + BOX => RTAG_BOX, Empty, 0; + BOXTYPE => RTAG_BOXTYPE, Int2, 2; + PLEX => RTAG_PLEX, Int4, 4; + BGNEXTN => RTAG_BGNEXTN, Int4, 4; + ENDEXTN => RTAG_ENDEXTN, Int4, 4; + STRCLASS => RTAG_STRCLASS, Int2, 2; + /// End of MASKS records + ENDMASKS => RTAG_ENDMASKS, Empty, 0; } pub struct LIBNAME; @@ -126,90 +152,121 @@ impl Record for UNITS { // (user_units_per_db_unit, db_units_per_meter) fn tag() -> u16 { RTAG_UNITS } fn expected_size() -> Option { Some(2 * 8) } -} -pub struct ENDLIB; -impl Record for ENDLIB { - fn tag() -> u16 { RTAG_ENDLIB } - fn expected_size() -> Option { Some(0) } -} - -pub struct BGNSTR; -impl Record for BGNSTR { - fn tag() -> u16 { RTAG_BGNSTR } - fn expected_size() -> Option { Some(2 * 6) } + fn read_data(input: &[u8], size: u16) -> IResult<'_, (f64, f64)> { + let (remaining, (user, meters)) = Real8Pair::read(input, size)?; + if !user.is_finite() || user <= 0.0 || !meters.is_finite() || meters <= 0.0 { + return fail(input, "UNITS must be positive finite values".to_string()); + } + Ok((remaining, (user, meters))) + } } pub struct STRNAME; impl Record for STRNAME { fn tag() -> u16 { RTAG_STRNAME } - fn expected_size() -> Option { Some(2 * 6) } -} - -pub struct ENDSTR; -impl Record for ENDSTR { - fn tag() -> u16 { RTAG_ENDSTR } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOUNDARY; -impl Record for BOUNDARY { - fn tag() -> u16 { RTAG_BOUNDARY } - fn expected_size() -> Option { Some(0) } -} - -pub struct PATH; -impl Record for PATH { - fn tag() -> u16 { RTAG_PATH } - fn expected_size() -> Option { Some(0) } -} - -pub struct SREF; -impl Record for SREF { - fn tag() -> u16 { RTAG_SREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct AREF; -impl Record for AREF { - fn tag() -> u16 { RTAG_AREF } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXT; -impl Record for TEXT { - fn tag() -> u16 { RTAG_TEXT } - fn expected_size() -> Option { Some(0) } -} - -pub struct LAYER; -impl Record for LAYER { - fn tag() -> u16 { RTAG_LAYER } - fn expected_size() -> Option { Some(2) } -} - -pub struct DATATYPE; -impl Record for DATATYPE { - fn tag() -> u16 { RTAG_DATATYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct WIDTH; -impl Record for WIDTH { - fn tag() -> u16 { RTAG_WIDTH } - fn expected_size() -> Option { Some(4) } + fn expected_size() -> Option { None } } pub struct XY; impl Record for XY { fn tag() -> u16 { RTAG_XY } fn expected_size() -> Option { None } + + fn check_size(size: u16) -> Result<(), String> { + if !size.is_multiple_of(8) { + return Err("XY payload must contain whole coordinate pairs".to_string()); + } + Ok(()) + } } -pub struct ENDEL; -impl Record for ENDEL { - fn tag() -> u16 { RTAG_ENDEL } - fn expected_size() -> Option { Some(0) } +impl XY { + pub(crate) fn point([x0, x1, x2, x3, y0, y1, y2, y3]: [u8; 8]) -> [i32; 2] { + [ + i32::from_be_bytes([x0, x1, x2, x3]), + i32::from_be_bytes([y0, y1, y2, y3]), + ] + } + + /// Iterate over a validated XY payload. + pub(crate) fn points(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .copied() + .map(Self::point) + } + + /// Bounds of a rectangle encoded as a closed XY ring. + pub(crate) fn closed_rectangle(data: &[u8]) -> Option<[i32; 4]> { + if data.len() != 40 { + return None; + } + let mut points = Self::points(data); + let corners: [[i32; 2]; 4] = std::array::from_fn(|_| points.next().unwrap()); + if points.next() != Some(corners[0]) { + return None; + } + Self::rectangle(&corners) + } + + /// Bounds of four corners in ring order, with the closing point omitted. + pub(crate) fn rectangle(corners: &[[i32; 2]]) -> Option<[i32; 4]> { + let corners: &[[i32; 2]; 4] = corners.try_into().ok()?; + let min_x = corners.iter().map(|point| point[0]).min().unwrap(); + let max_x = corners.iter().map(|point| point[0]).max().unwrap(); + let min_y = corners.iter().map(|point| point[1]).min().unwrap(); + let max_y = corners.iter().map(|point| point[1]).max().unwrap(); + if min_x == max_x || min_y == max_y { + return None; + } + + let mut seen = 0_u8; + for (index, &[x, y]) in corners.iter().enumerate() { + let [next_x, next_y] = corners[(index + 1) % corners.len()]; + if (x == next_x) == (y == next_y) { + return None; + } + if (x != min_x && x != max_x) || (y != min_y && y != max_y) { + return None; + } + let corner = u8::from(x == max_x) << 1 | u8::from(y == max_y); + seen |= 1 << corner; + } + (seen == 0b1111).then_some([min_x, min_y, max_x, max_y]) + } +} + +/// The array dimensions and its three coordinates in GDSII record order. +pub(crate) struct ArrayPlacement { + pub counts: [u16; 2], + pub points: [[i32; 2]; 3], +} + +impl ArrayPlacement { + pub fn decode<'a>(colrow: &'a [u8], xy: &'a [u8]) -> Result { + let (_, mut counts) = COLROW::read_data(colrow, colrow.len() as u16)?; + let columns = counts.next().unwrap(); + let rows = counts.next().unwrap(); + if columns <= 0 || rows <= 0 { + return Err(( + colrow, + ErrType::Failed("AREF column and row counts must be positive".to_string()), + )); + } + if xy.len() != 24 { + return Err(( + xy, + ErrType::Failed("AREF needs exactly three XY coordinate pairs".to_string()), + )); + } + let mut points = XY::points(xy); + Ok(Self { + counts: [columns as u16, rows as u16], + points: std::array::from_fn(|_| points.next().unwrap()), + }) + } } pub struct SNAME; @@ -218,30 +275,6 @@ impl Record for SNAME { fn expected_size() -> Option { None } } -pub struct COLROW; -impl Record for COLROW { - fn tag() -> u16 { RTAG_COLROW } - fn expected_size() -> Option { Some(4) } -} - -pub struct NODE; -impl Record for NODE { - fn tag() -> u16 { RTAG_NODE } - fn expected_size() -> Option { Some(0) } -} - -pub struct TEXTTYPE; -impl Record for TEXTTYPE { - fn tag() -> u16 { RTAG_TEXTTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PRESENTATION; -impl Record for PRESENTATION { - fn tag() -> u16 { RTAG_PRESENTATION } - fn expected_size() -> Option { Some(2) } -} - pub struct SPACING; impl Record for SPACING { fn tag() -> u16 { RTAG_SPACING } @@ -254,24 +287,6 @@ impl Record for STRING { fn expected_size() -> Option { None } } -pub struct STRANS; -impl Record for STRANS { - fn tag() -> u16 { RTAG_STRANS } - fn expected_size() -> Option { Some(2) } -} - -pub struct MAG; -impl Record for MAG { - fn tag() -> u16 { RTAG_MAG } - fn expected_size() -> Option { Some(8) } -} - -pub struct ANGLE; -impl Record for ANGLE { - fn tag() -> u16 { RTAG_ANGLE } - fn expected_size() -> Option { Some(8) } -} - pub struct UINTEGER; impl Record for UINTEGER { fn tag() -> u16 { RTAG_UINTEGER } @@ -288,10 +303,9 @@ pub struct REFLIBS; impl Record for REFLIBS { fn tag() -> u16 { RTAG_REFLIBS } fn expected_size() -> Option { None } -} -impl REFLIBS { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -303,10 +317,9 @@ pub struct FONTS; impl Record for FONTS { fn tag() -> u16 { RTAG_FONTS } fn expected_size() -> Option { None } -} -impl FONTS { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -314,26 +327,13 @@ impl FONTS { } } -pub struct PATHTYPE; -impl Record for PATHTYPE { - fn tag() -> u16 { RTAG_PATHTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct GENERATIONS; -impl Record for GENERATIONS { - fn tag() -> u16 { RTAG_GENERATIONS } - fn expected_size() -> Option { Some(2) } -} - pub struct ATTRTABLE; impl Record for ATTRTABLE { fn tag() -> u16 { RTAG_ATTRTABLE } fn expected_size() -> Option { None } -} -impl ATTRTABLE { - pub fn check_size(actual_size: usize) -> Result<(), String> { - if actual_size % 44 == 0 { + + fn check_size(actual_size: u16) -> Result<(), String> { + if actual_size.is_multiple_of(44) { Ok(()) } else { Err(format!("Expected record size divisible by 44, got {}", actual_size)) @@ -353,12 +353,6 @@ impl Record for STRTYPE { fn expected_size() -> Option { None } } -pub struct ELFLAGS; -impl Record for ELFLAGS { - fn tag() -> u16 { RTAG_ELFLAGS } - fn expected_size() -> Option { Some(2) } -} - pub struct ELKEY; impl Record for ELKEY { fn tag() -> u16 { RTAG_ELKEY } @@ -377,52 +371,10 @@ impl Record for LINKKEYS { fn expected_size() -> Option { Some(2) } } -pub struct NODETYPE; -impl Record for NODETYPE { - fn tag() -> u16 { RTAG_NODETYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PROPATTR; -impl Record for PROPATTR { - fn tag() -> u16 { RTAG_PROPATTR } - fn expected_size() -> Option { Some(2) } -} - pub struct PROPVALUE; impl Record for PROPVALUE { fn tag() -> u16 { RTAG_PROPVALUE } - fn expected_size() -> Option { Some(2) } -} - -pub struct BOX; -impl Record for BOX { - fn tag() -> u16 { RTAG_BOX } - fn expected_size() -> Option { Some(0) } -} - -pub struct BOXTYPE; -impl Record for BOXTYPE { - fn tag() -> u16 { RTAG_BOXTYPE } - fn expected_size() -> Option { Some(2) } -} - -pub struct PLEX; -impl Record for PLEX { - fn tag() -> u16 { RTAG_PLEX } - fn expected_size() -> Option { Some(4) } -} - -pub struct BGNEXTN; -impl Record for BGNEXTN { - fn tag() -> u16 { RTAG_BGNEXTN } - fn expected_size() -> Option { Some(4) } -} - -pub struct ENDEXTN; -impl Record for ENDEXTN { - fn tag() -> u16 { RTAG_ENDEXTN } - fn expected_size() -> Option { Some(4) } + fn expected_size() -> Option { None } } pub struct TAPENUM; @@ -437,12 +389,6 @@ impl Record for TAPECODE { fn expected_size() -> Option { Some(2 * 6) } } -pub struct STRCLASS; -impl Record for STRCLASS { - fn tag() -> u16 { RTAG_STRCLASS } - fn expected_size() -> Option { Some(2) } -} - pub struct RESERVED; impl Record for RESERVED { fn tag() -> u16 { RTAG_RESERVED } @@ -461,13 +407,6 @@ impl Record for MASK { fn expected_size() -> Option { None } } -/// End of MASKS records -pub struct ENDMASKS; -impl Record for ENDMASKS { - fn tag() -> u16 { RTAG_ENDMASKS } - fn expected_size() -> Option { Some(0) } -} - pub struct LIBDIRSIZE; impl Record for LIBDIRSIZE { fn tag() -> u16 { RTAG_LIBDIRSIZE } @@ -545,3 +484,23 @@ impl Record for CONTACT { fn tag() -> u16 { RTAG_CONTACT } fn expected_size() -> Option { Some(0) } } + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bgnstr_reads_both_timestamps() { + let mut bytes = vec![0, 28, 0x05, 0x02]; + for value in [126_i16, 11, 14, 12, 30, 0, 125, 10, 13, 9, 15, 1] { + bytes.extend_from_slice(&value.to_be_bytes()); + } + bytes.extend_from_slice(&[0, 4, 0x07, 0]); + + let (remaining, timestamps) = BGNSTR::read(&bytes).unwrap(); + + assert_eq!(timestamps, [[2026, 11, 14, 12, 30, 0], [2025, 10, 13, 9, 15, 1]]); + assert_eq!(remaining, &[0, 4, 0x07, 0]); + } +} diff --git a/src/review.rs b/src/review.rs new file mode 100644 index 0000000..baf99c4 --- /dev/null +++ b/src/review.rs @@ -0,0 +1,1679 @@ +//! Deterministic GDS object review without the normalization performed by the +//! Arrow reader. Raw payloads borrow the file; only polygon keys are copied. + +use crate::{ + basic::{ErrType, decode_name, decode_real8, encode_real8, unpad_ascii}, + record::{Record as _, RecordFrame}, + records::*, +}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet}, + fmt::{self, Write}, +}; + +pub mod changes; +mod hierarchy; +pub mod names; +mod pair; +pub mod report; +pub mod selection; +mod text; + +use changes::Replacement; +use pair::CellReuse; +pub use pair::RevisionPair; +use selection::{DisplayFilter, Scope}; +pub use text::{TextReport, TextReportOptions, TextStyle}; + +/// Input revision associated with a diagnostic. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Side { + Old, + New, +} + +impl fmt::Display for Side { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + out.write_str(match self { + Self::Old => "old", + Self::New => "new", + }) + } +} + +/// A review error or comparison limitation, with structured source context. +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub code: &'static str, + pub message: String, + pub side: Option, + pub offset: Option, + pub cell: Option, + pub layer: Option, + pub target: Option, +} + +impl Diagnostic { + pub fn new(code: &'static str, message: impl Into, side: Option) -> Self { + Self { + code, + message: message.into(), + side, + offset: None, + cell: None, + layer: None, + target: None, + } + } + + fn at(code: &'static str, offset: usize, message: String) -> Self { + Self { + offset: Some(offset), + ..Self::new(code, error(offset, message), None) + } + } +} + +impl fmt::Display for Diagnostic { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(side) = self.side { + write!(out, "{side}: ")?; + } + out.write_str(&self.message) + } +} + +/// The five meaningful combinations of completeness and comparison result. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Outcome { + Equal, + Different, + Incomplete { different: bool }, + Failed, +} + +impl Outcome { + pub fn status(self) -> &'static str { + match self { + Self::Equal | Self::Different => "complete", + Self::Incomplete { .. } => "incomplete", + Self::Failed => "failed", + } + } + + pub fn result(self) -> &'static str { + match self { + Self::Equal => "equal", + Self::Different | Self::Incomplete { different: true } => "different", + Self::Incomplete { different: false } | Self::Failed => "unknown", + } + } + + pub fn exit_code(self) -> u8 { + match self { + Self::Equal => 0, + Self::Different => 1, + _ => 2, + } + } + + pub fn is_equal(self) -> bool { + self == Self::Equal + } + + pub fn is_complete(self) -> bool { + matches!(self, Self::Equal | Self::Different) + } +} + +pub const PROFILE: &str = "pheon-structural-v1"; +pub type Layer = (u16, u16); +pub type Bounds = [i32; 4]; + +#[derive(Clone, Debug)] +pub struct Record<'a> { + pub offset: usize, + pub tag: u16, + pub data: &'a [u8], +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct Key<'a> { + kind: u16, + fields: Vec<(u16, Cow<'a, [u8]>)>, + properties: Vec<(u16, &'a [u8])>, +} + +#[derive(Clone, Debug)] +pub struct Object<'a> { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub multiplicity: u64, + key: Key<'a>, + metadata: bool, + issues: Vec, +} + +impl Object<'_> { + fn field(&self, tag: u16) -> Option<&[u8]> { + self.key + .fields + .iter() + .find(|(t, _)| *t == tag) + .map(|(_, v)| v.as_ref()) + } +} + +fn format_object<'a, W: fmt::Write>( + out: &mut W, + kind: u16, + layer: Option, + fields: impl Iterator + Clone, + properties: impl IntoIterator, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, +) -> fmt::Result { + let field = |tag| { + fields + .clone() + .find_map(|(t, data)| (t == tag).then_some(data)) + }; + let rectangle = if is_polygon(kind) { + field(RTAG_XY).and_then(XY::closed_rectangle) + } else { + None + }; + let target = if matches!(kind, RTAG_SREF | RTAG_AREF) { + field(RTAG_SNAME) + } else { + None + }; + let dimensions = if kind == RTAG_AREF { + field(RTAG_COLROW).and_then(array_dimensions) + } else { + None + }; + heading(out, object_name(kind, rectangle.is_some()))?; + if let Some(target) = target { + write!(out, " {}", quoted(target))?; + } + if let Some([columns, rows]) = dimensions { + write!(out, " ({columns} × {rows})")?; + } + for (tag, value) in fields { + if (layer.is_some() && matches_layer_field(kind, tag)) + || (tag == RTAG_SNAME && target.is_some()) + || (tag == RTAG_COLROW && dimensions.is_some()) + { + continue; + } + if tag == RTAG_XY && value.len().is_multiple_of(8) { + if let Some([xmin, ymin, xmax, ymax]) = rectangle { + write!(out, " ({xmin}, {ymin}) to ({xmax}, {ymax})")?; + } else { + write_points(out, value)?; + } + } else { + write!(out, " {}={}", tag_name(tag), format_exact_value(tag, value))?; + } + } + for (attribute, value) in properties { + write!(out, " property[{attribute}]={}", quoted(value))?; + } + Ok(()) +} + +fn array_dimensions(data: &[u8]) -> Option<[u16; 2]> { + let [column_hi, column_lo, row_hi, row_lo]: [u8; 4] = data.try_into().ok()?; + Some([ + u16::from_be_bytes([column_hi, column_lo]), + u16::from_be_bytes([row_hi, row_lo]), + ]) +} + +const POINTS_PER_LINE: usize = 4; + +fn write_points(out: &mut impl fmt::Write, data: &[u8]) -> fmt::Result { + let mut points = XY::points(data); + if points.len() == 1 { + let [x, y] = points.next().unwrap(); + return write!(out, " xy=({x}, {y})"); + } + out.write_str(" xy=[")?; + for (index, [x, y]) in points.enumerate() { + if index != 0 { + out.write_str(if index % POINTS_PER_LINE == 0 { + ",\n " + } else { + ", " + })?; + } + write!(out, "({x}, {y})")?; + } + out.write_str("]") +} + +impl fmt::Display for Object<'_> { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +fn write_object_heading( + out: &mut impl fmt::Write, + layer: Option, + name: &str, +) -> fmt::Result { + if let Some((layer, datatype)) = layer { + write!(out, "{layer}/{datatype}: ")?; + } + out.write_str(name) +} + +impl Object<'_> { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { + format_object( + out, + self.kind, + self.layer, + self.key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + self.key.properties.iter().copied(), + heading, + ) + } +} + +#[derive(Debug)] +pub struct Cell<'a> { + pub offset: usize, + /// Start of ENDSTR, for inserting records without rewriting the cell. + pub end_offset: usize, + pub objects: Vec>, + attributes: Vec<(u16, &'a [u8])>, + issues: Vec, +} + +#[derive(Debug)] +pub struct Document<'a> { + pub records: Vec>, + pub cells: BTreeMap>, + pub layer_names: BTreeMap, + /// Unsupported features or invalid reference graphs prevent an equality claim. + pub diagnostics: Vec, + header: Vec<(u16, &'a [u8])>, + global_issues: Vec, + known_cells: BTreeSet, + pub scope: Scope, + present: bool, +} + +fn error(offset: usize, message: impl std::fmt::Display) -> String { + format!("at byte {offset}: {message}") +} + +fn decoded_string(data: &[u8], offset: usize) -> Result { + if data.contains(&0) { + return Err(error(offset, "name or metadata contains an embedded NUL")); + } + std::str::from_utf8(data) + .map(str::to_owned) + .map_err(|_| error(offset, "name or metadata is not UTF-8")) +} + +fn integer(data: &[u8]) -> u16 { + u16::from_be_bytes(data.try_into().unwrap()) +} + +fn coordinate_bounds(points: impl IntoIterator) -> Bounds { + let mut points = points.into_iter(); + let [x, y] = points.next().unwrap(); + points.fold([x, y, x, y], |[xmin, ymin, xmax, ymax], [x, y]| { + [xmin.min(x), ymin.min(y), xmax.max(x), ymax.max(y)] + }) +} + +// Booth's least-rotation algorithm avoids quadratic work on repeated vertices. +fn least_rotation(n: usize, point: impl Fn(usize) -> [i32; 2]) -> usize { + let (mut i, mut j, mut k) = (0, 1, 0); + while i < n && j < n && k < n { + match point((i + k) % n).cmp(&point((j + k) % n)) { + std::cmp::Ordering::Equal => k += 1, + std::cmp::Ordering::Greater => { + i += k + 1; + if i <= j { + i = j + 1; + } + k = 0; + } + std::cmp::Ordering::Less => { + j += k + 1; + if j <= i { + j = i + 1; + } + k = 0; + } + } + } + i.min(j) +} + +fn canonical_ring(data: &[u8]) -> Option> { + let (points, _) = data.as_chunks::<8>(); + let (last, ring) = points.split_last()?; + if ring.len() < 3 || ring.first() != Some(last) { + return None; + } + let n = ring.len(); + let first = least_rotation(n, |index| XY::point(ring[index])); + let split = n - least_rotation(n, |index| XY::point(ring[n - 1 - index])); + let forward = ring[first..].iter().chain(&ring[..first]); + let backward = ring[..split].iter().rev().chain(ring[split..].iter().rev()); + let decode = |point: &[u8; 8]| XY::point(*point); + let use_forward = forward + .clone() + .map(decode) + .cmp(backward.clone().map(decode)) + .is_le(); + if use_forward && first == 0 { + return Some(Cow::Borrowed(data)); + } + let mut result = Vec::with_capacity(data.len()); + if use_forward { + result.extend(forward.flatten().copied()); + } else { + result.extend(backward.flatten().copied()); + } + result.extend_from_within(..8); + Some(Cow::Owned(result)) +} + +fn validate_payload(record: &Record<'_>) -> Result<(), String> { + let size = record.data.len() as u16; + check_payload_size(record.tag, size).map_err(|message| { + error( + record.offset, + format!( + "invalid payload size {size} for {}: {message}", + tag_name(record.tag) + ), + ) + })?; + if record.tag == RTAG_UNITS { + UNITS::read_data(record.data, size) + .map_err(|(_, reason)| payload_error(record.offset, reason))?; + } + Ok(()) +} + +fn payload_error(offset: usize, reason: ErrType) -> String { + match reason { + ErrType::Failed(message) => error(offset, message), + ErrType::Incomplete(_) => error(offset, "truncated record payload"), + } +} + +fn is_element(tag: u16) -> bool { + matches!( + tag, + RTAG_BOUNDARY | RTAG_PATH | RTAG_SREF | RTAG_AREF | RTAG_TEXT | RTAG_NODE | RTAG_BOX + ) +} + +fn is_polygon(kind: u16) -> bool { + matches!(kind, RTAG_BOUNDARY | RTAG_BOX) +} + +fn layer_type_tag(kind: u16) -> u16 { + match kind { + RTAG_TEXT => RTAG_TEXTTYPE, + RTAG_NODE => RTAG_NODETYPE, + RTAG_BOX => RTAG_BOXTYPE, + _ => RTAG_DATATYPE, + } +} + +fn matches_layer_field(kind: u16, tag: u16) -> bool { + tag == RTAG_LAYER || tag == layer_type_tag(kind) +} + +fn allowed_field(kind: u16, tag: u16) -> bool { + if matches!(tag, RTAG_ELFLAGS | RTAG_PLEX | RTAG_XY) { + return true; + } + match kind { + RTAG_BOUNDARY => matches!(tag, RTAG_LAYER | RTAG_DATATYPE), + RTAG_BOX => matches!(tag, RTAG_LAYER | RTAG_BOXTYPE), + RTAG_NODE => matches!(tag, RTAG_LAYER | RTAG_NODETYPE), + RTAG_PATH => matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_WIDTH | RTAG_PATHTYPE | RTAG_BGNEXTN | RTAG_ENDEXTN + ), + RTAG_SREF => matches!(tag, RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE), + RTAG_AREF => matches!( + tag, + RTAG_SNAME | RTAG_STRANS | RTAG_MAG | RTAG_ANGLE | RTAG_COLROW + ), + RTAG_TEXT => matches!( + tag, + RTAG_LAYER + | RTAG_TEXTTYPE + | RTAG_STRING + | RTAG_STRANS + | RTAG_MAG + | RTAG_ANGLE + | RTAG_PRESENTATION + | RTAG_PATHTYPE + | RTAG_WIDTH + ), + _ => false, + } +} + +fn object<'a>(records: &[Record<'a>], issues: &mut Vec) -> Result, String> { + let first = &records[0]; + let mut fields = Vec::new(); + let mut properties = Vec::new(); + let mut target = None; + let mut seen = BTreeSet::new(); + let mut i = 1; + while i + 1 < records.len() { + let record = &records[i]; + if record.tag == RTAG_PROPATTR { + let value = records + .get(i + 1) + .filter(|r| r.tag == RTAG_PROPVALUE) + .ok_or_else(|| error(record.offset, "PROPATTR without PROPVALUE"))?; + properties.push((integer(record.data), unpad_ascii(value.data))); + i += 2; + continue; + } + if !allowed_field(first.tag, record.tag) { + issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!( + "unsupported record 0x{:04x} in {}", + record.tag, + kind_name(first.tag) + ), + )); + } + if !seen.insert(record.tag) { + return Err(error(record.offset, "duplicate element field")); + } + let mut data = Cow::Borrowed(record.data); + if record.tag == RTAG_SNAME && matches!(first.tag, RTAG_SREF | RTAG_AREF) { + let name = decode_name(record.data).map_err(|message| error(record.offset, message))?; + target = Some(name.to_owned()); + data = Cow::Borrowed(name.as_bytes()); + } else if record.tag & 0xff == 6 { + data = Cow::Borrowed(unpad_ascii(record.data)); + } + if record.tag == RTAG_XY && is_polygon(first.tag) { + data = canonical_ring(record.data).ok_or_else(|| { + error( + record.offset, + "polygon must have at least three vertices and be closed", + ) + })?; + } + fields.push((record.tag, data)); + i += 1; + } + fields.sort(); + properties.sort(); + let mut object = Object { + offset: first.offset, + end_offset: records.last().unwrap().offset + 4, + kind: first.tag, + layer: None, + bounds: None, + target, + multiplicity: 1, + key: Key { + kind: first.tag, + fields, + properties, + }, + metadata: false, + issues: Vec::new(), + }; + let xy = object + .field(RTAG_XY) + .ok_or_else(|| error(first.offset, "element has no XY record"))?; + object.bounds = Some(if first.tag == RTAG_AREF { + let counts = object + .field(RTAG_COLROW) + .ok_or_else(|| error(first.offset, "array has no COLROW"))?; + let placement = ArrayPlacement::decode(counts, xy) + .map_err(|(_, reason)| payload_error(first.offset, reason))?; + let [columns, rows] = placement.counts; + object.multiplicity = u64::from(columns) * u64::from(rows); + coordinate_bounds(placement.points) + } else { + let points = XY::points(xy); + let valid_count = match first.tag { + RTAG_TEXT | RTAG_SREF => points.len() == 1, + RTAG_PATH => points.len() >= 2, + RTAG_NODE => points.len() > 0, + RTAG_BOX => points.len() == 5, + _ => true, + }; + if !valid_count { + return Err(error(first.offset, "invalid coordinate count")); + } + coordinate_bounds(points) + }); + if matches!(first.tag, RTAG_SREF | RTAG_AREF) { + if object.target.is_none() { + return Err(error(first.offset, "reference has no SNAME")); + } + } else { + let dtype = layer_type_tag(first.tag); + object.layer = Some(( + integer( + object + .field(RTAG_LAYER) + .ok_or_else(|| error(first.offset, "element has no LAYER"))?, + ), + integer( + object + .field(dtype) + .ok_or_else(|| error(first.offset, "element has no datatype/texttype"))?, + ), + )); + if first.tag == RTAG_TEXT && object.field(RTAG_STRING).is_none() { + return Err(error(first.offset, "text has no STRING")); + } + } + Ok(object) +} + +fn read_records(input: &[u8]) -> Result>, String> { + read_records_with(input, |_, _| {}) +} + +fn read_records_with<'a>( + input: &'a [u8], + mut observe: impl FnMut(usize, &Record<'a>), +) -> Result>, String> { + if input.starts_with(b"\x1f\x8b") { + return Err(error( + 0, + "gzip-compressed input. Decompress it before reading GDSII", + )); + } + if input.starts_with(b"%SEMI-OASIS\r\n") { + return Err(error(0, "OASIS input. Export the layout as GDSII")); + } + + let mut records = Vec::new(); + let mut remaining = input; + while !remaining.is_empty() { + let offset = input.len() - remaining.len(); + let (next, frame) = RecordFrame::read(remaining).map_err(|(_, reason)| match reason { + ErrType::Incomplete(_) => error(offset, "truncated record header"), + ErrType::Failed(message) => error(offset, message), + })?; + let record = Record { + offset, + tag: frame.tag, + data: frame.data, + }; + observe(records.len(), &record); + records.push(record); + remaining = next; + if frame.tag == RTAG_ENDLIB { + if remaining.iter().any(|b| *b != 0) { + return Err(error( + input.len() - remaining.len(), + "nonzero data after ENDLIB", + )); + } + break; + } + } + Ok(records) +} + +impl<'a> Document<'a> { + pub fn parse(input: &'a [u8]) -> Result { + let records = read_records(input)?; + let mut doc = Self::parse_records(&records, None)?; + doc.records = records; + Ok(doc) + } + + fn parse_records( + records: &[Record<'a>], + reuse: Option>, + ) -> Result { + for record in records { + validate_payload(record)?; + } + if records.first().map(|r| r.tag) != Some(RTAG_HEADER) + || records.get(1).map(|r| r.tag) != Some(RTAG_BGNLIB) + || records.last().map(|r| r.tag) != Some(RTAG_ENDLIB) + { + return Err("expected HEADER, BGNLIB, and a final ENDLIB".to_string()); + } + let mut doc = Self { + records: Vec::new(), + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + diagnostics: Vec::new(), + header: Vec::new(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: Scope::default(), + present: true, + }; + let mut i = 2; + let mut global_seen = BTreeSet::new(); + let mut cells_started = false; + doc.header.push((RTAG_HEADER, records[0].data)); + while i + 1 < records.len() { + let record = &records[i]; + if record.tag != RTAG_BGNSTR { + if cells_started { + return Err(error( + record.offset, + "library field outside the library header", + )); + } + if record.tag != RTAG_MASK && !global_seen.insert(record.tag) { + return Err(error(record.offset, "duplicate library field")); + } + if !matches!( + record.tag, + RTAG_LIBNAME + | RTAG_UNITS + | RTAG_REFLIBS + | RTAG_FONTS + | RTAG_GENERATIONS + | RTAG_ATTRTABLE + | RTAG_FORMAT + | RTAG_MASK + | RTAG_ENDMASKS + | RTAG_LIBDIRSIZE + | RTAG_SRFNAME + | RTAG_LIBSECUR + ) { + doc.global_issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported library record 0x{:04x}", record.tag), + )); + } + doc.header.push(( + record.tag, + if record.tag == RTAG_LIBNAME { + decode_name(record.data) + .map_err(|message| error(record.offset, message))? + .as_bytes() + } else if record.tag & 0xff == 6 { + unpad_ascii(record.data) + } else { + record.data + }, + )); + i += 1; + continue; + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err(error(record.offset, "LIBNAME and UNITS must precede cells")); + } + cells_started = true; + let offset = record.offset; + i += 1; + let name_record = records + .get(i) + .filter(|r| r.tag == RTAG_STRNAME) + .ok_or_else(|| error(offset, "BGNSTR must be followed by STRNAME"))?; + let name = decode_name(name_record.data) + .map_err(|message| error(name_record.offset, message))? + .to_owned(); + if name.is_empty() { + return Err(error(name_record.offset, "empty cell name")); + } + if let Some(reuse) = &reuse + && let Some((cell, end)) = reuse.cell(&name, offset, records) + { + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + i = end + 1; + continue; + } + i += 1; + let mut cell = Cell { + offset, + end_offset: 0, + objects: Vec::new(), + attributes: Vec::new(), + issues: Vec::new(), + }; + loop { + let record = records + .get(i) + .ok_or_else(|| error(offset, "missing ENDSTR"))?; + if record.tag == RTAG_ENDSTR { + cell.end_offset = record.offset; + i += 1; + break; + } + if is_element(record.tag) { + let start = i; + i += 1; + while records.get(i).is_some_and(|r| r.tag != RTAG_ENDEL) { + if matches!(records[i].tag, RTAG_ENDSTR | RTAG_ENDLIB | RTAG_BGNSTR) + || is_element(records[i].tag) + { + return Err(error(records[i].offset, "missing ENDEL")); + } + i += 1; + } + if i == records.len() { + return Err(error(offset, "missing ENDEL")); + } + let mut issues = Vec::new(); + let mut parsed = object(&records[start..=i], &mut issues)?; + parsed.issues = issues; + cell.objects.push(parsed); + i += 1; + } else { + if matches!(record.tag, RTAG_ENDLIB | RTAG_BGNSTR) { + return Err(error(record.offset, "missing ENDSTR")); + } + if record.tag != RTAG_STRCLASS { + cell.issues.push(Diagnostic::at( + "unsupported_record", + record.offset, + format!("unsupported cell record 0x{:04x}", record.tag), + )); + } else if !cell.objects.is_empty() + || cell.attributes.iter().any(|(tag, _)| *tag == RTAG_STRCLASS) + { + return Err(error( + record.offset, + "STRCLASS must occur once before elements", + )); + } + cell.attributes.push((record.tag, record.data)); + i += 1; + } + } + cell.attributes.sort(); + cell.objects.sort_by(|a, b| a.key.cmp(&b.key)); + if doc.cells.insert(name.clone(), cell).is_some() { + return Err(error(offset, format!("duplicate cell name {name:?}"))); + } + } + if !global_seen.contains(&RTAG_LIBNAME) || !global_seen.contains(&RTAG_UNITS) { + return Err("library must contain LIBNAME and UNITS".to_string()); + } + doc.header.sort(); + doc.read_metadata()?; + doc.known_cells = doc.cells.keys().cloned().collect(); + doc.refresh_diagnostics(); + Ok(doc) + } + + fn check_graph(&mut self) { + let mut unresolved = BTreeMap::new(); + for (parent, cell) in &self.cells { + for object in &cell.objects { + if let Some(target) = &object.target + && !self.known_cells.contains(target) + { + unresolved.entry((parent, target)).or_insert(object.offset); + } + } + } + for ((parent, target), offset) in unresolved { + self.diagnostics.push(Diagnostic { + code: "unresolved_reference", + side: None, + message: format!("cell {parent:?}: unresolved reference {target:?}"), + offset: Some(offset), + cell: Some(parent.clone()), + layer: None, + target: Some(target.clone()), + }); + } + let mut state = BTreeMap::<&str, u8>::new(); + for root in self.cells.keys() { + let mut stack = vec![(root.as_str(), false)]; + while let Some((name, leaving)) = stack.pop() { + if leaving { + state.insert(name, 2); + continue; + } + match state.get(name) { + Some(2) => continue, + Some(1) => { + self.diagnostics.push(Diagnostic { + code: "hierarchy_cycle", + side: None, + message: format!("cyclic hierarchy involving {name:?}"), + offset: self.cells.get(name).map(|c| c.offset), + cell: Some(name.to_string()), + layer: None, + target: None, + }); + continue; + } + _ => (), + } + state.insert(name, 1); + stack.push((name, true)); + if let Some(cell) = self.cells.get(name) { + for object in &cell.objects { + if let Some(target) = &object.target { + stack.push((target, false)); + } + } + } + } + } + } + + pub fn edges(&self) -> BTreeMap<(String, String), u64> { + self.cells + .iter() + .flat_map(|(name, cell)| { + let mut counts = BTreeMap::new(); + for object in &cell.objects { + if let Some(target) = object.target.as_deref() { + let count = counts.entry(target).or_insert(0_u64); + *count = count.saturating_add(object.multiplicity); + } + } + counts + .into_iter() + .map(|(target, count)| ((name.clone(), target.to_owned()), count)) + }) + .collect() + } + + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + let revision = Revision::from_document(self); + hierarchy::render_hierarchy(&revision, &revision, None, mermaid, max_depth) + } + + pub fn canonical_dump(&self) -> String { + let mut out = TextReport::default(); + self.write_canonical_dump(&mut out); + out.into_string() + } + + /// Append the canonical object dump and its presentation roles. + pub fn write_canonical_dump(&self, out: &mut TextReport) { + if !self.present { + out.write_styled(TextStyle::Heading, format_args!("library absent")); + out.push('\n'); + return; + } + let version_index = self + .header + .iter() + .position(|(tag, _)| *tag == RTAG_HEADER) + .unwrap(); + let header = |tag| self.header.iter().find(|(t, _)| *t == tag).unwrap().1; + let (user, meters) = header(RTAG_UNITS).split_at(8); + let version = i16::from_be_bytes(self.header[version_index].1.try_into().unwrap()); + out.write_styled( + TextStyle::Heading, + format_args!( + "library {} (GDSII {version}, {} user units/dbu, {} m/dbu)", + quoted(header(RTAG_LIBNAME)), + format_exact_value(RTAG_UNITS, user), + format_exact_value(RTAG_UNITS, meters) + ), + ); + out.push('\n'); + self.scope.write_restrictions(out); + for (index, (tag, data)) in self.header.iter().enumerate() { + if index != version_index && !matches!(*tag, RTAG_LIBNAME | RTAG_UNITS) { + writeln!(out, "{}={}", tag_name(*tag), format_exact_value(*tag, data)).unwrap(); + } + } + for ((layer, dtype), name) in &self.layer_names { + out.write_styled(TextStyle::Layer, format_args!("{layer}/{dtype}")); + writeln!(out, ": name {name:?}").unwrap(); + } + for (name, cell) in &self.cells { + out.write_styled(TextStyle::Heading, format_args!("cell {name:?}")); + out.push('\n'); + for (tag, data) in &cell.attributes { + writeln!( + out, + " {}={}", + tag_name(*tag), + format_exact_value(*tag, data) + ) + .unwrap(); + } + let mut objects = cell + .objects + .iter() + .filter(|object| !object.metadata) + .peekable(); + while let Some(object) = objects.next() { + let mut count = 1; + while objects.next_if(|next| next.key == object.key).is_some() { + count += 1; + } + out.push_str(" "); + if count != 1 { + write!(out, "{count}x ").unwrap(); + } + object + .format_with_heading(out, |out, name| { + if let Some((layer, datatype)) = object.layer { + out.write_styled(TextStyle::Layer, format_args!("{layer}/{datatype}")); + out.push_str(": "); + } + if matches!(object.kind, RTAG_SREF | RTAG_AREF) && object.layer.is_none() { + out.write_styled(TextStyle::Layer, format_args!("{name}")); + } else { + out.push_str(name); + } + Ok(()) + }) + .unwrap(); + out.push('\n'); + } + } + for diagnostic in &self.diagnostics { + out.write_styled( + TextStyle::Warning, + format_args!("INCOMPLETE: {}", diagnostic.message), + ); + out.push('\n'); + } + } + + pub fn record_dump(&self) -> String { + format_records(&self.records) + } +} + +/// Inspect record boundaries even when object grammar or metadata is invalid. +pub fn dump_records(input: &[u8]) -> Result { + let mut out = TextReport::default(); + write_record_dump(&mut out, input)?; + Ok(out.into_string()) +} + +/// Append the raw record dump and its presentation roles. +pub fn write_record_dump(out: &mut TextReport, input: &[u8]) -> Result<(), String> { + write_records(out, &read_records(input)?); + Ok(()) +} + +fn format_records(records: &[Record<'_>]) -> String { + let mut out = TextReport::default(); + write_records(&mut out, records); + out.into_string() +} + +fn write_records(out: &mut TextReport, records: &[Record<'_>]) { + for record in records { + write!(out, "{:010} {:04x} ", record.offset, record.tag).unwrap(); + out.write_styled(TextStyle::Layer, format_args!("{}", tag_name(record.tag))); + writeln!( + out, + " size={} {}", + record.data.len() + 4, + format_data( + record.tag, + if record.tag & 0xff == 6 { + unpad_ascii(record.data) + } else { + record.data + } + ) + ) + .unwrap(); + } +} + +/// A retained comparison field. Polygon coordinates and string padding follow +/// the comparison profile; REAL encodings and unsupported payloads are exact. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FieldValue { + pub tag: u16, + pub data: Vec, +} + +fn owned_fields<'a>(fields: impl IntoIterator) -> Vec { + fields + .into_iter() + .map(|(tag, data)| FieldValue { + tag, + data: data.to_vec(), + }) + .collect() +} + +#[derive(Debug)] +pub struct ObjectChange { + pub offset: usize, + pub end_offset: usize, + pub kind: u16, + pub layer: Option, + pub bounds: Option, + pub target: Option, + pub origin: Option<[i32; 2]>, + pub width: Option, + pub counts: Option<[u16; 2]>, + pub fields: Vec, + pub properties: Vec<(u16, Vec)>, +} + +impl fmt::Display for ObjectChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + self.format_with_heading(out, |out, name| write_object_heading(out, self.layer, name)) + } +} + +impl ObjectChange { + fn format_with_heading( + &self, + out: &mut W, + heading: impl FnOnce(&mut W, &'static str) -> fmt::Result, + ) -> fmt::Result { + format_object( + out, + self.kind, + self.layer, + self.fields + .iter() + .map(|field| (field.tag, field.data.as_slice())), + self.properties + .iter() + .map(|(attribute, data)| (*attribute, data.as_slice())), + heading, + ) + } +} + +impl From<&Object<'_>> for ObjectChange { + fn from(object: &Object<'_>) -> Self { + Self { + offset: object.offset, + end_offset: object.end_offset, + kind: object.kind, + layer: object.layer, + bounds: object.bounds, + target: object.target.clone(), + origin: object + .field(RTAG_XY) + .and_then(|data| XY::points(data).next()), + width: object + .field(RTAG_WIDTH) + .map(|data| i32::from_be_bytes(data.try_into().unwrap())), + counts: object.target.as_ref().and_then(|_| { + if object.kind == RTAG_AREF { + object.field(RTAG_COLROW).and_then(array_dimensions) + } else { + Some([1, 1]) + } + }), + fields: owned_fields( + object + .key + .fields + .iter() + .map(|(tag, data)| (*tag, data.as_ref())), + ), + properties: object + .key + .properties + .iter() + .map(|(a, v)| (*a, v.to_vec())) + .collect(), + } + } +} + +/// An existing cell's source location and retained attributes. +#[derive(Debug)] +pub struct CellRevision { + pub offset: usize, + pub attributes: Vec, +} + +impl From<&Cell<'_>> for CellRevision { + fn from(cell: &Cell<'_>) -> Self { + Self { + offset: cell.offset, + attributes: owned_fields(cell.attributes.iter().copied()), + } + } +} + +#[derive(Debug)] +pub struct CellChange { + pub name: String, + pub old: Option, + pub new: Option, + pub removed: Vec, + pub added: Vec, + pub replacements: Vec, +} + +impl CellChange { + pub fn attributes_changed(&self) -> bool { + self.old.as_ref().map(|cell| &cell.attributes) + != self.new.as_ref().map(|cell| &cell.attributes) + } +} + +#[derive(Debug)] +pub struct LayerNameChange { + pub layer: Layer, + pub old: Option, + pub new: Option, +} + +/// Library presence or exact record payloads, formatted only when rendered. +#[derive(Debug)] +pub enum LibraryChange { + Presence { + old: bool, + new: bool, + }, + Record { + tag: u16, + old: Vec>, + new: Vec>, + }, +} + +impl fmt::Display for LibraryChange { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Presence { old, new } => out.write_str(match (old, new) { + (false, true) => "added", + (true, false) => "removed", + (true, true) => "present", + (false, false) => "absent", + }), + Self::Record { tag, old, new } => { + let values = if old.len() <= 1 && new.len() <= 1 { + format_field_change( + *tag, + old.first().map(Vec::as_slice), + new.first().map(Vec::as_slice), + ) + } else { + let list = |items: &[Vec]| { + if items.is_empty() { + return "absent".to_string(); + } + let values = items + .iter() + .map(|data| format_exact_value(*tag, data)) + .collect::>() + .join(", "); + format!("[{values}]") + }; + format!("{} -> {}", list(old), list(new)) + }; + write!(out, "{} {values}", tag_name(*tag)) + } + } + } +} + +/// Owned revision metadata and reference counts captured before documents are dropped. +#[derive(Debug)] +pub struct Revision { + pub scope: Scope, + pub present: bool, + pub units: Option>, + cells: BTreeSet, + known_cells: BTreeSet, + edges: BTreeMap<(String, String), u64>, +} + +impl Revision { + fn from_document(doc: &Document<'_>) -> Self { + Self { + scope: doc.scope.clone(), + present: doc.present, + units: doc + .header + .iter() + .find(|(tag, _)| *tag == RTAG_UNITS && doc.present) + .map(|(_, data)| data.to_vec()), + cells: doc.cells.keys().cloned().collect(), + known_cells: doc.known_cells.clone(), + edges: doc.edges(), + } + } +} + +#[derive(Debug)] +pub struct Comparison { + pub old: Revision, + pub new: Revision, + /// Unchanged layer names available to the text report. + pub layer_legend: BTreeMap, + pub library_changes: Vec, + pub layer_names: Vec, + pub cells: Vec, + pub diagnostics: Vec, +} + +/// A comparison for text and hierarchy reports. +#[derive(Debug)] +pub struct TextComparison { + comparison: Comparison, +} + +impl TextComparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new()) + } + + fn collect(old: &Document<'_>, new: &Document<'_>, reused_cells: &BTreeSet<&str>) -> Self { + Self { + comparison: Comparison::collect(old, new, reused_cells, false), + } + } + + pub fn outcome(&self) -> Outcome { + self.comparison.outcome() + } + + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + self.comparison.hierarchy(mermaid, max_depth) + } +} + +impl Comparison { + pub fn between(old: &Document<'_>, new: &Document<'_>) -> Self { + Self::collect(old, new, &BTreeSet::new(), true) + } + + fn collect( + old: &Document<'_>, + new: &Document<'_>, + reused_cells: &BTreeSet<&str>, + include_whole_cell_objects: bool, + ) -> Self { + let mut result = Self { + old: Revision::from_document(old), + new: Revision::from_document(new), + layer_legend: old + .layer_names + .iter() + .filter(|(pair, name)| new.layer_names.get(pair) == Some(*name)) + .map(|(pair, name)| (*pair, name.clone())) + .collect(), + library_changes: Vec::new(), + layer_names: Vec::new(), + cells: Vec::new(), + diagnostics: Vec::new(), + }; + if old.present != new.present { + result.library_changes.push(LibraryChange::Presence { + old: old.present, + new: new.present, + }); + } + for (side, doc) in [(Side::Old, old), (Side::New, new)] { + result + .diagnostics + .extend(doc.diagnostics.iter().cloned().map(|mut diagnostic| { + diagnostic.side = Some(side); + diagnostic + })); + } + if old.scope != new.scope { + result.diagnostics.push(Diagnostic::new( + "scope_mismatch", + "comparison scopes differ", + None, + )); + } + let header_tags: BTreeSet<_> = old + .header + .iter() + .chain(&new.header) + .map(|(tag, _)| *tag) + .collect(); + for tag in header_tags { + let a: Vec<_> = old + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + let b: Vec<_> = new + .header + .iter() + .filter(|(t, _)| *t == tag) + .map(|(_, value)| *value) + .collect(); + if a != b { + result.library_changes.push(LibraryChange::Record { + tag, + old: a.into_iter().map(<[u8]>::to_vec).collect(), + new: b.into_iter().map(<[u8]>::to_vec).collect(), + }); + } + } + let layers: BTreeSet<_> = old + .layer_names + .keys() + .chain(new.layer_names.keys()) + .copied() + .collect(); + for layer in layers { + if old.layer_names.get(&layer) != new.layer_names.get(&layer) { + result.layer_names.push(LayerNameChange { + layer, + old: old.layer_names.get(&layer).cloned(), + new: new.layer_names.get(&layer).cloned(), + }); + } + } + let names: BTreeSet<_> = old.cells.keys().chain(new.cells.keys()).collect(); + for name in names { + if reused_cells.contains(name.as_str()) { + continue; + } + let a = old.cells.get(name); + let b = new.cells.get(name); + let mut change = CellChange { + name: name.clone(), + old: a.map(CellRevision::from), + new: b.map(CellRevision::from), + removed: Vec::new(), + added: Vec::new(), + replacements: Vec::new(), + }; + if !include_whole_cell_objects && (a.is_none() || b.is_none()) { + let details = if a.is_some() { + &mut change.removed + } else { + &mut change.added + }; + // The hierarchy uses the changed reference targets. + details.extend( + a.or(b) + .into_iter() + .flat_map(|cell| &cell.objects) + .filter(|object| !object.metadata && object.target.is_some()) + .map(ObjectChange::from), + ); + result.cells.push(change); + continue; + } + let a: Vec<_> = a + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let b: Vec<_> = b + .into_iter() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .collect(); + let (mut i, mut j) = (0, 0); + while i < a.len() || j < b.len() { + let order = match (a.get(i), b.get(j)) { + (Some(a), Some(b)) => a.key.cmp(&b.key), + (Some(_), None) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Greater, + }; + match order { + std::cmp::Ordering::Equal => { + i += 1; + j += 1; + } + std::cmp::Ordering::Less => { + change.removed.push(a[i].into()); + i += 1; + } + std::cmp::Ordering::Greater => { + change.added.push(b[j].into()); + j += 1; + } + } + } + if change.attributes_changed() || !change.removed.is_empty() || !change.added.is_empty() + { + change.replacements = changes::replacements(&change.removed, &change.added); + result.cells.push(change); + } + } + result + } + + pub fn outcome(&self) -> Outcome { + let different = !self.library_changes.is_empty() + || !self.layer_names.is_empty() + || !self.cells.is_empty(); + if !self.diagnostics.is_empty() { + Outcome::Incomplete { different } + } else if different { + Outcome::Different + } else { + Outcome::Equal + } + } + + /// Render the affected hierarchy. ASCII output shows relationships between cells. + pub fn hierarchy(&self, mermaid: bool, max_depth: usize) -> String { + hierarchy::render_hierarchy(&self.old, &self.new, Some(&self.cells), mermaid, max_depth) + } +} + +fn cell_marker(old: bool, new: bool) -> char { + match (old, new) { + (true, false) => '-', + (false, true) => '+', + _ => '*', + } +} + +fn cell_label(name: &str) -> Cow<'_, str> { + if name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'?' | b'.' | b'-')) + { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("{name:?}")) + } +} + +fn quoted(data: &[u8]) -> String { + let mut out = "\"".to_string(); + for byte in data { + for escaped in std::ascii::escape_default(*byte) { + out.push(escaped as char); + } + } + out.push('"'); + out +} + +fn real8_words(data: &[u8]) -> impl ExactSizeIterator + '_ { + data.as_chunks::<8>() + .0 + .iter() + .copied() + .map(u64::from_be_bytes) +} + +fn format_real8(data: &[u8], show_bytes: bool) -> String { + let mut values = real8_words(data).map(decode_real8); + let mut out = if values.len() == 1 { + format!("{:?}", values.next().unwrap()) + } else { + format!("{:?}", values.collect::>()) + }; + if show_bytes { + write!(out, " raw={}", hex(data)).unwrap(); + } + out +} + +fn is_real8(tag: u16, data: &[u8]) -> bool { + tag & 0xff == 5 && data.len().is_multiple_of(8) +} + +fn format_exact_value(tag: u16, data: &[u8]) -> String { + if is_real8(tag, data) { + let show_bytes = + real8_words(data).any(|bits| encode_real8(decode_real8(bits)).ok() != Some(bits)); + format_real8(data, show_bytes) + } else { + format_data(tag, data) + } +} + +fn format_field_change(tag: u16, before: Option<&[u8]>, after: Option<&[u8]>) -> String { + let show_bytes = before.zip(after).is_some_and(|(a, b)| { + is_real8(tag, a) + && is_real8(tag, b) + && real8_words(a) + .zip(real8_words(b)) + .any(|(a, b)| a != b && decode_real8(a).to_bits() == decode_real8(b).to_bits()) + }); + let format_value = |data: Option<&[u8]>| match data { + Some(data) if is_real8(tag, data) => format_real8(data, show_bytes), + Some(data) => format_data(tag, data), + None => "absent".to_string(), + }; + format!("{} -> {}", format_value(before), format_value(after)) +} + +fn format_data(tag: u16, data: &[u8]) -> String { + if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_BOXTYPE | RTAG_NODETYPE + ) && data.len() == 2 + { + return integer(data).to_string(); + } + match tag & 0xff { + 0 if data.is_empty() => String::new(), + 2 if data.len().is_multiple_of(2) => format!( + "{:?}", + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + 3 if data.len().is_multiple_of(4) => format!( + "{:?}", + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + 5 if data.len().is_multiple_of(8) => format!( + "{:?} raw={}", + real8_words(data).map(decode_real8).collect::>(), + hex(data) + ), + 6 => quoted(data), + _ => hex(data), + } +} + +fn object_name(kind: u16, rectangular: bool) -> &'static str { + match kind { + RTAG_BOUNDARY if rectangular => "rectangle", + RTAG_BOUNDARY => "polygon", + RTAG_BOX => "box", + RTAG_PATH => "path", + RTAG_SREF => "reference", + RTAG_AREF => "array reference", + RTAG_TEXT => "label", + RTAG_NODE => "node", + _ => kind_name(kind), + } +} + +fn hex(data: &[u8]) -> String { + let mut out = "0x".to_string(); + for byte in data { + write!(out, "{byte:02x}").unwrap(); + } + out +} + +fn kind_name(tag: u16) -> &'static str { + match tag { + RTAG_BOUNDARY => "BOUNDARY", + RTAG_PATH => "PATH", + RTAG_SREF => "SREF", + RTAG_AREF => "AREF", + RTAG_TEXT => "TEXT", + RTAG_NODE => "NODE", + RTAG_BOX => "BOX", + _ => "UNKNOWN", + } +} + +fn tag_name(tag: u16) -> String { + let name = match tag { + RTAG_HEADER => "HEADER", + RTAG_BGNLIB => "BGNLIB", + RTAG_LIBNAME => "LIBNAME", + RTAG_UNITS => "UNITS", + RTAG_ENDLIB => "ENDLIB", + RTAG_BGNSTR => "BGNSTR", + RTAG_STRNAME => "STRNAME", + RTAG_ENDSTR => "ENDSTR", + RTAG_ENDEL => "ENDEL", + RTAG_LAYER => "layer", + RTAG_DATATYPE => "datatype", + RTAG_TEXTTYPE => "texttype", + RTAG_XY => "xy", + RTAG_SNAME => "target", + RTAG_COLROW => "counts", + RTAG_STRANS => "strans", + RTAG_MAG => "mag", + RTAG_ANGLE => "angle", + RTAG_WIDTH => "width", + RTAG_PATHTYPE => "pathtype", + RTAG_STRING => "string", + RTAG_PROPATTR => "PROPATTR", + RTAG_PROPVALUE => "PROPVALUE", + RTAG_PRESENTATION => "presentation", + RTAG_BGNEXTN => "bgnextn", + RTAG_ENDEXTN => "endextn", + RTAG_BOXTYPE => "boxtype", + RTAG_NODETYPE => "nodetype", + RTAG_ELFLAGS => "elflags", + RTAG_PLEX => "plex", + RTAG_STRCLASS => "STRCLASS", + tag if is_element(tag) => kind_name(tag), + _ => return format!("record_{tag:04x}"), + }; + name.to_string() +} diff --git a/src/review/changes.rs b/src/review/changes.rs new file mode 100644 index 0000000..157a349 --- /dev/null +++ b/src/review/changes.rs @@ -0,0 +1,662 @@ +//! Unique replacement matches that explain removed and added objects. + +use super::{text::CellText, *}; + +#[derive(Debug)] +pub struct Replacement { + pub removed: usize, + pub added: usize, + /// Explanation matching is never object identity or geometric equivalence. + pub match_rule: &'static str, +} + +type Group = (u16, Option, Option, Option); + +fn group(object: &ObjectChange, bounds: bool) -> Group { + ( + object.kind, + object.layer, + object.target.clone(), + if bounds && object.target.is_none() { + object.bounds + } else { + None + }, + ) +} + +fn unique_groups<'a, K: Ord>( + objects: &'a [ObjectChange], + key: impl Fn(usize, &'a ObjectChange) -> Option, +) -> BTreeMap> { + let mut groups = BTreeMap::new(); + for (index, object) in objects.iter().enumerate() { + if let Some(key) = key(index, object) { + groups + .entry(key) + .and_modify(|value| *value = None) + .or_insert(Some(index)); + } + } + groups +} + +type ShapeKey<'a> = (u16, Vec<(u16, &'a [u8])>, &'a [(u16, Vec)]); + +fn shape_without_layer(object: &ObjectChange) -> Option> { + if object.layer.is_none() || !(is_polygon(object.kind) || object.kind == RTAG_PATH) { + return None; + } + let fields = object + .fields + .iter() + .filter(|field| !matches_layer_field(object.kind, field.tag)) + .map(|field| (field.tag, field.data.as_slice())) + .collect(); + Some((object.kind, fields, &object.properties)) +} + +fn layer_replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + // Most small edits touch a single numeric pair. + let Some(layer) = old.iter().chain(new).find_map(|object| object.layer) else { + return Vec::new(); + }; + if !old + .iter() + .chain(new) + .any(|object| object.layer.is_some_and(|pair| pair != layer)) + { + return Vec::new(); + } + let a = unique_groups(old, |_, object| shape_without_layer(object)); + let b = unique_groups(new, |_, object| shape_without_layer(object)); + a.into_iter() + .filter_map(|(key, removed)| { + let removed = removed?; + let added = b.get(&key).copied().flatten()?; + (old[removed].layer != new[added].layer).then_some(Replacement { + removed, + added, + match_rule: "unique_layer_change", + }) + }) + .collect() +} + +/// Pair only unique candidates in the remaining multisets. Ambiguous groups +/// remain additions/removals; no identity or geometric equivalence is inferred. +pub(super) fn replacements(old: &[ObjectChange], new: &[ObjectChange]) -> Vec { + if old.is_empty() || new.is_empty() { + return Vec::new(); + } + let mut result = layer_replacements(old, new); + let (mut used_old, mut used_new): (BTreeSet<_>, BTreeSet<_>) = + result.iter().map(|pair| (pair.removed, pair.added)).unzip(); + for bounds in [true, false] { + let a = unique_groups(old, |i, object| { + (!used_old.contains(&i)).then(|| group(object, bounds)) + }); + let b = unique_groups(new, |i, object| { + (!used_new.contains(&i)).then(|| group(object, bounds)) + }); + for (key, index) in a { + if let (Some(i), Some(&Some(j))) = (index, b.get(&key)) { + result.push(Replacement { + removed: i, + added: j, + match_rule: if old[i].target.is_some() { + "unique_kind_target" + } else if bounds { + "unique_kind_layer_bounds" + } else { + "unique_kind_layer" + }, + }); + used_old.insert(i); + used_new.insert(j); + } + } + } + result.sort_by_key(|r| (r.removed, r.added)); + result +} + +pub(super) fn field_changes<'a>( + old: &'a ObjectChange, + new: &'a ObjectChange, +) -> impl Iterator, Option<&'a [u8]>)> { + let mut fields = BTreeMap::, Option<&[u8]>)>::new(); + for field in &old.fields { + fields.entry(field.tag).or_default().0 = Some(&field.data); + } + for field in &new.fields { + fields.entry(field.tag).or_default().1 = Some(&field.data); + } + fields + .into_iter() + .filter_map(|(tag, (before, after))| (before != after).then_some((tag, before, after))) +} + +fn object_heading(object: &ObjectChange, kind: &str, with_layer: bool) -> String { + match object.layer.filter(|_| with_layer) { + Some((layer, datatype)) => format!("{layer}/{datatype}: {kind}"), + None => kind.to_string(), + } +} + +fn object_xy(object: &ObjectChange) -> &[u8] { + object + .fields + .iter() + .find(|field| field.tag == RTAG_XY) + .map_or(&[], |field| field.data.as_slice()) +} + +fn write_cell_heading(out: &mut CellText<'_>, layer: Option, name: &str) { + if let Some(layer) = layer { + out.write_layer(layer, None); + out.push_str(": "); + } + out.push_str(name); +} + +pub(super) fn write_unmatched(out: &mut CellText<'_>, object: &ObjectChange, details: bool) { + if !details + && object.kind == RTAG_PATH + && let Some([xmin, ymin, xmax, ymax]) = object.bounds + { + let vertices = XY::points(object_xy(object)).len(); + if vertices > POINTS_PER_LINE { + write_cell_heading(out, object.layer, "path"); + let width = object + .width + .map_or_else(|| "absent".to_string(), |width| format!("{width} dbu")); + write!( + out, + " · width {width} · {vertices} vertices · bounds ({xmin}, {ymin}) to ({xmax}, {ymax}) dbu" + ) + .unwrap(); + return; + } + } + object + .format_with_heading(out, |out, name| { + write_cell_heading(out, object.layer, name); + Ok(()) + }) + .unwrap(); +} + +fn vertex_count(xy: &[u8]) -> usize { + XY::points(xy).len().saturating_sub(1) +} + +fn rectangle_size(xy: &[u8]) -> Option<[i64; 2]> { + let [xmin, ymin, xmax, ymax] = XY::closed_rectangle(xy)?; + Some([ + i64::from(xmax) - i64::from(xmin), + i64::from(ymax) - i64::from(ymin), + ]) +} + +fn translation(before: &[u8], after: &[u8]) -> Option<[i64; 2]> { + let mut old = XY::points(before); + let mut new = XY::points(after); + if old.len() != new.len() { + return None; + } + let (Some(a), Some(b)) = (old.next(), new.next()) else { + return None; + }; + let displacement = |[x0, y0]: [i32; 2], [x1, y1]: [i32; 2]| { + [i64::from(x1) - i64::from(x0), i64::from(y1) - i64::from(y0)] + }; + let shift = displacement(a, b); + (shift != [0, 0] && old.zip(new).all(|(a, b)| displacement(a, b) == shift)).then_some(shift) +} + +#[derive(Default)] +struct ShapeCounts { + rectangles: usize, + polygons: usize, + boxes: usize, +} + +impl ShapeCounts { + fn total(&self) -> usize { + self.rectangles + self.polygons + self.boxes + } + + fn add(&mut self, object: &ObjectChange, replacement: Option<&ObjectChange>) { + let is_rectangle = + |object: &ObjectChange| XY::closed_rectangle(object_xy(object)).is_some(); + match object.kind { + RTAG_BOX => self.boxes += 1, + _ if is_rectangle(object) && replacement.is_none_or(is_rectangle) => { + self.rectangles += 1 + } + _ => self.polygons += 1, + } + } +} + +#[derive(Default)] +struct LayerPolygonChanges { + changed: ShapeCounts, + moved: ShapeCounts, + removed: ShapeCounts, + added: ShapeCounts, + displacement: Option<[i64; 2]>, +} + +#[derive(Default)] +pub(super) struct PolygonChanges(BTreeMap, LayerPolygonChanges>); + +impl PolygonChanges { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange) { + let counts = self.0.entry(old.layer).or_default(); + let displacement = if old.properties == new.properties { + translation(object_xy(old), object_xy(new)) + .filter(|_| field_changes(old, new).all(|(tag, _, _)| tag == RTAG_XY)) + } else { + None + }; + if let Some(shift) = displacement { + counts.displacement = Some(shift); + counts.moved.add(old, Some(new)); + } else { + counts.changed.add(old, Some(new)); + } + } + + pub fn unmatched(&mut self, object: &ObjectChange, removed: bool) { + let counts = self.0.entry(object.layer).or_default(); + let shapes = if removed { + &mut counts.removed + } else { + &mut counts.added + }; + shapes.add(object, None); + } + + pub fn write(&self, out: &mut CellText<'_>) { + for (layer, counts) in &self.0 { + out.push_str(" "); + out.style(Some(TextStyle::Layer)); + if let Some(layer) = layer { + out.write_layer(*layer, None); + } else { + out.push_str("unknown layer"); + } + out.style(None); + out.push_str(": "); + let groups = [ + ("changed", TextStyle::Changed, &counts.changed), + ("moved", TextStyle::Changed, &counts.moved), + ("removed", TextStyle::Removed, &counts.removed), + ("added", TextStyle::Added, &counts.added), + ]; + if let Some([dx, dy]) = counts.displacement + && groups + .iter() + .map(|(_, _, shapes)| shapes.total()) + .sum::() + == 1 + { + let shape = if counts.moved.rectangles == 1 { + "rectangle" + } else if counts.moved.boxes == 1 { + "box" + } else { + "polygon" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "{shape} moved by ({dx}, {dy}) dbu").unwrap(); + continue; + } + let mut separator = ""; + for (action, style, shapes) in groups { + let terms = [ + (shapes.rectangles, "rectangle", "rectangles"), + (shapes.polygons, "polygon", "polygons"), + (shapes.boxes, "box", "boxes"), + ] + .into_iter() + .filter(|(count, _, _)| *count != 0) + .map(|(count, singular, plural)| { + let shape = if count == 1 { singular } else { plural }; + format!("{count} {shape}") + }) + .collect::>(); + if !terms.is_empty() { + out.push_str(separator); + out.style(Some(style)); + write!(out, "{} {action}", terms.join(", ")).unwrap(); + out.style(None); + separator = " · "; + } + } + out.push('\n'); + } + } +} + +#[derive(Default)] +pub(super) struct ReferenceCounts { + singles: usize, + arrays: usize, + placements: u128, +} + +impl ReferenceCounts { + pub fn add(&mut self, object: &ObjectChange) { + if object.kind == RTAG_AREF { + self.arrays += 1; + } else { + self.singles += 1; + } + self.placements += reference_placements(object); + } + + pub fn write( + &self, + out: &mut CellText<'_>, + sign: char, + target: &str, + first: &ObjectChange, + ) -> usize { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if self.len() == 1 { + write!(out, "{first}").unwrap(); + } else { + self.write_counts(out, ""); + write!(out, " to {}", cell_label(target)).unwrap(); + self.write_placements(out); + } + let placements = out.reserve_placements(); + out.push('\n'); + placements + } + + fn len(&self) -> usize { + self.singles + self.arrays + } + + pub fn write_change(&self, out: &mut CellText<'_>, target: &str, added: &Self) -> usize { + write!(out, " {} · ", cell_label(target)).unwrap(); + out.style(Some(TextStyle::Removed)); + self.write_counts(out, "-"); + self.write_placements(out); + out.style(None); + out.push_str(", "); + out.style(Some(TextStyle::Added)); + added.write_counts(out, "+"); + added.write_placements(out); + out.style(None); + let placements = out.reserve_placements(); + out.push('\n'); + placements + } + + fn write_counts(&self, out: &mut CellText<'_>, sign: &str) { + if self.singles > 0 { + let suffix = if self.singles == 1 { "" } else { "s" }; + write!(out, "{sign}{} ref{suffix}", self.singles).unwrap(); + } + if self.arrays > 0 { + if self.singles > 0 { + out.push_str(", "); + } + let suffix = if self.arrays == 1 { "" } else { "s" }; + write!(out, "{sign}{} array ref{suffix}", self.arrays).unwrap(); + } + } + + fn write_placements(&self, out: &mut CellText<'_>) { + if self.arrays > 0 { + let suffix = if self.placements == 1 { "" } else { "s" }; + write!(out, " ({} placement{suffix})", self.placements).unwrap(); + } + } +} + +fn reference_placements(object: &ObjectChange) -> u128 { + match object.counts { + Some([columns, rows]) => u128::from(columns) * u128::from(rows), + None => 0, + } +} + +#[derive(Default)] +pub(super) struct ShownReferences { + shown: usize, + total: usize, + removed: u128, + added: u128, + placement_slot: usize, +} + +impl ShownReferences { + pub fn replacement(&mut self, old: &ObjectChange, new: &ObjectChange, placement_slot: usize) { + self.shown += 2; + self.removed += reference_placements(old); + self.added += reference_placements(new); + self.placement_slot = placement_slot; + } + + pub fn unmatched( + &mut self, + removed: &ReferenceCounts, + added: &ReferenceCounts, + placement_slot: usize, + ) { + self.shown += removed.len() + added.len(); + self.removed += removed.placements; + self.added += added.placements; + self.placement_slot = placement_slot; + } + + pub fn individual( + &mut self, + object: &ObjectChange, + removed: bool, + count: usize, + placement_slot: usize, + ) { + let placements = if removed { + &mut self.removed + } else { + &mut self.added + }; + *placements += reference_placements(object) * count as u128; + self.shown += count; + self.placement_slot = placement_slot; + } + + pub fn count_change(&mut self) { + self.total += 1; + } + + pub fn complete(&self) -> bool { + self.shown == self.total + } + + pub fn write_placements(&self, out: &mut CellText<'_>, old: u64, new: u64) { + if (u128::from(old), u128::from(new)) != (self.removed, self.added) { + out.set_placements(self.placement_slot, old, new); + } + } +} + +pub(super) fn write_layer_transition( + out: &mut CellText<'_>, + old: &ObjectChange, + new: &ObjectChange, +) { + out.write_layer(old.layer.unwrap(), Some(Side::Old)); + out.push_str(" → "); + out.write_layer(new.layer.unwrap(), Some(Side::New)); + write!(out, " · {}", transition_shape(old)).unwrap(); +} + +fn transition_shape(old: &ObjectChange) -> &'static str { + let rectangular = is_polygon(old.kind) && rectangle_size(object_xy(old)).is_some(); + object_name(old.kind, rectangular) +} + +fn polygon_replacement(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let before = object_xy(old); + let after = object_xy(new); + let old_size = rectangle_size(before); + let new_size = rectangle_size(after); + let name = object_name(old.kind, old_size.is_some() && new_size.is_some()); + let mut out = object_heading(old, name, with_layer); + if before == after { + out.push_str(" modified"); + } else { + match (old_size, new_size) { + (Some(a), Some(b)) if a == b => out.push_str(" moved"), + (Some([w0, h0]), Some([w1, h1])) => { + write!(out, " dimensions changed ({w0} x {h0} -> {w1} x {h1} dbu)").unwrap(); + } + _ if translation(before, after).is_some() => out.push_str(" moved"), + _ => { + out.push_str(" vertices changed"); + let (a, b) = (vertex_count(before), vertex_count(after)); + if a != b { + write!(out, " ({a} -> {b} vertices)").unwrap(); + } + } + } + } + out +} + +fn replacement_heading(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let rectangular = is_polygon(old.kind) + && rectangle_size(object_xy(old)).is_some() + && rectangle_size(object_xy(new)).is_some(); + let mut heading = object_heading(old, object_name(old.kind, rectangular), with_layer); + if let Some(target) = &old.target { + write!(heading, " {target:?}").unwrap(); + } + heading +} + +pub(super) fn write_replacement_objects( + out: &mut CellText<'_>, + old: &ObjectChange, + new: &ObjectChange, +) { + for (sign, object) in [('-', old), ('+', new)] { + let description = object.to_string(); + let mut lines = description.split('\n'); + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + writeln!(out, "{sign} {}", lines.next().unwrap()).unwrap(); + for line in lines { + writeln!(out, " {line}").unwrap(); + } + } +} + +pub(super) fn replacement_summary(old: &ObjectChange, new: &ObjectChange) -> String { + if let (Some((from, from_type)), Some((to, to_type))) = (old.layer, new.layer) + && old.layer != new.layer + { + return format!( + "{from}/{from_type} → {to}/{to_type} · {}", + transition_shape(old) + ); + } + replacement_fields(old, new, true) +} + +pub(super) fn replacement_description( + old: &ObjectChange, + new: &ObjectChange, + details: bool, +) -> String { + if details { + replacement_heading(old, new, false) + } else { + replacement_fields(old, new, false) + } +} + +fn replacement_fields(old: &ObjectChange, new: &ObjectChange, with_layer: bool) -> String { + let polygon = is_polygon(old.kind); + let mut out = if polygon { + polygon_replacement(old, new, with_layer) + } else { + replacement_heading(old, new, with_layer) + }; + + let mut separator = if polygon { "\n " } else { " " }; + for (tag, before, after) in field_changes(old, new) { + if polygon && tag == RTAG_XY { + continue; + } + let change = match (tag, before, after) { + (RTAG_XY, Some(a), Some(b)) => { + if let Some([dx, dy]) = translation(a, b) { + format!("moved by ({dx}, {dy}) dbu") + } else if matches!(old.kind, RTAG_PATH | RTAG_NODE) { + let (a, b) = (XY::points(a).len(), XY::points(b).len()); + if a == b { + "vertices changed".to_string() + } else { + format!("vertices changed ({a} -> {b} vertices)") + } + } else { + let points = |data: &[u8]| XY::points(data).collect::>(); + format!("coordinates {:?} -> {:?} dbu", points(a), points(b)) + } + } + (RTAG_COLROW, _, _) if old.kind == RTAG_AREF => { + let counts = |dimensions: Option<[u16; 2]>| { + dimensions.map_or_else(|| "absent".to_string(), |[a, b]| format!("{a} x {b}")) + }; + format!( + "columns/rows {} -> {}", + counts(old.counts), + counts(new.counts) + ) + } + (RTAG_WIDTH, _, _) => { + let width = + |n: Option| n.map_or_else(|| "absent".to_string(), |n| n.to_string()); + format!("width {} -> {} dbu", width(old.width), width(new.width)) + } + _ => { + let name = if tag == RTAG_STRING { + "text".to_string() + } else { + tag_name(tag) + }; + format!("{name} {}", format_field_change(tag, before, after)) + } + }; + write!(out, "{separator}{change}").unwrap(); + separator = "\n "; + } + if old.properties != new.properties { + let properties = |items: &[(u16, Vec)]| { + items + .iter() + .map(|(attribute, value)| format!("{attribute}={}", quoted(value))) + .collect::>() + .join(", ") + }; + write!( + out, + "{separator}properties [{}] -> [{}]", + properties(&old.properties), + properties(&new.properties) + ) + .unwrap(); + } + out +} diff --git a/src/review/hierarchy.rs b/src/review/hierarchy.rs new file mode 100644 index 0000000..4d5342e --- /dev/null +++ b/src/review/hierarchy.rs @@ -0,0 +1,495 @@ +//! Hierarchy analysis, traversal, and presentation. + +use super::*; + +/// Graph facts shared by text, Mermaid, and JSON renderers. +pub(super) struct HierarchyAnalysis<'a> { + pub(super) old_edges: &'a BTreeMap<(String, String), u64>, + pub(super) new_edges: &'a BTreeMap<(String, String), u64>, + pub(super) edges: BTreeSet<(String, String)>, + pub(super) direct: BTreeSet, + pub(super) affected: BTreeSet, + pub(super) changed_refs: BTreeSet<(String, String)>, + old: &'a Revision, + new: &'a Revision, + comparison: bool, +} + +impl<'a> HierarchyAnalysis<'a> { + pub(super) fn between( + old: &'a Revision, + new: &'a Revision, + changes: Option<&[CellChange]>, + ) -> Self { + let old_edges = &old.edges; + let new_edges = &new.edges; + let edges: BTreeSet<_> = old_edges.keys().chain(new_edges.keys()).cloned().collect(); + let direct: BTreeSet<_> = changes + .into_iter() + .flatten() + .map(|cell| cell.name.clone()) + .collect(); + let changed_refs: BTreeSet<_> = changes + .into_iter() + .flatten() + .flat_map(|c| { + c.removed + .iter() + .chain(&c.added) + .filter_map(|o| o.target.as_ref().map(|t| (c.name.clone(), t.clone()))) + }) + .collect(); + let mut parents = BTreeMap::<&str, Vec<&str>>::new(); + for (parent, child) in &edges { + parents.entry(child).or_default().push(parent); + } + let mut affected = direct.clone(); + let mut pending: Vec<_> = direct.iter().cloned().collect(); + while let Some(child) = pending.pop() { + for parent in parents.get(child.as_str()).into_iter().flatten() { + if affected.insert((*parent).to_string()) { + pending.push((*parent).to_string()); + } + } + } + Self { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + old, + new, + comparison: changes.is_some(), + } + } + + fn visible_rows( + &self, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, + ) -> Option { + let mut visible: BTreeSet<_> = if self.comparison { + self.affected + .iter() + .cloned() + .chain(self.changed_refs.iter().map(|(_, t)| t.clone())) + .collect() + } else { + self.old + .cells + .iter() + .chain(self.new.cells.iter()) + .cloned() + .chain(self.edges.iter().map(|(_, t)| t.clone())) + .collect() + }; + let mut children = BTreeMap::>::new(); + let mut targets = BTreeSet::new(); + let mut shared = BTreeSet::new(); + let fold_paths = self.comparison && !mermaid; + for (parent, target) in &self.edges { + if visible.contains(parent) && visible.contains(target) { + children + .entry(parent.clone()) + .or_default() + .push(target.clone()); + if !targets.insert(target.clone()) && fold_paths { + shared.insert(target.clone()); + } + } + } + if let Some(covered) = covered_edges { + if !covered.is_empty() { + // Keep changed or shared descendants, branches, and target status notes. + let redundant: BTreeSet<_> = covered + .iter() + .map(|(_, target)| target) + .filter(|&target| { + targets.contains(target) + && !shared.contains(target) + && !children.contains_key(target) + && !self.affected.contains(target) + && self.old.cells.contains(target) + && self.new.cells.contains(target) + }) + .collect(); + children.retain(|_, children| { + children.retain(|target| !redundant.contains(target)); + !children.is_empty() + }); + targets.retain(|target| !redundant.contains(target)); + } + visible.retain(|name| children.contains_key(name) || targets.contains(name)); + } + if fold_paths && children.is_empty() { + return None; + } + let rows = traverse(&visible, &children, &targets, max_depth); + Some(VisibleHierarchy { + rows, + children, + shared, + }) + } + + fn single_placement(&self, parent: &str, target: &str) -> bool { + let key = (parent.to_string(), target.to_string()); + self.old_edges.get(&key) == Some(&1) + && self.new_edges.get(&key) == Some(&1) + && !self.changed_refs.contains(&key) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RowNote { + ChildrenOmitted, + SubtreeElided, +} + +struct HierarchyRow { + name: String, + depth: usize, + parent: Option, + note: Option, +} + +fn traverse( + visible: &BTreeSet, + children: &BTreeMap>, + targets: &BTreeSet, + max_depth: usize, +) -> Vec { + let roots: Vec<_> = visible.difference(targets).cloned().collect(); + // Distinguish rootless cycles from descendants hidden by the depth limit. + let mut reachable = BTreeSet::new(); + let mut pending = roots.clone(); + while let Some(name) = pending.pop() { + if reachable.insert(name.clone()) { + pending.extend(children.get(&name).into_iter().flatten().cloned()); + } + } + let mut included = BTreeSet::new(); + let mut rows = Vec::new(); + let mut stack: Vec<_> = roots + .into_iter() + .rev() + .map(|name| (name, 0, None::)) + .collect(); + while let Some((name, depth, parent)) = stack.pop() { + let seen = !included.insert(name.clone()); + let descendants = children.get(&name).map(Vec::as_slice).unwrap_or(&[]); + let note = if descendants.is_empty() { + None + } else if depth >= max_depth { + Some(RowNote::ChildrenOmitted) + } else if seen { + Some(RowNote::SubtreeElided) + } else { + None + }; + if !seen && depth < max_depth { + for child in descendants.iter().rev() { + stack.push((child.clone(), depth + 1, Some(name.clone()))); + } + } + rows.push(HierarchyRow { + name, + depth, + parent, + note, + }); + } + for name in visible.difference(&reachable) { + rows.push(HierarchyRow { + name: name.clone(), + depth: 0, + parent: None, + note: Some(RowNote::ChildrenOmitted), + }); + } + rows +} + +struct VisibleHierarchy { + rows: Vec, + children: BTreeMap>, + shared: BTreeSet, +} + +struct AsciiPath<'a> { + indent: usize, + rows: &'a [HierarchyRow], +} + +impl VisibleHierarchy { + fn unbranched(&self, name: &str) -> bool { + !self.shared.contains(name) + && self + .children + .get(name) + .is_none_or(|descendants| descendants.len() == 1) + } + + fn can_join( + &self, + graph: &HierarchyAnalysis<'_>, + parent: &HierarchyRow, + child: &HierarchyRow, + ) -> bool { + child.parent.as_deref() == Some(&parent.name) + && parent.note.is_none() + && !graph.direct.contains(&parent.name) + && graph.affected.contains(&parent.name) + && self.unbranched(&child.name) + && (graph.old.cells.contains(&child.name) || graph.new.cells.contains(&child.name)) + && graph.single_placement(&parent.name, &child.name) + } + + fn ascii_paths<'a>( + &'a self, + graph: &'a HierarchyAnalysis<'_>, + ) -> impl Iterator> { + let mut index = 0; + let mut ancestor_depths = Vec::new(); + std::iter::from_fn(move || { + let first = self.rows.get(index)?; + let mut end = index; + if graph.comparison + && self.unbranched(&first.name) + && first + .parent + .as_ref() + .is_none_or(|parent| graph.single_placement(parent, &first.name)) + { + while self + .rows + .get(end + 1) + .is_some_and(|next| self.can_join(graph, &self.rows[end], next)) + { + end += 1; + } + // Fold paths with at least two edges. + if end - index < 2 { + end = index; + } + } + while ancestor_depths + .last() + .is_some_and(|&depth| depth >= first.depth) + { + ancestor_depths.pop(); + } + let rows = &self.rows[index..=end]; + let indent = ancestor_depths.len(); + ancestor_depths.push(self.rows[end].depth); + index = end + 1; + Some(AsciiPath { indent, rows }) + }) + } +} + +/// The graph joins both revisions, retaining removed branches and old callers. +pub(super) fn render_hierarchy( + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, +) -> String { + let mut out = TextReport::default(); + write_hierarchy(&mut out, old, new, changes, mermaid, max_depth, None); + out.into_string() +} + +pub(super) fn write_hierarchy( + out: &mut TextReport, + old: &Revision, + new: &Revision, + changes: Option<&[CellChange]>, + mermaid: bool, + max_depth: usize, + covered_edges: Option<&BTreeSet<(String, String)>>, +) { + if !mermaid && changes.is_some_and(<[CellChange]>::is_empty) { + return; + } + let graph = HierarchyAnalysis::between(old, new, changes); + let Some(view) = graph.visible_rows(mermaid, max_depth, covered_edges) else { + return; + }; + if mermaid { + graph.write_mermaid(out, &view.rows); + } else { + graph.write_ascii(out, view.ascii_paths(&graph)); + } +} + +impl HierarchyAnalysis<'_> { + fn status(&self, name: &str, mermaid: bool) -> &'static str { + if !self.old.cells.contains(name) && !self.new.cells.contains(name) { + if self.old.known_cells.contains(name) || self.new.known_cells.contains(name) { + " [outside scope]" + } else { + " [unresolved]" + } + } else if self.direct.contains(name) { + if mermaid { " [cell changed]" } else { "" } + } else if self.affected.contains(name) { + " [affected]" + } else if mermaid && self.comparison { + " [cell unchanged]" + } else { + "" + } + } + + fn edge_label(&self, parent: &str, target: &str, mermaid: bool) -> String { + let key = (parent.to_string(), target.to_string()); + let a = self.old_edges.get(&key).copied().unwrap_or(0); + let b = self.new_edges.get(&key).copied().unwrap_or(0); + let count = if a == b { + format!("x{b}") + } else { + format!("x{a} -> x{b}") + }; + format!( + "{count}{}", + if (mermaid || a == b) && self.changed_refs.contains(&key) { + " [reference changed]" + } else { + "" + } + ) + } + + fn write_mermaid(&self, out: &mut TextReport, rows: &[HierarchyRow]) { + let ids: BTreeMap<_, _> = rows + .iter() + .map(|row| &row.name) + .collect::>() + .into_iter() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + out.push_str("flowchart TD\n"); + if !self.comparison && self.new.scope != Scope::default() { + writeln!(out, " %% {}", self.new.scope.description()).unwrap(); + } + for (name, id) in &ids { + let label = format!("{name}{}", self.status(name, true)); + let label = label + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") + .replace(['\r', '\n'], " "); + writeln!(out, " n{id}[\"{label}\"]").unwrap(); + } + for (parent, target) in &self.edges { + if let (Some(a), Some(b)) = (ids.get(parent), ids.get(target)) { + writeln!( + out, + " n{a} -->|\"{}\"| n{b}", + self.edge_label(parent, target, true) + ) + .unwrap(); + } + } + if rows.is_empty() { + out.push_str(" empty[\"No changed hierarchy\"]\n"); + } + } + + fn write_ascii<'a>(&self, out: &mut TextReport, paths: impl Iterator>) { + out.write_styled(TextStyle::Heading, format_args!("Hierarchy")); + out.push('\n'); + if !self.comparison { + self.new.scope.write_restrictions(out); + } + let mut paths = paths.peekable(); + if paths.peek().is_none() { + out.push_str("No changed hierarchy\n"); + } + for path in paths { + out.push_str(&" ".repeat(path.indent)); + let mut change = None; + for (i, row) in path.rows.iter().enumerate() { + let name = &row.name; + if i != 0 { + out.push_str(" → "); + } + if self.direct.contains(name) { + let marker = + cell_marker(self.old.cells.contains(name), self.new.cells.contains(name)); + change = Some((out.len(), TextStyle::change(marker))); + write!(out, "{marker} ").unwrap(); + } + if self.comparison { + out.push_str(&cell_label(name)); + } else { + write!(out, "{name:?}").unwrap(); + } + if i == 0 + && let Some(parent) = &row.parent + { + write!(out, " {}", self.edge_label(parent, name, false)).unwrap(); + } + } + let last = path.rows.last().unwrap(); + let note = match last.note { + Some(RowNote::ChildrenOmitted) => " [children omitted]", + Some(RowNote::SubtreeElided) => " [subtree elided]", + None => "", + }; + write!(out, "{}{note}", self.status(&last.name, false)).unwrap(); + if let Some((start, style)) = change { + out.mark(start, style); + } + out.push('\n'); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn traversal_distinguishes_depth_limits_from_rootless_components() { + let children = BTreeMap::from([ + ("root".into(), vec!["branch".into()]), + ("branch".into(), vec!["leaf".into()]), + ("cycle_a".into(), vec!["cycle_b".into()]), + ("cycle_b".into(), vec!["cycle_a".into()]), + ]); + let visible = ["root", "branch", "leaf", "cycle_a", "cycle_b"] + .map(String::from) + .into_iter() + .collect(); + let targets = children.values().flatten().cloned().collect(); + let rows = traverse(&visible, &children, &targets, 1); + let rows: Vec<_> = rows + .iter() + .map(|row| { + ( + row.name.as_str(), + row.depth, + row.parent.as_deref(), + row.note, + ) + }) + .collect(); + assert_eq!( + rows, + [ + ("root", 0, None, None), + ("branch", 1, Some("root"), Some(RowNote::ChildrenOmitted)), + ("cycle_a", 0, None, Some(RowNote::ChildrenOmitted)), + ("cycle_b", 0, None, Some(RowNote::ChildrenOmitted)), + ] + ); + } +} diff --git a/src/review/names.rs b/src/review/names.rs new file mode 100644 index 0000000..c551526 --- /dev/null +++ b/src/review/names.rs @@ -0,0 +1,205 @@ +//! Embedded layer names: parsing, recognition, and record-preserving updates. + +use super::*; + +const LAYER_PREFIX: &[u8] = b"PHEON_LAYER_NAMES_"; +pub const LAYER_SENTINEL: &str = "PHEON_LAYER_NAMES_V1"; + +impl Document<'_> { + pub(super) fn read_metadata(&mut self) -> Result<(), String> { + for (cell_name, cell) in &mut self.cells { + for object in cell.objects.iter().filter(|o| o.kind == RTAG_TEXT) { + let text = object.field(RTAG_STRING).unwrap(); + if text.starts_with(LAYER_PREFIX) && text != LAYER_SENTINEL.as_bytes() { + // Layer names apply across the library. An unknown version + // may describe layers used outside this carrier or cell, so + // geometry scope selection cannot establish its relevance. + let mut diagnostic = Diagnostic::at( + "unsupported_metadata_version", + object.offset, + "unsupported layer-name metadata version".to_string(), + ); + diagnostic.cell = Some(cell_name.clone()); + diagnostic.layer = object.layer; + diagnostic.message = format!("cell {cell_name:?}: {}", diagnostic.message); + self.global_issues.push(diagnostic); + } + } + let carriers: BTreeSet<_> = cell + .objects + .iter() + .filter(|o| { + o.kind == RTAG_TEXT && o.field(RTAG_STRING) == Some(LAYER_SENTINEL.as_bytes()) + }) + .filter_map(|o| o.layer) + .collect(); + for object in &mut cell.objects { + if object.kind != RTAG_TEXT || !object.layer.is_some_and(|l| carriers.contains(&l)) + { + continue; + } + let text = decoded_string(object.field(RTAG_STRING).unwrap(), object.offset)?; + if text != LAYER_SENTINEL { + let ((layer, dtype), name) = + parse_layer_name(&text).map_err(|message| error(object.offset, message))?; + if self + .layer_names + .get(&(layer, dtype)) + .is_some_and(|old| old != &name) + { + return Err(error( + object.offset, + format!("conflicting names for {layer}/{dtype}"), + )); + } + self.layer_names.insert((layer, dtype), name.to_string()); + } + object.metadata = true; + } + } + Ok(()) + } +} + +pub fn parse_layer(value: &str) -> Result { + let (a, b) = value + .split_once('/') + .ok_or("expected layer/type, for example 900/7")?; + Ok((decimal(a)?, decimal(b)?)) +} + +/// Find the unique numeric pair for an exact name across the supplied revisions. +pub fn resolve_layer_name<'a>( + name: &str, + revisions: impl IntoIterator>, +) -> Result { + let matches: BTreeSet<_> = revisions + .into_iter() + .flatten() + .filter_map(|(&pair, label)| (label == name).then_some(pair)) + .collect(); + match matches.len() { + 0 => Err(format!("unknown layer name {name:?}")), + 1 => Ok(*matches.first().unwrap()), + _ => { + let choices = matches + .into_iter() + .map(|(layer, datatype)| format!("{layer}/{datatype}")) + .collect::>() + .join(", "); + Err(format!("ambiguous layer name {name:?} (choose {choices})")) + } + } +} + +fn decimal(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err("layer numbers must be unsigned decimal integers".to_string()); + } + value + .parse() + .map_err(|_| "layer number exceeds 65535".to_string()) +} + +pub fn parse_layer_name(value: &str) -> Result<(Layer, String), String> { + let mut parts = value.splitn(3, ':'); + let a = parts.next().ok_or("expected layer:datatype:name")?; + let b = parts.next().ok_or("expected layer:datatype:name")?; + let name = parts + .next() + .filter(|s| !s.is_empty() && !s.contains('\0')) + .ok_or("layer name must be nonempty and contain no NUL")?; + Ok(((decimal(a)?, decimal(b)?), name.to_string())) +} + +fn record(output: &mut Vec, tag: u16, payload: &[u8]) -> Result<(), String> { + let padding = usize::from(tag & 0xff == 6 && !payload.len().is_multiple_of(2)); + let size = payload + .len() + .checked_add(padding + 4) + .and_then(|n| u16::try_from(n).ok()) + .ok_or("metadata label exceeds GDS record size")?; + output.extend_from_slice(&size.to_be_bytes()); + output.extend_from_slice(&tag.to_be_bytes()); + output.extend_from_slice(payload); + if padding != 0 { + output.push(0); + } + Ok(()) +} + +fn label(output: &mut Vec, carrier: Layer, text: &str) -> Result<(), String> { + record(output, RTAG_TEXT, &[])?; + record(output, RTAG_LAYER, &carrier.0.to_be_bytes())?; + record(output, RTAG_TEXTTYPE, &carrier.1.to_be_bytes())?; + record(output, RTAG_XY, &[0; 8])?; + record(output, RTAG_STRING, text.as_bytes())?; + record(output, RTAG_ENDEL, &[]) +} + +/// Consolidate recognized metadata into an explicitly chosen existing cell and +/// carrier. Other records, including timestamps and unknown records, are copied +/// byte-for-byte. Updates override existing names; the input is never modified. +pub fn write_layer_names( + input: &[u8], + cell_name: &str, + carrier: Layer, + updates: &BTreeMap, +) -> Result, String> { + let doc = Document::parse(input)?; + if doc + .diagnostics + .iter() + .any(|i| i.code == "unsupported_metadata_version") + { + return Err("cannot rewrite an unsupported layer-name metadata version".to_string()); + } + let cell = doc + .cells + .get(cell_name) + .ok_or_else(|| format!("cell {cell_name:?} does not exist"))?; + if cell + .objects + .iter() + .any(|o| o.layer == Some(carrier) && !o.metadata) + { + return Err( + "carrier pair already contains ordinary design objects in the selected cell" + .to_string(), + ); + } + let mut names = doc.layer_names.clone(); + for (&pair, name) in updates { + if name.is_empty() || name.contains('\0') { + return Err("layer names must be nonempty and contain no NUL".to_string()); + } + names.insert(pair, name.clone()); + } + let mut legend = Vec::new(); + label(&mut legend, carrier, LAYER_SENTINEL)?; + for ((layer, dtype), name) in names { + label(&mut legend, carrier, &format!("{layer}:{dtype}:{name}"))?; + } + let mut edits: Vec<_> = doc + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| o.metadata) + .map(|o| (o.offset, o.end_offset, None)) + .collect(); + edits.push((cell.end_offset, cell.end_offset, Some(legend.as_slice()))); + edits.sort_by_key(|(start, end, _)| (*start, *end)); + let mut result = Vec::with_capacity(input.len() + legend.len()); + let mut cursor = 0; + for (start, end, replacement) in edits { + result.extend_from_slice(&input[cursor..start]); + if let Some(bytes) = replacement { + result.extend_from_slice(bytes); + } + cursor = end; + } + result.extend_from_slice(&input[cursor..]); + // Validate the final declaration set before the CLI creates an output file. + Document::parse(&result)?; + Ok(result) +} diff --git a/src/review/pair.rs b/src/review/pair.rs new file mode 100644 index 0000000..3b247e3 --- /dev/null +++ b/src/review/pair.rs @@ -0,0 +1,223 @@ +//! Adaptive parsing for unscoped comparisons of two revisions. + +use super::*; +use std::{sync::Mutex, thread}; + +struct CellSpan<'a> { + body: &'a [u8], + end_record: usize, +} + +struct FramedRevision<'a> { + records: Vec>, + spans: Option>>, +} + +impl<'a> FramedRevision<'a> { + fn read(input: &'a [u8]) -> Result { + let mut markers = Vec::new(); + let records = read_records_with(input, |index, record| { + if matches!(record.tag, RTAG_BGNSTR | RTAG_ENDSTR) { + markers.push(index); + } + })?; + let spans = index_cells(input, &records, markers); + Ok(Self { records, spans }) + } +} + +// Failure to index a layout leaves error reporting to the regular parser. +fn index_cells<'a>( + input: &'a [u8], + records: &[Record<'a>], + markers: Vec, +) -> Option>> { + let mut cells = BTreeMap::new(); + let mut start = None; + for index in markers { + let record = &records[index]; + match record.tag { + RTAG_BGNSTR => { + if start.is_some() { + return None; + } + let name_record = records.get(index + 1).filter(|r| r.tag == RTAG_STRNAME)?; + let name = decode_name(name_record.data) + .ok() + .filter(|name| !name.is_empty())?; + start = Some((name, name_record.offset)); + } + RTAG_ENDSTR => { + let (name, begin) = start.take()?; + let body = &input[begin..record.offset + 4]; + let span = CellSpan { + body, + end_record: index, + }; + if cells.insert(name, span).is_some() { + return None; + } + } + _ => (), + } + } + start.is_none().then_some(cells) +} + +fn matching_cells<'a>( + old: &FramedRevision<'a>, + new: &FramedRevision<'a>, + input_size: usize, +) -> BTreeMap<&'a str, usize> { + let mut matches = BTreeMap::new(); + if let (Some(old), Some(new)) = (&old.spans, &new.spans) { + let mut bytes = 0; + for (&name, cell) in new { + if old.get(name).is_some_and(|old| old.body == cell.body) { + bytes += cell.body.len(); + matches.insert(name, cell.end_record); + } + } + // Reuse serializes some parsing work, so require substantial overlap. + if bytes < input_size - input_size / 10 { + matches.clear(); + } + } + matches +} + +pub(super) struct CellReuse<'a, 'b> { + old: &'b Document<'a>, + matches: &'b BTreeMap<&'a str, usize>, +} + +impl<'a> CellReuse<'a, '_> { + pub(super) fn cell( + &self, + name: &str, + offset: usize, + records: &[Record<'a>], + ) -> Option<(Cell<'a>, usize)> { + let end = *self.matches.get(name)?; + let previous = &self.old.cells[name]; + // Re-run library metadata and reference checks with the new offsets. + let objects = previous + .objects + .iter() + .filter(|object| object.target.is_some() || object.kind == RTAG_TEXT) + .cloned() + .map(|mut object| { + object.offset = offset + (object.offset - previous.offset); + object.end_offset = offset + (object.end_offset - previous.offset); + object + }) + .collect(); + Some(( + Cell { + offset, + end_offset: records[end].offset, + objects, + attributes: previous.attributes.clone(), + issues: Vec::new(), + }, + end, + )) + } +} + +/// Two unscoped GDSII revisions prepared for comparison. +#[derive(Debug)] +pub struct RevisionPair<'a> { + old: Document<'a>, + new: Document<'a>, + reused_cells: BTreeSet<&'a str>, +} + +impl<'a> RevisionPair<'a> { + pub fn parse(old: &'a [u8], new: &'a [u8]) -> Result { + let invalid = |message, side| Diagnostic::new("invalid_input", message, Some(side)); + let input_size = old.len().max(new.len()); + let parallel = thread::available_parallelism().is_ok_and(|count| count.get() > 1); + let (old, new) = run_together(parallel, old, new, FramedRevision::read); + let old = old.map_err(|message| invalid(message, Side::Old))?; + let new = match new { + Ok(new) => new, + Err(message) => { + Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + return Err(invalid(message, Side::New)); + } + }; + let mut matches = matching_cells(&old, &new, input_size); + let (old, new) = if matches.is_empty() { + let (old, new) = run_together(parallel, old, new, |revision| { + Document::parse_records(&revision.records, None) + }); + (old.map_err(|message| invalid(message, Side::Old))?, new) + } else { + let old_doc = Document::parse_records(&old.records, None) + .map_err(|message| invalid(message, Side::Old))?; + drop(old); + // The regular parser gives unsupported records their own offsets + // and messages, so parse those cells independently on each side. + matches.retain(|name, _| { + old_doc.cells.get(*name).is_some_and(|cell| { + cell.issues.is_empty() + && cell.objects.iter().all(|object| object.issues.is_empty()) + }) + }); + let reuse = CellReuse { + old: &old_doc, + matches: &matches, + }; + let new = Document::parse_records(&new.records, Some(reuse)); + (old_doc, new) + }; + Ok(Self { + old, + new: new.map_err(|message| invalid(message, Side::New))?, + reused_cells: matches.into_keys().collect(), + }) + } + + pub fn compare(self) -> Comparison { + Comparison::collect(&self.old, &self.new, &self.reused_cells, true) + } + + /// Resolve a layer name from the embedded metadata in either revision. + pub fn resolve_layer_name(&self, name: &str) -> Result { + names::resolve_layer_name(name, [&self.old.layer_names, &self.new.layer_names]) + } + + pub fn compare_text(self) -> TextComparison { + TextComparison::collect(&self.old, &self.new, &self.reused_cells) + } +} + +fn run_together( + parallel: bool, + old: I, + new: I, + process: impl Fn(I) -> T + Copy + Send, +) -> (T, T) { + if !parallel { + return (process(old), process(new)); + } + // Transfer ownership only once a worker starts, so a spawn failure can + // process the same input on the calling thread. + let input = Mutex::new(Some(old)); + let run_old = { + let input = &input; + move || { + let old = input.lock().unwrap().take().unwrap(); + process(old) + } + }; + thread::scope(|scope| { + if let Ok(worker) = thread::Builder::new().spawn_scoped(scope, run_old) { + let new = process(new); + return (worker.join().unwrap(), new); + } + (run_old(), process(new)) + }) +} diff --git a/src/review/report.rs b/src/review/report.rs new file mode 100644 index 0000000..8004f48 --- /dev/null +++ b/src/review/report.rs @@ -0,0 +1,486 @@ +//! Versioned machine-readable diff reports. No consumer needs to parse prose. +use super::{hierarchy::HierarchyAnalysis, *}; +use serde::{ + Serialize, Serializer, + ser::{SerializeMap, SerializeSeq}, +}; +use serde_json::{Value, json}; +use std::io; +use std::path::{Path, PathBuf}; + +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, Default)] +pub struct ReportInputs { + pub old: Option, + pub new: Option, +} + +/// Limits affect evidence only. The default includes every changed object. +#[derive(Clone, Debug, Default)] +pub struct ReportOptions { + pub filter: DisplayFilter, + pub max_objects: Option, + pub hierarchy: bool, +} + +fn layer(pair: Layer) -> Value { + json!({"number": pair.0, "purpose": pair.1}) +} + +fn scope(value: &Scope) -> Value { + json!({"top": value.top, "cell": value.cell, "layer": value.layer.map(layer)}) +} + +fn input(path: Option<&Path>, revision: Option<&Revision>) -> Value { + let units = revision + .and_then(|r| r.units.as_deref()) + .map(|data| field(RTAG_UNITS, data)); + json!({ + "path": path.and_then(Path::to_str), + "path_encoded_hex": path.filter(|p| p.to_str().is_none()) + .map(|p| hex(p.as_os_str().as_encoded_bytes())), + "present": revision.map(|r| r.present), + "units": units + }) +} + +fn envelope( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + outcome: Outcome, + revisions: Option<(&Revision, &Revision)>, +) -> Value { + json!({ + "schema_version": SCHEMA_VERSION, + "kind": "diff", + "producer": {"name": "pheon", "version": env!("CARGO_PKG_VERSION")}, + "comparison_profile": PROFILE, + "inputs": {"old": input(inputs.old.as_deref(), revisions.map(|r| r.0)), "new": input(inputs.new.as_deref(), revisions.map(|r| r.1))}, + "scope": scope(selection), + "coordinates": {"unit": "database_unit", "frame": "cell_local", "bounds": "coordinate_envelope"}, + "status": outcome.status(), + "result": outcome.result(), + "exit_code": outcome.exit_code(), + "summary": null, + "report_selection": { + "cell": options.filter.cell, + "layer": options.filter.layer.map(layer), + "max_objects": options.max_objects.map(|n| n.to_string()), + "truncated": false, + "filtered_cells": "0", "filtered_objects": "0", "limited_objects": "0", "emitted_objects": "0" + }, + "layer_legend": [], + "library_changes": [], "layer_name_changes": [], "cell_changes": [], + "hierarchy": null, + "diagnostics": [] + }) +} + +/// Input/selection failures still produce a valid report. Unknown locations are +/// null rather than inferred by parsing an error's human-readable message. +pub fn failure_report( + inputs: &ReportInputs, + selection: &Scope, + options: &ReportOptions, + error: &Diagnostic, +) -> Value { + let mut report = envelope(inputs, selection, options, Outcome::Failed, None); + report["diagnostics"] = json!([diagnostic(error)]); + report +} + +fn diagnostic(value: &Diagnostic) -> Value { + json!({ + "code": value.code, "severity": "error", "message": value.message, + "side": value.side.map(|side| side.to_string()), + "byte_offset": value.offset.map(|offset| offset.to_string()), + "cell": value.cell, "layer": value.layer.map(layer), "target": value.target + }) +} + +fn field(tag: u16, data: &[u8]) -> Value { + let (encoding, value) = if tag == RTAG_XY && data.len().is_multiple_of(8) { + ("xy", json!(XY::points(data).collect::>())) + } else if matches!( + tag, + RTAG_LAYER | RTAG_DATATYPE | RTAG_TEXTTYPE | RTAG_NODETYPE | RTAG_BOXTYPE + ) && data.len().is_multiple_of(2) + { + ( + "uint16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ) + } else { + match tag & 0xff { + 0 if data.is_empty() => ("none", Value::Null), + 1 if data.len().is_multiple_of(2) => ( + "bits", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_be_bytes) + .collect::>() + ), + ), + 2 if data.len().is_multiple_of(2) => ( + "int16", + json!( + data.as_chunks::<2>() + .0 + .iter() + .copied() + .map(i16::from_be_bytes) + .collect::>() + ), + ), + 3 if data.len().is_multiple_of(4) => ( + "int32", + json!( + data.as_chunks::<4>() + .0 + .iter() + .copied() + .map(i32::from_be_bytes) + .collect::>() + ), + ), + 5 if data.len().is_multiple_of(8) => ( + "real8", + json!( + data.as_chunks::<8>() + .0 + .iter() + .map(|&v| decode_real8(u64::from_be_bytes(v))) + .collect::>() + ), + ), + 6 => ("ascii", json!(std::str::from_utf8(data).ok())), + _ => ("bytes", Value::Null), + } + }; + json!({"tag": tag, "name": tag_name(tag), "encoding": encoding, "value": value, "payload_hex": hex(data)}) +} + +fn fields(values: &[FieldValue]) -> Vec { + values.iter().map(|f| field(f.tag, &f.data)).collect() +} + +fn object(value: &ObjectChange, id: &str, side: &str) -> Value { + json!({ + "id": id, "kind": kind_name(value.kind), "tag": value.kind, + "source": {"side": side, "byte_offset": value.offset.to_string(), "end_offset": value.end_offset.to_string()}, + "layer": value.layer.map(layer), "bounds": value.bounds, "target": value.target, + "origin": value.origin.filter(|_| matches!(value.kind, RTAG_SREF | RTAG_AREF | RTAG_TEXT)), "width": value.width, "counts": value.counts, + "fields": fields(&value.fields), + "properties": value.properties.iter().map(|(attribute, bytes)| json!({ + "attribute": attribute, "value": std::str::from_utf8(bytes).ok(), "payload_hex": hex(bytes) + })).collect::>() + }) +} + +fn field_changes(old: &ObjectChange, new: &ObjectChange) -> Vec { + changes::field_changes(old, new) + .map(|(tag, before, after)| { + json!({ + "tag": tag, + "before": before.map(|data| field(tag, data)), + "after": after.map(|data| field(tag, data)) + }) + }) + .collect() +} + +struct SelectedObjects<'a> { + // Up to the last emitted object. A layer filter can leave gaps in this prefix. + objects: &'a [ObjectChange], + layer: Option, + cell_index: usize, + side: &'static str, +} + +impl SelectedObjects<'_> { + fn id(&self, index: usize) -> String { + format!("c{}:{}:{index}", self.cell_index, self.side) + } + + fn contains(&self, index: usize) -> bool { + self.objects + .get(index) + .is_some_and(|object| self.layer.is_none_or(|layer| object.layer == Some(layer))) + } +} + +impl Serialize for SelectedObjects<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(None)?; + for (index, change) in self.objects.iter().enumerate() { + if self.layer.is_none_or(|layer| change.layer == Some(layer)) { + sequence.serialize_element(&object(change, &self.id(index), self.side))?; + } + } + sequence.end() + } +} + +struct SelectedCell<'a> { + cell: &'a CellChange, + removed: SelectedObjects<'a>, + added: SelectedObjects<'a>, +} + +impl Serialize for SelectedCell<'_> { + fn serialize(&self, serializer: S) -> Result { + let revision = |revision: &CellRevision| json!({"byte_offset": revision.offset.to_string(), "attributes": fields(&revision.attributes)}); + let mut map = serializer.serialize_map(Some(7))?; + map.serialize_entry("added", &self.added)?; + map.serialize_entry("after", &self.cell.new.as_ref().map(revision))?; + map.serialize_entry("attributes_changed", &self.cell.attributes_changed())?; + map.serialize_entry("before", &self.cell.old.as_ref().map(revision))?; + map.serialize_entry("name", &self.cell.name)?; + map.serialize_entry("removed", &self.removed)?; + map.serialize_entry("replacements", &SelectedReplacements(self))?; + map.end() + } +} + +struct SelectedReplacements<'a>(&'a SelectedCell<'a>); + +impl Serialize for SelectedReplacements<'_> { + fn serialize(&self, serializer: S) -> Result { + let selection = self.0; + let mut sequence = serializer.serialize_seq(None)?; + for replacement in &selection.cell.replacements { + if selection.removed.contains(replacement.removed) + && selection.added.contains(replacement.added) + { + let old = &selection.cell.removed[replacement.removed]; + let new = &selection.cell.added[replacement.added]; + sequence.serialize_element(&json!({ + "removed": selection.removed.id(replacement.removed), + "added": selection.added.id(replacement.added), + "match_rule": replacement.match_rule, + "explanation": changes::replacement_summary(old, new), + "field_changes": field_changes(old, new), + "properties_changed": old.properties != new.properties + }))?; + } + } + sequence.end() + } +} + +struct JsonReport<'a> { + metadata: Value, + cells: Vec>, +} + +impl Serialize for JsonReport<'_> { + fn serialize(&self, serializer: S) -> Result { + let metadata = self + .metadata + .as_object() + .expect("report metadata is an object"); + let mut map = serializer.serialize_map(Some(metadata.len()))?; + for (key, value) in metadata { + if key == "cell_changes" { + map.serialize_entry(key, &self.cells)?; + } else { + map.serialize_entry(key, value)?; + } + } + map.end() + } +} + +impl Comparison { + /// Render the captured comparison evidence without retaining its source documents. + /// IDs are local to this report, not persistent object identities. + pub fn json_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> Value { + serde_json::to_value(self.prepare_report(inputs, options)) + .expect("serialize comparison report") + } + + /// Stream a pretty-printed report to a writer. + pub fn write_json_report( + &self, + writer: impl io::Write, + inputs: &ReportInputs, + options: &ReportOptions, + ) -> serde_json::Result<()> { + serde_json::to_writer_pretty(writer, &self.prepare_report(inputs, options)) + } + + fn prepare_report(&self, inputs: &ReportInputs, options: &ReportOptions) -> JsonReport<'_> { + let outcome = self.outcome(); + let mut report = envelope( + inputs, + &self.old.scope, + options, + outcome, + Some((&self.old, &self.new)), + ); + report["summary"] = json!({ + "counts": if outcome.is_complete() { "exact" } else { "observed" }, + "changed_cells": self.cells.len().to_string(), + "removed_objects": self.cells.iter().map(|c| c.removed.len()).sum::().to_string(), + "added_objects": self.cells.iter().map(|c| c.added.len()).sum::().to_string(), + "layer_name_changes": self.layer_names.len().to_string(), + "library_field_changes": self.library_changes.len().to_string() + }); + report["layer_legend"] = json!( + self.layer_legend + .iter() + .map(|(pair, name)| json!({"layer": layer(*pair), "name": name})) + .collect::>() + ); + report["layer_name_changes"] = json!( + self.layer_names + .iter() + .map(|c| json!({ + "layer": layer(c.layer), "before": c.old, "after": c.new + })) + .collect::>() + ); + report["library_changes"] = json!(self.library_changes.iter().map(|change| { + match change { + LibraryChange::Presence { old, new } => { + json!({"field": "presence", "tag": null, "before": old, "after": new}) + } + LibraryChange::Record { tag, old, new } => { + let values = |data: &[Vec]| { + data.iter().map(|bytes| field(*tag, bytes)).collect::>() + }; + json!({"field": tag_name(*tag), "tag": tag, "before": values(old), "after": values(new)}) + } + } + }).collect::>()); + let (mut emitted, mut filtered_objects, mut limited, mut filtered_cells) = + (0_usize, 0_usize, 0_usize, 0_usize); + let mut cells = Vec::new(); + for (cell_index, cell) in self.cells.iter().enumerate() { + if options + .filter + .cell + .as_ref() + .is_some_and(|name| name != &cell.name) + { + filtered_cells += 1; + filtered_objects += cell.removed.len() + cell.added.len(); + continue; + } + let mut select = |objects: &[ObjectChange]| { + let mut end = 0; + for (index, change) in objects.iter().enumerate() { + if options + .filter + .layer + .is_some_and(|l| change.layer != Some(l)) + { + filtered_objects += 1; + } else if options.max_objects.is_some_and(|max| emitted >= max) { + limited += 1; + } else { + end = index + 1; + emitted += 1; + } + } + end + }; + let layer = options.filter.layer; + cells.push(SelectedCell { + cell, + removed: SelectedObjects { + objects: &cell.removed[..select(&cell.removed)], + layer, + cell_index, + side: "old", + }, + added: SelectedObjects { + objects: &cell.added[..select(&cell.added)], + layer, + cell_index, + side: "new", + }, + }); + } + report["report_selection"]["truncated"] = + json!(filtered_cells > 0 || filtered_objects > 0 || limited > 0); + for (key, count) in [ + ("filtered_cells", filtered_cells), + ("filtered_objects", filtered_objects), + ("limited_objects", limited), + ("emitted_objects", emitted), + ] { + report["report_selection"][key] = json!(count.to_string()); + } + report["diagnostics"] = json!(self.diagnostics.iter().map(diagnostic).collect::>()); + if options.hierarchy { + report["hierarchy"] = graph(self); + } + JsonReport { + metadata: report, + cells, + } + } +} + +fn graph(diff: &Comparison) -> Value { + let old = &diff.old; + let new = &diff.new; + let HierarchyAnalysis { + old_edges, + new_edges, + edges, + direct, + affected, + changed_refs, + .. + } = HierarchyAnalysis::between(old, new, Some(&diff.cells)); + let names: BTreeSet<_> = old + .cells + .iter() + .chain(new.cells.iter()) + .cloned() + .chain( + edges + .iter() + .flat_map(|(parent, child)| [parent.clone(), child.clone()]), + ) + .collect(); + let old_targets: BTreeSet<_> = old_edges.keys().map(|(_, t)| t.as_str()).collect(); + let new_targets: BTreeSet<_> = new_edges.keys().map(|(_, t)| t.as_str()).collect(); + let state = |doc: &Revision, targets: &BTreeSet<&str>, name: &String| { + if doc.cells.contains(name) { + "included" + } else if doc.known_cells.contains(name) { + "outside_scope" + } else if targets.contains(name.as_str()) { + "unresolved" + } else { + "absent" + } + }; + json!({ + "nodes": names.iter().map(|name| json!({ + "cell": name, "before": state(old, &old_targets, name), "after": state(new, &new_targets, name), + "change": if direct.contains(name) { "direct" } else if affected.contains(name) { "affected" } else { "unchanged" } + })).collect::>(), + "edges": edges.iter().map(|edge| json!({ + "parent": edge.0, "target": edge.1, + "before_count": old_edges.get(edge).copied().unwrap_or(0).to_string(), + "after_count": new_edges.get(edge).copied().unwrap_or(0).to_string(), + "reference_changed": changed_refs.contains(edge) + })).collect::>() + }) +} diff --git a/src/review/selection.rs b/src/review/selection.rs new file mode 100644 index 0000000..4e3491f --- /dev/null +++ b/src/review/selection.rs @@ -0,0 +1,350 @@ +//! Comparison scope and display-only filters. + +use super::*; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Scope { + pub top: Option, + pub cell: Option, + pub layer: Option, +} + +/// Presentation filters never modify comparison status, totals, or hierarchy. +#[derive(Clone, Debug, Default)] +pub struct DisplayFilter { + pub cell: Option, + pub layer: Option, +} + +#[derive(Default)] +pub(super) struct TextSelection<'a> { + pub removed_cells: Vec<&'a str>, + pub added_cells: Vec<&'a str>, + pub modified_cells: Vec>, + pub filtered_objects: usize, + pub limited_objects: usize, + pub filtered_cells: usize, +} + +pub(super) struct TextCellSelection<'a> { + pub cell: &'a CellChange, + pub attributes_changed: bool, + pub replacements: Vec<(&'a ObjectChange, &'a ObjectChange)>, + pub removed: Vec<&'a ObjectChange>, + pub added: Vec<&'a ObjectChange>, +} + +impl<'a> TextSelection<'a> { + pub fn new(cells: &'a [CellChange], show: &DisplayFilter, mut remaining: usize) -> Self { + let mut selection = Self::default(); + let visible = + |object: &ObjectChange| show.layer.is_none_or(|layer| object.layer == Some(layer)); + for cell in cells { + let marker = cell_marker(cell.old.is_some(), cell.new.is_some()); + let attributes_changed = marker == '*' && cell.attributes_changed(); + if show.cell.as_ref().is_some_and(|name| name != &cell.name) { + selection.filtered_cells += usize::from(marker != '*' || attributes_changed); + if marker == '*' { + selection.filtered_objects += cell.removed.len() + cell.added.len(); + } + continue; + } + match marker { + '-' => { + selection.removed_cells.push(&cell.name); + continue; + } + '+' => { + selection.added_cells.push(&cell.name); + continue; + } + _ => (), + } + + let mut selected = TextCellSelection { + cell, + attributes_changed, + replacements: Vec::new(), + removed: Vec::new(), + added: Vec::new(), + }; + let mut paired_old = BTreeSet::new(); + let mut paired_new = BTreeSet::new(); + for pair in &cell.replacements { + let old = &cell.removed[pair.removed]; + let new = &cell.added[pair.added]; + if visible(old) && visible(new) { + paired_old.insert(pair.removed); + paired_new.insert(pair.added); + if remaining >= 2 { + selected.replacements.push((old, new)); + remaining -= 2; + } else { + selection.limited_objects += 2; + } + } + } + // A layer filter may leave one side of a pair eligible on its own. + for (objects, paired, selected) in [ + (&cell.removed, &paired_old, &mut selected.removed), + (&cell.added, &paired_new, &mut selected.added), + ] { + for (index, object) in objects.iter().enumerate() { + if !visible(object) { + selection.filtered_objects += 1; + } else if !paired.contains(&index) { + if remaining == 0 { + selection.limited_objects += 1; + } else { + selected.push(object); + remaining -= 1; + } + } + } + } + if attributes_changed + || !selected.replacements.is_empty() + || !selected.removed.is_empty() + || !selected.added.is_empty() + { + selection.modified_cells.push(selected); + } + } + selection + } +} + +impl Scope { + pub(super) fn write_restrictions(&self, out: &mut TextReport) { + if self == &Self::default() { + return; + } + let start = out.len(); + out.push_str("Scope"); + if let Some(top) = &self.top { + write!(out, " top={top:?}").unwrap(); + } + if let Some(cell) = &self.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = self.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + + pub fn description(&self) -> String { + let top = self + .top + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let cell = self + .cell + .as_ref() + .map(|s| format!("{s:?}")) + .unwrap_or_else(|| "all".to_string()); + let layer = self + .layer + .map(|(a, b)| format!("{a}/{b}")) + .unwrap_or_else(|| "all".to_string()); + format!("scope top={top} cell={cell} layer={layer}") + } + + pub fn validate_pair(&self, old: &Document<'_>, new: &Document<'_>) -> Result<(), String> { + for (kind, name) in [("top", &self.top), ("cell", &self.cell)] { + if let Some(name) = name + && !old.cells.contains_key(name) + && !new.cells.contains_key(name) + { + return Err(format!( + "selected {kind} {name:?} exists in neither revision" + )); + } + } + if let (Some(top), Some(cell)) = (&self.top, &self.cell) + && !old.reachable(top).contains(cell) + && !new.reachable(top).contains(cell) + { + return Err(format!( + "selected cell {cell:?} is not reachable from {top:?}" + )); + } + Ok(()) + } +} + +impl<'a> Document<'a> { + /// A missing Git revision has no cells or declarations. Retain the other + /// revision's library header to avoid inventing a unit/name change. + pub fn empty_like(&self) -> Self { + Self { + records: Vec::new(), + cells: BTreeMap::new(), + layer_names: BTreeMap::new(), + diagnostics: Vec::new(), + header: self.header.clone(), + global_issues: Vec::new(), + known_cells: BTreeSet::new(), + scope: self.scope.clone(), + present: false, + } + } + fn reachable(&self, root: &str) -> BTreeSet { + let mut found = BTreeSet::new(); + let mut pending = vec![root.to_string()]; + while let Some(name) = pending.pop() { + if let Some(cell) = self.cells.get(&name) + && found.insert(name) + { + pending.extend(cell.objects.iter().filter_map(|o| o.target.clone())); + } + } + found + } + + pub(super) fn refresh_diagnostics(&mut self) { + self.diagnostics = self.global_issues.clone(); + for (name, cell) in &self.cells { + for issue in &cell.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + for object in &cell.objects { + for issue in &object.issues { + let mut issue = issue.clone(); + issue.cell = Some(name.clone()); + issue.layer = object.layer; + issue.message = format!("cell {name:?}: {}", issue.message); + self.diagnostics.push(issue); + } + } + } + self.check_graph(); + } + + /// Project a parsed document. Missing selectors are allowed only for one + /// side of a comparison, after Scope::validate_pair has checked both sides. + pub fn selected(mut self, scope: &Scope, allow_missing: bool) -> Result { + if self.scope != Scope::default() { + return Err("document is already scoped; combine selectors in one Scope".to_string()); + } + if !allow_missing { + scope.validate_pair(&self, &self)?; + } + let mut wanted = scope + .top + .as_ref() + .map(|t| self.reachable(t)) + .unwrap_or_else(|| self.cells.keys().cloned().collect()); + if let Some(cell) = &scope.cell { + wanted.retain(|n| n == cell); + } + // Determine layer coverage before projecting cells: exact-cell scopes + // still need to know whether a referenced definition uses this layer. + let relevant = scope.layer.map(|layer| { + let mut relevant: BTreeSet<_> = self + .cells + .iter() + .filter(|(_, c)| { + !c.issues.is_empty() + || c.objects.iter().any(|o| { + !o.metadata + && (o.layer == Some(layer) + || (!o.issues.is_empty() && o.layer.is_none())) + }) + }) + .map(|(n, _)| n.clone()) + .collect(); + let mut parents = BTreeMap::>::new(); + for (parent, cell) in &self.cells { + for o in &cell.objects { + if let Some(target) = &o.target { + parents + .entry(target.clone()) + .or_default() + .push(parent.clone()); + if !self.known_cells.contains(target) { + relevant.insert(parent.clone()); + } + } + } + } + let mut pending: Vec<_> = relevant.iter().cloned().collect(); + // A cycle cannot be proven to have no effect on a layer. Preserve + // cycles (and their descendants) so graph validation remains visible. + let mut indegree: BTreeMap<_, usize> = + self.cells.keys().map(|n| (n.clone(), 0)).collect(); + for cell in self.cells.values() { + for target in cell.objects.iter().filter_map(|o| o.target.as_ref()) { + if let Some(n) = indegree.get_mut(target) { + *n += 1; + } + } + } + let mut ready: Vec<_> = indegree + .iter() + .filter(|(_, n)| **n == 0) + .map(|(n, _)| n.clone()) + .collect(); + while let Some(name) = ready.pop() { + for target in self.cells[&name] + .objects + .iter() + .filter_map(|o| o.target.as_ref()) + { + if let Some(n) = indegree.get_mut(target) { + *n -= 1; + if *n == 0 { + ready.push(target.clone()); + } + } + } + } + for (name, _) in indegree.iter().filter(|(_, n)| **n != 0) { + if relevant.insert(name.clone()) { + pending.push(name.clone()); + } + } + while let Some(child) = pending.pop() { + for parent in parents.get(&child).into_iter().flatten() { + if relevant.insert(parent.clone()) { + pending.push(parent.clone()); + } + } + } + relevant + }); + self.cells.retain(|name, _| { + wanted.contains(name) && relevant.as_ref().is_none_or(|r| r.contains(name)) + }); + if let (Some(layer), Some(relevant)) = (scope.layer, relevant) { + for cell in self.cells.values_mut() { + cell.objects.retain(|o| { + o.metadata + || o.layer == Some(layer) + || o.target + .as_ref() + .is_some_and(|t| relevant.contains(t) || !self.known_cells.contains(t)) + || (!o.issues.is_empty() && o.layer.is_none()) + }); + } + self.layer_names.retain(|pair, _| *pair == layer); + } else if scope.top.is_some() || scope.cell.is_some() { + let layers: BTreeSet<_> = self + .cells + .values() + .flat_map(|c| &c.objects) + .filter(|o| !o.metadata) + .filter_map(|o| o.layer) + .collect(); + self.layer_names.retain(|pair, _| layers.contains(pair)); + } + self.scope = scope.clone(); + self.refresh_diagnostics(); + Ok(self) + } +} diff --git a/src/review/text.rs b/src/review/text.rs new file mode 100644 index 0000000..d47ed16 --- /dev/null +++ b/src/review/text.rs @@ -0,0 +1,699 @@ +//! Text comparison reports and presentation roles shared with the terminal. + +use super::{hierarchy::write_hierarchy, *}; +use std::ops::Range; + +/// Additional presentation options for a text comparison. +#[derive(Clone, Copy, Debug, Default)] +pub struct TextReportOptions { + /// Print individual object evidence, including both sides of replacements. + pub details: bool, + /// Append the ASCII hierarchy at this depth when provided. + pub ascii_depth: Option, +} + +/// A presentation role for a span of report text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextStyle { + Heading, + Layer, + Added, + Removed, + Changed, + Success, + Warning, + Error, +} + +impl TextStyle { + pub(super) fn change(marker: char) -> Self { + match marker { + '+' => Self::Added, + '-' => Self::Removed, + _ => Self::Changed, + } + } +} + +/// A plain text report with optional style spans. +#[derive(Default)] +pub struct TextReport { + text: String, + spans: Option, TextStyle)>>, +} + +impl TextReport { + pub fn new(styled: bool) -> Self { + Self { + text: String::new(), + spans: styled.then(Vec::new), + } + } + + pub fn as_str(&self) -> &str { + &self.text + } + + pub fn into_string(self) -> String { + self.text + } + + /// Byte ranges are in output order and never overlap. + pub fn styles(&self) -> &[(Range, TextStyle)] { + self.spans.as_deref().unwrap_or(&[]) + } + + pub(super) fn is_styled(&self) -> bool { + self.spans.is_some() + } + + pub fn write_styled(&mut self, style: TextStyle, text: fmt::Arguments<'_>) { + let start = self.len(); + self.write_fmt(text).unwrap(); + self.mark(start, style); + } + + pub(super) fn push_str(&mut self, text: &str) { + self.text.push_str(text); + } + + pub(super) fn push(&mut self, ch: char) { + self.text.push(ch); + } + + pub(super) fn len(&self) -> usize { + self.text.len() + } + + pub(super) fn mark(&mut self, start: usize, style: TextStyle) { + let end = self.text.len(); + if start < end + && let Some(spans) = &mut self.spans + { + if let Some((range, previous)) = spans.last_mut() + && range.end == start + && *previous == style + { + range.end = end; + } else { + spans.push((start..end, style)); + } + } + } +} + +impl fmt::Write for TextReport { + fn write_str(&mut self, text: &str) -> fmt::Result { + self.push_str(text); + Ok(()) + } +} + +enum Part { + Text(String), + Layer(Layer), + Placements(Option<(u64, u64)>), +} + +/// Cell details awaiting inline layer labels and reference placement totals. +pub(super) struct CellText<'a> { + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + layers: BTreeMap, + parts: Vec<(Option, Part)>, + styled: bool, + style: Option, +} + +impl<'a> CellText<'a> { + pub fn new( + names: &'a BTreeMap, + name_changes: &'a [LayerNameChange], + styled: bool, + ) -> Self { + Self { + names, + name_changes, + layers: BTreeMap::new(), + parts: Vec::new(), + styled, + style: None, + } + } + + pub fn style(&mut self, style: Option) { + self.style = style.filter(|_| self.styled); + } + + pub fn write_layer(&mut self, layer: Layer, side: Option) { + if self.names.contains_key(&layer) { + self.layers + .entry(layer) + .and_modify(|repeated| *repeated = true) + .or_insert(false); + self.parts.push((self.style, Part::Layer(layer))); + } else { + write!(self, "{}/{}", layer.0, layer.1).unwrap(); + if let Some(name) = self.transition_name(layer, side) { + write!(self, " ({})", cell_label(name)).unwrap(); + } + } + } + + pub fn reserve_placements(&mut self) -> usize { + let index = self.parts.len(); + self.parts.push((self.style, Part::Placements(None))); + index + } + + pub fn set_placements(&mut self, index: usize, old: u64, new: u64) { + self.parts[index].1 = Part::Placements(Some((old, new))); + } + + pub fn push_str(&mut self, text: &str) { + self.write_str(text).unwrap(); + } + + pub fn push(&mut self, ch: char) { + self.write_char(ch).unwrap(); + } + + fn append(&mut self, text: &str) { + if text.is_empty() { + return; + } + if let Some((style, Part::Text(current))) = self.parts.last_mut() + && *style == self.style + { + current.push_str(text); + } else { + self.parts.push((self.style, Part::Text(text.to_string()))); + } + } + + fn transition_name(&self, layer: Layer, side: Option) -> Option<&'a str> { + let side = side?; + let at = self + .name_changes + .binary_search_by_key(&layer, |change| change.layer) + .ok()?; + let change = &self.name_changes[at]; + match side { + Side::Old => change.old.as_deref(), + Side::New => change.new.as_deref(), + } + } + + pub fn write_legend(&self, out: &mut TextReport) { + for (&(layer, datatype), &repeated) in &self.layers { + if repeated { + writeln!( + out, + "{layer}/{datatype}: name {:?}", + self.names[&(layer, datatype)] + ) + .unwrap(); + } + } + } + + pub fn write_body(self, out: &mut TextReport) { + for (style, part) in self.parts { + let start = out.len(); + match part { + Part::Text(text) => out.push_str(&text), + Part::Layer((layer, datatype)) => { + write!(out, "{layer}/{datatype}").unwrap(); + if !self.layers[&(layer, datatype)] { + write!(out, " ({})", cell_label(&self.names[&(layer, datatype)])).unwrap(); + } + } + Part::Placements(Some((old, new))) if old == new => { + let suffix = if new == 1 { "" } else { "s" }; + write!(out, " · {new} total placement{suffix}").unwrap(); + } + Part::Placements(Some((old, new))) => { + write!(out, " · placements {old} -> {new}").unwrap(); + } + Part::Placements(None) => (), + } + if let Some(style) = style { + out.mark(start, style); + } + } + } +} + +impl fmt::Write for CellText<'_> { + fn write_str(&mut self, text: &str) -> fmt::Result { + if self.style.is_some() + && let Some((first, rest)) = text.split_once('\n') + { + self.append(first); + self.style = None; + self.append("\n"); + self.append(rest); + } else { + self.append(text); + } + Ok(()) + } +} + +impl TextComparison { + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.comparison + .render_text(max_objects, show, show_library_presence) + } + + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + self.comparison + .render_text_with_options(max_objects, show, show_library_presence, options) + } + + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + self.comparison + .write_text_report(out, max_objects, show, show_library_presence, options); + } + + /// Render the cell changes and their ASCII hierarchy together. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.comparison.render_text_with_ascii_hierarchy( + max_objects, + show, + show_library_presence, + max_depth, + ) + } +} + +impl Comparison { + /// Limit presentation only: the complete comparison has already been made. + pub fn text(&self, max_objects: usize) -> String { + self.text_filtered(max_objects, &DisplayFilter::default()) + } + + pub fn text_filtered(&self, max_objects: usize, show: &DisplayFilter) -> String { + self.render_text(max_objects, show, true) + } + + /// Render a text report. Set `show_library_presence` to false when a file + /// header already announces the library addition or deletion. + pub fn render_text( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions::default(), + ) + } + + /// Render the cell changes and omit simple hierarchy edges already shown there. + pub fn render_text_with_ascii_hierarchy( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + max_depth: usize, + ) -> String { + self.render_text_with_options( + max_objects, + show, + show_library_presence, + TextReportOptions { + ascii_depth: Some(max_depth), + ..TextReportOptions::default() + }, + ) + } + + /// Render the selected changes with optional object details and ASCII hierarchy. + pub fn render_text_with_options( + &self, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) -> String { + let mut out = TextReport::default(); + self.write_text_report(&mut out, max_objects, show, show_library_presence, options); + out.into_string() + } + + /// Append the selected changes and their presentation roles to a text report. + pub fn write_text_report( + &self, + out: &mut TextReport, + max_objects: usize, + show: &DisplayFilter, + show_library_presence: bool, + options: TextReportOptions, + ) { + let selection = selection::TextSelection::new(&self.cells, show, max_objects); + let outcome = self.outcome(); + let headline = out.len(); + let (result, style) = match outcome { + Outcome::Equal => ("Equal", TextStyle::Success), + Outcome::Different => ("Different", TextStyle::Warning), + Outcome::Incomplete { .. } => ("Incomplete", TextStyle::Warning), + Outcome::Failed => ("Failed", TextStyle::Error), + }; + out.push_str(result); + let has_omissions = selection.filtered_objects != 0 + || selection.limited_objects != 0 + || selection.filtered_cells != 0; + if !self.cells.is_empty() + && (self.cells.len() > 3 || !outcome.is_complete() || has_omissions) + { + let (mut modified, mut added, mut removed) = (0, 0, 0); + for cell in &self.cells { + match cell_marker(cell.old.is_some(), cell.new.is_some()) { + '-' => removed += 1, + '+' => added += 1, + _ => modified += 1, + } + } + let mut counts = [ + (modified, "modified"), + (added, "added"), + (removed, "removed"), + ] + .into_iter() + .filter(|(count, _)| *count != 0); + if let Some((count, action)) = counts.next() { + let noun = if count == 1 { "cell" } else { "cells" }; + write!(out, " · {count} {noun} {action}").unwrap(); + for (count, action) in counts { + write!(out, ", {count} {action}").unwrap(); + } + } + } + let library_changes = self.library_changes.iter().filter(|change| { + show_library_presence || !matches!(change, LibraryChange::Presence { .. }) + }); + for (count, label) in [ + (library_changes.clone().count(), "library change"), + (self.layer_names.len(), "layer name change"), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural}").unwrap(); + } + } + out.mark(headline, style); + out.push('\n'); + self.old.scope.write_restrictions(out); + if show.cell.is_some() || show.layer.is_some() { + let start = out.len(); + out.push_str("Display filter"); + if let Some(cell) = &show.cell { + write!(out, " cell={cell:?}").unwrap(); + } + if let Some((layer, datatype)) = show.layer { + write!(out, " layer={layer}/{datatype}").unwrap(); + } + out.mark(start, TextStyle::Heading); + out.push('\n'); + } + if has_omissions { + let start = out.len(); + out.push_str("Omitted"); + for (count, label, reason) in [ + (selection.filtered_objects, "object change", "filtered out"), + (selection.limited_objects, "object change", "over the limit"), + ( + selection.filtered_cells, + "cell-level change", + "filtered out", + ), + ] { + if count != 0 { + let plural = if count == 1 { "" } else { "s" }; + write!(out, " · {count} {label}{plural} {reason}").unwrap(); + } + } + out.mark(start, TextStyle::Warning); + out.push('\n'); + } + for change in library_changes { + writeln!(out, "library {change}").unwrap(); + } + if self.library_changes.iter().any(|change| { + matches!( + change, + LibraryChange::Record { + tag: RTAG_UNITS, + .. + } + ) + }) { + out.push_str("Coordinates use each file's database units\n"); + } + for change in &self.layer_names { + let name = |value: Option<&str>| { + value.map_or_else(|| "absent".to_string(), |value| format!("{value:?}")) + }; + writeln!( + out, + "{}/{}: name {} -> {}", + change.layer.0, + change.layer.1, + name(change.old.as_deref()), + name(change.new.as_deref()) + ) + .unwrap(); + } + + let mut body = CellText::new(&self.layer_legend, &self.layer_names, out.is_styled()); + let mut covered_edges = BTreeSet::new(); + for selected in &selection.modified_cells { + self.write_cell_details( + &mut body, + selected, + options.details, + (options.ascii_depth.is_some() && outcome.is_complete()) + .then_some(&mut covered_edges), + ); + } + body.write_legend(out); + if !selection.modified_cells.is_empty() + || !selection.removed_cells.is_empty() + || !selection.added_cells.is_empty() + { + out.write_styled(TextStyle::Heading, format_args!("Cells:")); + out.push('\n'); + for name in selection.removed_cells { + out.write_styled(TextStyle::Removed, format_args!("- {}", cell_label(name))); + out.push('\n'); + } + for name in selection.added_cells { + out.write_styled(TextStyle::Added, format_args!("+ {}", cell_label(name))); + out.push('\n'); + } + body.write_body(out); + } + // General comparison errors precede revision-specific errors in text. + if !self.diagnostics.is_empty() { + out.write_styled(TextStyle::Heading, format_args!("Diagnostics")); + out.push('\n'); + for diagnostic in self + .diagnostics + .iter() + .filter(|d| d.side.is_none()) + .chain(self.diagnostics.iter().filter(|d| d.side.is_some())) + { + writeln!(out, " {diagnostic}").unwrap(); + } + } + if let Some(max_depth) = options.ascii_depth { + write_hierarchy( + out, + &self.old, + &self.new, + Some(&self.cells), + false, + max_depth, + Some(&covered_edges), + ); + } + } + + fn write_cell_details( + &self, + out: &mut CellText<'_>, + selected: &selection::TextCellSelection<'_>, + details: bool, + mut covered_edges: Option<&mut BTreeSet<(String, String)>>, + ) { + let cell = selected.cell; + let name = cell_label(&cell.name); + let note = if selected.attributes_changed { + " (attributes changed)" + } else { + "" + }; + out.style(Some(TextStyle::Changed)); + writeln!(out, "* {name}{note}").unwrap(); + let mut polygons = changes::PolygonChanges::default(); + let mut shown_refs = BTreeMap::<&str, changes::ShownReferences>::new(); + for &(a, b) in &selected.replacements { + if !details && a.layer == b.layer && is_polygon(a.kind) { + polygons.replacement(a, b); + continue; + } + out.push_str(" "); + out.style(Some(TextStyle::Changed)); + out.push_str("~ "); + if a.layer != b.layer { + changes::write_layer_transition(out, a, b); + out.push('\n'); + } else { + if let Some(layer) = a.layer { + out.write_layer(layer, None); + out.push_str(": "); + } + let description = changes::replacement_description(a, b, details); + let (heading, rest) = match description.split_once('\n') { + Some((heading, rest)) => (heading, Some(rest)), + None => (description.as_str(), None), + }; + out.push_str(heading); + if let Some(target) = &a.target { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .replacement(a, b, placements); + } + out.push('\n'); + if let Some(rest) = rest { + writeln!(out, "{rest}").unwrap(); + } + } + if details { + changes::write_replacement_objects(out, a, b); + } + } + let mut removed_refs = BTreeMap::new(); + let mut added_refs = BTreeMap::new(); + for (removed, objects, references) in [ + (true, &selected.removed, &mut removed_refs), + (false, &selected.added, &mut added_refs), + ] { + let sign = if removed { '-' } else { '+' }; + let mut unmatched: Vec<(&ObjectChange, usize)> = Vec::new(); + for &object in objects { + if !details && is_polygon(object.kind) { + polygons.unmatched(object, removed); + } else if !details && let Some(target) = object.target.as_deref() { + let (_, counts) = references + .entry(target) + .or_insert_with(|| (object, changes::ReferenceCounts::default())); + counts.add(object); + } else if let Some((last, count)) = unmatched.last_mut() + && last.kind == object.kind + && last.fields == object.fields + && last.properties == object.properties + { + *count += 1; + } else { + unmatched.push((object, 1)); + } + } + // The comparison orders these objects by kind, fields, and properties. + for (object, count) in unmatched { + out.push_str(" "); + out.style(Some(TextStyle::change(sign))); + write!(out, "{sign} ").unwrap(); + if count > 1 { + write!(out, "{count}× ").unwrap(); + } + changes::write_unmatched(out, object, details); + if let Some(target) = object.target.as_deref() { + let placements = out.reserve_placements(); + shown_refs + .entry(target) + .or_default() + .individual(object, removed, count, placements); + } + out.push('\n'); + } + } + let no_refs = changes::ReferenceCounts::default(); + for (target, (first, counts)) in removed_refs { + if let Some((_, added)) = added_refs.remove(target) { + let placements = counts.write_change(out, target, &added); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &added, placements); + } else { + let placements = counts.write(out, '-', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&counts, &no_refs, placements); + } + } + for (target, (first, counts)) in added_refs { + let placements = counts.write(out, '+', target, first); + shown_refs + .entry(target) + .or_default() + .unmatched(&no_refs, &counts, placements); + } + if !shown_refs.is_empty() { + for target in cell + .removed + .iter() + .chain(&cell.added) + .filter_map(|o| o.target.as_deref()) + { + if let Some(shown) = shown_refs.get_mut(target) { + shown.count_change(); + } + } + for (target, shown) in shown_refs { + if shown.complete() { + let edge = (cell.name.clone(), target.to_owned()); + let old = self.old.edges.get(&edge).copied().unwrap_or(0); + let new = self.new.edges.get(&edge).copied().unwrap_or(0); + shown.write_placements(out, old, new); + if let Some(covered) = covered_edges.as_deref_mut() { + covered.insert(edge); + } + } + } + } + polygons.write(out); + } +} diff --git a/test.py b/test.py new file mode 100644 index 0000000..448e8e8 --- /dev/null +++ b/test.py @@ -0,0 +1,8 @@ +"""Read a GDSII file through Pheon's supported Python API.""" +import sys + +from pheon import arrow + + +if __name__ == "__main__": + print(arrow.read_path(sys.argv[1])) diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..993caa8 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,20 @@ +use pheon::{basic::encode_real8, records}; + +pub fn record(out: &mut Vec, tag: u16, data: &[u8]) { + out.extend_from_slice(&u16::try_from(data.len() + 4).unwrap().to_be_bytes()); + out.extend_from_slice(&tag.to_be_bytes()); + out.extend_from_slice(data); +} + +pub fn library_header() -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_BGNLIB, &[0; 24]); + record(&mut bytes, records::RTAG_LIBNAME, b"test"); + let units: Vec = [0.001, 1e-9] + .into_iter() + .flat_map(|v| encode_real8(v).unwrap().to_be_bytes()) + .collect(); + record(&mut bytes, records::RTAG_UNITS, &units); + bytes +} diff --git a/tests/ffi_errors.rs b/tests/ffi_errors.rs new file mode 100644 index 0000000..3b81821 --- /dev/null +++ b/tests/ffi_errors.rs @@ -0,0 +1,157 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{last_error_message, read_path, read_cells_bytes, scan_bytes, records}; +use std::{ffi::{CStr, CString}, fs, path::PathBuf, ptr}; + +struct TestFile(PathBuf); + +impl TestFile { + fn new(name: &str, contents: &[u8]) -> Self { + let path = std::env::temp_dir().join(format!("pheon-{name}-{}.gds", std::process::id())); + fs::write(&path, contents).unwrap(); + Self(path) + } + + fn cpath(&self) -> CString { + CString::new(self.0.to_str().unwrap()).unwrap() + } +} + +impl Drop for TestFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +fn error_message() -> String { + let message = last_error_message(); + assert!(!message.is_null()); + unsafe { CStr::from_ptr(message) }.to_str().unwrap().to_owned() +} + +#[test] +fn path_errors_preserve_outputs_and_clear_after_success() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("path pointer was null")); + assert!(array.is_released()); + + // A mismatched timestamp size returns a parse error with a byte offset. + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]); + let file = TestFile::new("invalid-timestamps", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1); + assert!(error_message().contains("read failed at byte 10: Expected record size")); + assert!(array.is_released()); + + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("valid-empty", &bytes); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(!array.is_released()); + assert!(last_error_message().is_null()); +} + +#[test] +fn other_threads_do_not_clear_or_free_error_messages() { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + let pointer = last_error_message(); + let message = error_message(); + + std::thread::spawn(|| { + assert!(last_error_message().is_null()); + let mut bytes = common::library_header(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let file = TestFile::new("thread-empty", &bytes); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1); + assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, 0); + assert!(last_error_message().is_null()); + }).join().unwrap(); + + assert_eq!(last_error_message(), pointer); + assert_eq!(unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(), message); +} + +#[test] +fn null_arrow_outputs_return_an_error() { + assert_eq!(unsafe { read_path(ptr::null(), ptr::null_mut(), ptr::null_mut()) }, -1); + assert!(error_message().contains("Arrow output pointer was null")); +} + +#[test] +fn large_malformed_inputs_report_offsets_without_dumping_contents() { + let mut bytes = vec![0; 1024 * 1024 + 4]; + bytes[..4].copy_from_slice(&[0, 4, 0xff, 0xff]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + let file = TestFile::new("large-malformed", &bytes); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let scan_message = error_message(); + assert_eq!( + unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, + -1 + ); + let read_message = error_message(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + -1 + ); + let selected_message = error_message(); + for message in [scan_message, read_message, selected_message] { + assert!(message.contains("at byte 4"), "{message}"); + assert!(message.contains("Unexpected record"), "{message}"); + assert!(message.len() < 256); + } + assert!(array.is_released()); +} + +#[test] +fn selected_cell_errors_use_file_offsets_for_bounded_ranges() { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let cell_end = bytes.len(); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let range_end = bytes.len() as u64; + bytes.resize(bytes.len() + 1024, 0); + let ranges = [start, range_end]; + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!(error_message().contains(&format!( + "at byte {cell_end}: Cell byte range extends past ENDSTR" + ))); +} diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json new file mode 100644 index 0000000..2ef560a --- /dev/null +++ b/tests/golden/report-equal.json @@ -0,0 +1,78 @@ +{ + "cell_changes": [], + "comparison_profile": "pheon-structural-v1", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 0, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "equal", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-failed.json b/tests/golden/report-failed.json new file mode 100644 index 0000000..b4de07a --- /dev/null +++ b/tests/golden/report-failed.json @@ -0,0 +1,64 @@ +{ + "cell_changes": [], + "comparison_profile": "pheon-structural-v1", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": null, + "cell": null, + "code": "invalid_input", + "layer": null, + "message": "truncated record", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": null, + "units": null + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": null, + "units": null + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "failed", + "summary": null +} diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json new file mode 100644 index 0000000..c5c5fe7 --- /dev/null +++ b/tests/golden/report-geometry.json @@ -0,0 +1,360 @@ +{ + "cell_changes": [ + { + "added": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:new:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "438", + "side": "new" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "after": { + "attributes": [], + "byte_offset": "346" + }, + "attributes_changed": false, + "before": { + "attributes": [], + "byte_offset": "346" + }, + "name": "leaf", + "removed": [ + { + "bounds": [ + 0, + 0, + 10, + 10 + ], + "counts": null, + "fields": [ + { + "encoding": "uint16", + "name": "layer", + "payload_hex": "0x0001", + "tag": 3330, + "value": [ + 1 + ] + }, + { + "encoding": "uint16", + "name": "datatype", + "payload_hex": "0x0000", + "tag": 3586, + "value": [ + 0 + ] + }, + { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + } + ], + "id": "c0:old:0", + "kind": "BOUNDARY", + "layer": { + "number": 1, + "purpose": 0 + }, + "origin": null, + "properties": [], + "source": { + "byte_offset": "382", + "end_offset": "446", + "side": "old" + }, + "tag": 2048, + "target": null, + "width": null + } + ], + "replacements": [ + { + "added": "c0:new:0", + "explanation": "1/0: polygon vertices changed (4 -> 3 vertices)", + "field_changes": [ + { + "after": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "before": { + "encoding": "xy", + "name": "xy", + "payload_hex": "0x0000000000000000000000000000000a0000000a0000000a0000000a000000000000000000000000", + "tag": 4099, + "value": [ + [ + 0, + 0 + ], + [ + 0, + 10 + ], + [ + 10, + 10 + ], + [ + 10, + 0 + ], + [ + 0, + 0 + ] + ] + }, + "tag": 4099 + } + ], + "match_rule": "unique_kind_layer_bounds", + "properties_changed": false, + "removed": "c0:old:0" + } + ] + } + ], + "comparison_profile": "pheon-structural-v1", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": { + "edges": [ + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "left" + }, + { + "after_count": "1", + "before_count": "1", + "parent": "CHIP", + "reference_changed": false, + "target": "right" + }, + { + "after_count": "2", + "before_count": "2", + "parent": "left", + "reference_changed": false, + "target": "leaf" + }, + { + "after_count": "3", + "before_count": "3", + "parent": "right", + "reference_changed": false, + "target": "leaf" + } + ], + "nodes": [ + { + "after": "included", + "before": "included", + "cell": "CHIP", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "leaf", + "change": "direct" + }, + { + "after": "included", + "before": "included", + "cell": "left", + "change": "affected" + }, + { + "after": "included", + "before": "included", + "cell": "right", + "change": "affected" + } + ] + }, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "2", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "1", + "changed_cells": "1", + "counts": "exact", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "1" + } +} diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json new file mode 100644 index 0000000..b1871dd --- /dev/null +++ b/tests/golden/report-incomplete.json @@ -0,0 +1,105 @@ +{ + "cell_changes": [], + "comparison_profile": "pheon-structural-v1", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [ + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "old", + "target": null + }, + { + "byte_offset": "158", + "cell": "CHIP", + "code": "unsupported_record", + "layer": { + "number": 1, + "purpose": 0 + }, + "message": "cell \"CHIP\": at byte 158: unsupported record 0x7f02 in BOUNDARY", + "severity": "error", + "side": "new", + "target": null + } + ], + "exit_code": 2, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "unknown", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "incomplete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "observed", + "layer_name_changes": "0", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json new file mode 100644 index 0000000..2b219da --- /dev/null +++ b/tests/golden/report-metadata.json @@ -0,0 +1,87 @@ +{ + "cell_changes": [], + "comparison_profile": "pheon-structural-v1", + "coordinates": { + "bounds": "coordinate_envelope", + "frame": "cell_local", + "unit": "database_unit" + }, + "diagnostics": [], + "exit_code": 1, + "hierarchy": null, + "inputs": { + "new": { + "path": "after.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + }, + "old": { + "path": "before.gds", + "path_encoded_hex": null, + "present": true, + "units": { + "encoding": "real8", + "name": "UNITS", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", + "tag": 773, + "value": [ + 0.001, + 1e-9 + ] + } + } + }, + "kind": "diff", + "layer_legend": [], + "layer_name_changes": [ + { + "after": "CORE:WG", + "before": "WG", + "layer": { + "number": 1, + "purpose": 0 + } + } + ], + "library_changes": [], + "producer": { + "name": "pheon", + "version": "test" + }, + "report_selection": { + "cell": null, + "emitted_objects": "0", + "filtered_cells": "0", + "filtered_objects": "0", + "layer": null, + "limited_objects": "0", + "max_objects": null, + "truncated": false + }, + "result": "different", + "schema_version": 1, + "scope": { + "cell": null, + "layer": null, + "top": null + }, + "status": "complete", + "summary": { + "added_objects": "0", + "changed_cells": "0", + "counts": "exact", + "layer_name_changes": "1", + "library_field_changes": "0", + "removed_objects": "0" + } +} diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt new file mode 100644 index 0000000..8546367 --- /dev/null +++ b/tests/golden/review-canonical.txt @@ -0,0 +1,4 @@ +library "test" (GDSII 600, 0.001 user units/dbu, 1e-9 m/dbu) +1/0: name "WG" +cell "CHIP" + 1/0: rectangle (0, 0) to (10, 10) diff --git a/tests/golden/review-diff.txt b/tests/golden/review-diff.txt new file mode 100644 index 0000000..60cb697 --- /dev/null +++ b/tests/golden/review-diff.txt @@ -0,0 +1,4 @@ +Different +Cells: +* CHIP + 1/0: 1 polygon changed diff --git a/tests/golden/review-hierarchy.mmd b/tests/golden/review-hierarchy.mmd new file mode 100644 index 0000000..86cffd6 --- /dev/null +++ b/tests/golden/review-hierarchy.mmd @@ -0,0 +1,9 @@ +flowchart TD + n0["CHIP [affected]"] + n1["leaf [cell changed]"] + n2["left [affected]"] + n3["right [affected]"] + n0 -->|"x1"| n2 + n0 -->|"x1"| n3 + n2 -->|"x2"| n1 + n3 -->|"x3"| n1 diff --git a/tests/golden/review-hierarchy.txt b/tests/golden/review-hierarchy.txt new file mode 100644 index 0000000..60542a2 --- /dev/null +++ b/tests/golden/review-hierarchy.txt @@ -0,0 +1,6 @@ +Hierarchy +CHIP [affected] + left x1 [affected] + * leaf x2 + right x1 [affected] + * leaf x3 diff --git a/tests/numeric_packing.rs b/tests/numeric_packing.rs new file mode 100644 index 0000000..14a6109 --- /dev/null +++ b/tests/numeric_packing.rs @@ -0,0 +1,103 @@ +use pheon::*; +use std::{mem::size_of, slice}; + +// Read the packed prefix of each source slot, as the Python wrapper does. +fn packed_bytes(values: &[T], width: usize) -> Vec { + let bytes = unsafe { slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) }; + bytes + .chunks_exact(size_of::()) + .flat_map(|slot| slot[..width].iter().copied()) + .collect() +} + +macro_rules! check_int2 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 256 as $ty, 32767 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!(packed_bytes(&values, 2), [0, 0, 0, 1, 1, 0, 0x7f, 0xff]); + }}; +} + +macro_rules! check_int4 { + ($ty:ty, $function:ident) => {{ + let mut values = [0 as $ty, 1 as $ty, 65536 as $ty, 2147483520_i64 as $ty]; + assert_eq!( + unsafe { $function(values.as_mut_ptr(), values.len()) }, + 0 as $ty + ); + assert_eq!( + packed_bytes(&values, 4), + [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0x7f, 0xff, 0xff, 0x80] + ); + }}; +} + +#[test] +fn all_source_types_pack_to_big_endian_integer_bytes() { + check_int2!(u16, u16_to_i16); + check_int2!(u32, u32_to_i16); + check_int2!(u64, u64_to_i16); + check_int2!(i16, i16_to_i16); + check_int2!(i32, i32_to_i16); + check_int2!(i64, i64_to_i16); + check_int2!(f32, f32_to_i16); + check_int2!(f64, f64_to_i16); + check_int4!(u32, u32_to_i32); + check_int4!(u64, u64_to_i32); + check_int4!(i32, i32_to_i32); + check_int4!(i64, i64_to_i32); + check_int4!(f32, f32_to_i32); + check_int4!(f64, f64_to_i32); +} + +#[test] +fn signed_limits_and_unsigned_overflows_are_checked() { + let mut signed = [i64::from(i16::MIN), -1]; + assert_eq!(unsafe { i64_to_i16(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!(packed_bytes(&signed, 2), [0x80, 0, 0xff, 0xff]); + let mut signed = [i64::from(i32::MIN), i64::from(i32::MAX)]; + assert_eq!(unsafe { i64_to_i32(signed.as_mut_ptr(), signed.len()) }, 0); + assert_eq!( + packed_bytes(&signed, 4), + [0x80, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff] + ); + + for value in [32768_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i16(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } + for value in [2147483648_u64, u64::MAX] { + let mut input = [value]; + assert_eq!(unsafe { u64_to_i32(input.as_mut_ptr(), 1) }, value); + assert_eq!(input, [value]); + } +} + +#[test] +fn nonfinite_floats_and_rounded_up_f32_limit_are_rejected() { + for convert in [f32_to_i16, f32_to_i32] { + for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let mut input = [value]; + let result = unsafe { convert(input.as_mut_ptr(), 1) }; + assert_ne!(result, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + for convert in [f64_to_i16, f64_to_i32] { + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut input = [value]; + assert_ne!(unsafe { convert(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(input[0].to_bits(), value.to_bits()); + } + } + let mut input = [2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 2147483648_f32); + assert_eq!(input, [2147483648_f32]); + let mut input = [-2147483648_f32]; + assert_eq!(unsafe { f32_to_i32(input.as_mut_ptr(), 1) }, 0.0); + assert_eq!(packed_bytes(&input, 4), [0x80, 0, 0, 0]); +} diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs new file mode 100644 index 0000000..be3ede4 --- /dev/null +++ b/tests/parser_errors.rs @@ -0,0 +1,287 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use pheon::{ + basic::{ErrType, encode_real8}, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_bytes, read_cells_bytes, + record::RecordHeader, + records, + review::{Comparison, Document, dump_records}, + scan_bytes, +}; +use std::ffi::CStr; + +fn fixture() -> (Vec, (u64, u64)) { + let mut bytes = common::library_header(); + let start = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_SREF, &[]); + common::record(&mut bytes, records::RTAG_SNAME, "töp".as_bytes()); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + common::record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_TEXT, &[]); + common::record(&mut bytes, records::RTAG_LAYER, &[0; 2]); + common::record(&mut bytes, records::RTAG_TEXTTYPE, &[0; 2]); + common::record(&mut bytes, records::RTAG_XY, &[0; 8]); + common::record(&mut bytes, records::RTAG_STRING, b"label\0"); + common::record(&mut bytes, records::RTAG_ENDEL, &[]); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + let end = bytes.len() as u64; + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + (bytes, (start, end)) +} + +fn payload_range(bytes: &[u8], tag: u16) -> std::ops::Range { + let mut offset = 0; + loop { + let (_, header) = RecordHeader::read(&bytes[offset..]).unwrap(); + let start = offset + 4; + let end = start + usize::from(header.data_size); + if header.tag == tag { + return start..end; + } + offset = end; + } +} + +fn check_ffi_error( + expected: &str, + call: impl FnOnce(&mut FFI_ArrowArray, &mut FFI_ArrowSchema) -> i32, +) { + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!(call(&mut array, &mut schema), -1); + let pointer = last_error_message(); + assert!(!pointer.is_null()); + let error = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); + assert!(error.contains(expected), "{error}"); + assert!( + error.len() < 256, + "error should not dump the input: {error}" + ); + assert!(array.is_released()); +} + +fn check_read_errors(bytes: &[u8], range: (u64, u64), offset: usize, message: &str, scan: bool) { + let ranges = [range]; + let mut results = vec![read_library(bytes), read_selected_library(bytes, &ranges)]; + if scan { + results.push(scan_library(bytes)); + } else { + // Hierarchy scans deliberately skip text geometry and its string payloads. + assert!(scan_library(bytes).is_ok()); + } + for result in results { + let (remaining, error) = result.unwrap_err(); + assert_eq!(remaining.as_ptr(), bytes[offset..].as_ptr()); + assert!(matches!(error, ErrType::Failed(reason) if reason.starts_with(message))); + } + let expected = format!("at byte {offset}: {message}"); + check_ffi_error(&expected, |a, s| unsafe { + read_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + let offsets = [range.0, range.1]; + check_ffi_error(&expected, |a, s| unsafe { + read_cells_bytes(bytes.as_ptr(), bytes.len(), offsets.as_ptr(), 1, a, s) + }); + if scan { + check_ffi_error(&expected, |a, s| unsafe { + scan_bytes(bytes.as_ptr(), bytes.len(), a, s) + }); + } +} + +#[test] +fn invalid_utf8_returns_located_parse_errors_in_each_reader() { + let (bytes, range) = fixture(); + assert!(read_library(&bytes).is_ok()); + assert!(read_selected_library(&bytes, &[range]).is_ok()); + assert!(scan_library(&bytes).is_ok()); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + records::RTAG_STRING, + records::RTAG_PROPVALUE, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0xff; + // UTF-8 is checked at the current parser cursor, after the record/header. + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + check_read_errors( + &malformed, + range, + offset, + "Invalid UTF-8 string:", + tag != records::RTAG_STRING, + ); + } +} + +#[test] +fn extra_name_padding_is_ignored_by_comparison_and_arrow_readers() { + let (mut bytes, (start, end)) = fixture(); + let target = payload_range(&bytes, records::RTAG_SNAME); + bytes[target].copy_from_slice(b"leaf"); + bytes.truncate(end as usize); + common::record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + common::record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + common::record(&mut bytes, records::RTAG_ENDSTR, &[]); + common::record(&mut bytes, records::RTAG_ENDLIB, &[]); + let mut padded = bytes.clone(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let payload = payload_range(&padded, tag); + let mut data = padded[payload.clone()].to_vec(); + data.extend_from_slice(&[0; 2]); + let mut record = Vec::new(); + common::record(&mut record, tag, &data); + padded.splice(payload.start - 4..payload.end, record); + } + let baseline = Document::parse(&bytes).unwrap(); + let document = Document::parse(&padded).unwrap(); + assert_eq!(document.cells["töp"].objects.len(), 2); + assert!( + Comparison::between(&baseline, &document) + .outcome() + .is_equal() + ); + + let full = read_library(&padded).unwrap().1; + let scan = scan_library(&padded).unwrap().1; + let ranges = [(start + 2, end + 6), (end + 6, padded.len() as u64 - 4)]; + let selected = read_selected_library(&padded, &ranges).unwrap().1; + let expected = read_library(&bytes).unwrap().1; + for actual in [&full, &scan, &selected] { + for field in ["lib_name", "cell_names"] { + assert_eq!(actual.column_by_name(field), expected.column_by_name(field)); + } + } + assert_eq!(&full, &selected); +} + +#[test] +fn interior_nuls_in_names_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + for tag in [ + records::RTAG_LIBNAME, + records::RTAG_STRNAME, + records::RTAG_SNAME, + ] { + let mut malformed = bytes.clone(); + let payload = payload_range(&malformed, tag); + malformed[payload.start] = 0; + let offset = if tag == records::RTAG_LIBNAME { + payload_range(&malformed, records::RTAG_UNITS).end + } else { + payload.end + }; + let message = "name contains an embedded NUL"; + check_read_errors(&malformed, range, offset, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", payload.start - 4) + ); + } +} + +#[test] +fn malformed_record_frames_are_rejected_at_the_same_offset() { + let (bytes, range) = fixture(); + // STRING also exercises framing in the Arrow scan's geometry skip path. + for tag in [ + records::RTAG_STRNAME, + records::RTAG_XY, + records::RTAG_STRING, + ] { + let start = payload_range(&bytes, tag).start - 4; + for (size, message) in [ + (2_u16, "Record size 2 is smaller than its header"), + (5, "odd record length"), + (u16::MAX - 1, "truncated record"), + ] { + let mut malformed = bytes.clone(); + malformed[start..start + 2].copy_from_slice(&size.to_be_bytes()); + check_read_errors(&malformed, range, start, message, true); + assert_eq!( + dump_records(&malformed).unwrap_err(), + format!("at byte {start}: {message}") + ); + } + } +} + +#[test] +fn nonpositive_unit_scales_are_rejected_by_comparison_and_arrow_readers() { + let (bytes, range) = fixture(); + let units = payload_range(&bytes, records::RTAG_UNITS); + let message = "UNITS must be positive finite values"; + for field in [units.start, units.start + 8] { + for scale in [0.0, -1.0] { + let mut malformed = bytes.clone(); + malformed[field..field + 8] + .copy_from_slice(&encode_real8(scale).unwrap().to_be_bytes()); + check_read_errors(&malformed, range, units.start, message, true); + assert_eq!( + Document::parse(&malformed).unwrap_err(), + format!("at byte {}: {message}", units.start - 4) + ); + } + } +} + +#[test] +fn coordinate_payloads_require_whole_pairs() { + let (mut bytes, (start, end)) = fixture(); + let after_text = payload_range(&bytes, records::RTAG_TEXT).end; + let xy = payload_range(&bytes[after_text..], records::RTAG_XY); + let payload_start = after_text + xy.start; + let mut replacement = Vec::new(); + common::record(&mut replacement, records::RTAG_XY, &[0; 12]); + bytes.splice(payload_start - 4..after_text + xy.end, replacement); + + let message = "XY payload must contain whole coordinate pairs"; + check_read_errors(&bytes, (start, end + 4), payload_start, message, false); + assert!(Document::parse(&bytes).unwrap_err().contains(message)); +} + +#[test] +fn large_invalid_utf8_strings_do_not_dump_their_contents() { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, records::RTAG_PROPVALUE); + let mut replacement = Vec::new(); + let mut value = vec![b'a'; 60_000]; + value[30_000] = 0xff; + common::record(&mut replacement, records::RTAG_PROPVALUE, &value); + let added = replacement.len() - (payload.end - payload.start + 4); + bytes.splice(payload.start - 4..payload.end, replacement); + let range = (range.0, range.1 + added as u64); + let offset = payload_range(&bytes, records::RTAG_PROPVALUE).end; + check_read_errors(&bytes, range, offset, "Invalid UTF-8 string:", true); +} + +#[test] +fn overflowing_library_and_cell_years_return_the_same_error_in_each_reader() { + for tag in [records::RTAG_BGNLIB, records::RTAG_BGNSTR] { + for timestamp in [0, 1] { + let (mut bytes, range) = fixture(); + let payload = payload_range(&bytes, tag); + let offset = payload.start + timestamp * 12; + bytes[offset..offset + 2].copy_from_slice(&i16::MAX.to_be_bytes()); + check_read_errors(&bytes, range, offset, "Timestamp year", true); + } + } +} diff --git a/tests/python/test_arrow.py b/tests/python/test_arrow.py new file mode 100644 index 0000000..64db3f2 --- /dev/null +++ b/tests/python/test_arrow.py @@ -0,0 +1,141 @@ +"""Public Arrow readers, lifetime management and input validation.""" + +import gc +import gzip +import io +import mmap +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy + +from pheon import PheonError, arrow +from pheon._native import ffi +from test_ffi import layout + + +class ArrowTests(unittest.TestCase): + def test_full_scan_and_selected_reads_own_their_data(self): + data = bytearray(layout()) + full = arrow.read_bytes(data) + scan = arrow.scan_bytes(data) + cells = scan.field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + ) + ) + selected = arrow.read_cells_bytes(data, ranges) + data[:] = b"\0" * len(data) + del data + gc.collect() + self.assertEqual(full.to_pylist(), selected.to_pylist()) + self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"]) + self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"]) + + def test_paths_gzip_magic_and_closed_mmap(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "layout with spaces.gds" + path.write_bytes(layout()) + expected = arrow.read_path(path).to_pylist() + with path.open("rb") as stream: + with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer: + result = arrow.read_bytes(buffer) + self.assertEqual(result.to_pylist(), expected) + path.write_bytes(gzip.compress(layout())) + self.assertEqual(arrow.read_path(path).to_pylist(), expected) + + def test_path_reader_keeps_using_the_opened_stream(self): + for payload in (layout(), gzip.compress(layout())): + with self.subTest(compressed=payload.startswith(b"\x1f\x8b")): + source = io.BytesIO(payload) + with patch("builtins.open", return_value=source) as opened: + result = arrow.read_path("opened-stream-only.gds") + opened.assert_called_once_with("opened-stream-only.gds", "rb") + self.assertTrue(source.closed) + self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"]) + + def test_empty_and_noncontiguous_cell_selections(self): + data = layout() + empty = arrow.read_cells_bytes(data, []) + self.assertEqual(empty.field("cells")[0].as_py(), []) + values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2] + self.assertFalse(values.flags.c_contiguous) + self.assertEqual( + arrow.read_cells_bytes(data, values).to_pylist(), + arrow.read_bytes(data).to_pylist(), + ) + + def test_bad_inputs_fail_before_importing_arrow(self): + with self.assertRaises(PheonError): + arrow.read_bytes(b"invalid") + with self.assertRaises(PheonError): + arrow.scan_bytes(b"") + with self.assertRaises(PheonError): + arrow.read_cells_bytes(layout(), [[0, 10]]) + with self.assertRaises(ValueError): + arrow.read_bytes(memoryview(layout())[::2]) + for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]: + with self.assertRaises(ValueError): + arrow.read_cells_bytes(layout(), ranges) + with self.assertRaises(TypeError): + arrow.read_cells_bytes(layout(), [[1.5, 2.5]]) + + def test_malformed_strings_and_years_raise_located_parse_errors(self): + data = layout() + cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0] + start = cell["struct_start_offset"] + ranges = [[start, cell["struct_end_offset"]]] + cases = [ + (38, b"\xff", "Invalid UTF-8"), + (start + 32, b"\xff", "Invalid UTF-8"), + ] + cases += [ + (offset, b"\x7f\xff", "Timestamp year") + for offset in (10, 22, start + 4, start + 16) + ] + for offset, value, message in cases: + malformed = bytearray(data) + malformed[offset:offset + len(value)] = value + for read in ( + arrow.read_bytes, + arrow.scan_bytes, + lambda payload: arrow.read_cells_bytes(payload, ranges), + ): + with self.subTest(offset=offset, reader=read): + with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"): + read(malformed) + + def test_import_failure_releases_both_native_outputs(self): + releases = [] + + @ffi.callback("void(struct ArrowArray *)") + def release_array(value): + releases.append("array") + value.release = ffi.NULL + + @ffi.callback("void(struct ArrowSchema *)") + def release_schema(value): + releases.append("schema") + value.release = ffi.NULL + + def export(array, schema): + array.release = release_array + schema.release = release_schema + return 0 + + def fail_import(*args): + raise ValueError("import failed") + + with patch.object( + arrow, + "pyarrow", + SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)), + ): + with self.assertRaisesRegex(ValueError, "import failed"): + arrow._export(export) + self.assertEqual(releases, ["array", "schema"]) diff --git a/tests/python/test_ffi.py b/tests/python/test_ffi.py new file mode 100644 index 0000000..4dba750 --- /dev/null +++ b/tests/python/test_ffi.py @@ -0,0 +1,70 @@ +"""Exercise the explicit C ABI declarations against the actual shared library.""" +import struct +import tempfile +from pathlib import Path +import unittest + +from pheon._native import ffi, lib + + +def record(tag, data=b""): + return struct.pack(">HH", len(data) + 4, tag) + data + + +def layout(): + return b"".join([ + record(0x0002, struct.pack(">h", 600)), + record(0x0102, b"\0" * 24), + record(0x0206, b"test"), + record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")), + record(0x0502, b"\0" * 24), + record(0x0606, b"cell"), + record(0x0700), + record(0x0400), + ]) + + +class FfiTests(unittest.TestCase): + def check_export(self, call): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + self.assertEqual(call(array, schema), 0) + self.assertEqual(lib.last_error_message(), ffi.NULL) + self.assertEqual(array.length, 1) + self.assertEqual(ffi.string(schema.format), b"+s") + self.assertNotEqual(array.release, ffi.NULL) + self.assertNotEqual(schema.release, ffi.NULL) + self.assertGreaterEqual(array.n_children, 7) + self.assertEqual(array.n_children, schema.n_children) + self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit") + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + def test_scan_selected_read_and_path_exports(self): + data = layout() + self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s)) + self.check_export(lambda a, s: lib.read_bytes(data, len(data), a, s)) + # The library header is 62 bytes; the one cell ends just before ENDLIB. + ranges = ffi.new("uint64_t[]", [62, len(data) - 4]) + self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "file with spaces.gds" + path.write_bytes(data) + self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s)) + + def test_failures_leave_outputs_empty_and_expose_error_message(self): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1) + self.assertNotEqual(lib.last_error_message(), ffi.NULL) + self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message())) + self.assertEqual(array.release, ffi.NULL) + self.assertEqual(schema.release, ffi.NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_loader.py b/tests/python/test_loader.py new file mode 100644 index 0000000..4724495 --- /dev/null +++ b/tests/python/test_loader.py @@ -0,0 +1,36 @@ +"""Reject incompatible native libraries before exposing callable functions.""" + +from pathlib import Path +import runpy +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import pheon._native + + +class LoaderTests(unittest.TestCase): + def load(self, library): + with patch("cffi.FFI.dlopen", return_value=library): + return runpy.run_path(str(Path(pheon._native.__file__))) + + def test_matching_versions_are_accepted(self): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: 1, pheon_arrow_schema_version=lambda: 1 + ) + ) + + def test_incompatible_abi_or_schema_is_rejected(self): + for abi, schema in [(2, 1), (1, 2)]: + with self.assertRaisesRegex(ImportError, "Incompatible Pheon library"): + self.load( + SimpleNamespace( + pheon_abi_version=lambda: abi, + pheon_arrow_schema_version=lambda: schema, + ) + ) + + def test_unversioned_library_is_rejected(self): + with self.assertRaisesRegex(ImportError, "version API"): + self.load(SimpleNamespace()) diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py new file mode 100644 index 0000000..75ce0cd --- /dev/null +++ b/tests/python/test_packing.py @@ -0,0 +1,67 @@ +import unittest + +import numpy as np + +from pheon import pack_int2, pack_int4 + + +class PackingTests(unittest.TestCase): + def test_scalars_lists_and_dimensions(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]: + for shape in [(), (1,), (2, 2), (2, 2, 2)]: + count = int(np.prod(shape)) if shape else 1 + data = np.arange(count, dtype=dtype).reshape(shape) + before = data.copy() + with self.subTest(width=width, dtype=dtype, shape=shape): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + np.testing.assert_array_equal(data, before) + for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]: + with self.subTest(width=width, data=data): + self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes()) + + def test_noncontiguous_readonly_and_endian_inputs(self): + original = np.arange(48, dtype=np.int64).reshape(6, 8) + readonly = original.copy() + readonly.setflags(write=False) + for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly, + original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]: + before = data.copy() + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + with self.subTest(width=width, strides=data.strides, dtype=data.dtype): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C")) + np.testing.assert_array_equal(data, before) + + def test_signed_limits_and_native_integer_widths(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1 + data = np.array([[low, -1], [0, high]], dtype=np.int64) + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + for dtype in [np.int8, np.uint8, np.int16, np.uint16]: + data = np.array([[1, 2], [3, 4]], dtype=dtype) + self.assertEqual(pack_int2(data), data.astype(">i2").tobytes()) + self.assertEqual(pack_int4(data), data.astype(">i4").tobytes()) + + def test_failed_conversions_preserve_caller_data(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + limit = 1 << (8 * width - 1) + for dtype in [np.dtype("i8"), np.dtype("i8").newbyteorder("S")]: + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, dtype=dtype, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + for dtype in [np.float32, np.float64]: + for invalid in [np.nan, np.inf, -np.inf]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, invalid=invalid, dtype=dtype): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/real8.rs b/tests/real8.rs new file mode 100644 index 0000000..fc632a1 --- /dev/null +++ b/tests/real8.rs @@ -0,0 +1,75 @@ +use pheon::basic::{decode_real8, encode_real8, pack_real8}; + +#[test] +fn normalized_real8_has_known_encodings_and_exact_values() { + for (value, expected) in [ + (0.0, 0), + (-0.0, 0), + (1.0, 0x4110_0000_0000_0000), + (-2.0, 0xc120_0000_0000_0000), + (0.5, 0x4080_0000_0000_0000), + (0.001, 0x3e41_8937_4bc6_a7f0), + (1e-9, 0x3944_b82f_a09b_5a54), + (2_f64.powi(-260), 0x0010_0000_0000_0000), + ] { + let bits = encode_real8(value).unwrap(); + assert_eq!(bits, expected, "{value:e}"); + assert_eq!(decode_real8(bits), value); + } +} + +#[test] +fn real8_round_trips_each_supported_binary_exponent_and_mantissa_boundary() { + for exponent in -260..=251_i32 { + for mantissa in [ + 0, + 1, + 0x0008_0000_0000_0000, + 0x000f_ffff_ffff_fffe, + 0x000f_ffff_ffff_ffff, + ] { + for sign in [0, 1_u64 << 63] { + let ieee = sign | (((exponent + 1023) as u64) << 52) | mantissa; + let value = f64::from_bits(ieee); + let encoded = encode_real8(value).unwrap(); + assert_ne!(encoded & 0x00f0_0000_0000_0000, 0, "{value:e}"); + assert_eq!(decode_real8(encoded).to_bits(), ieee, "{value:e}"); + } + } + } +} + +#[test] +fn real8_underflow_is_canonical_and_overflow_is_an_error() { + let minimum = 2_f64.powi(-260); + let maximum_exclusive = 2_f64.powi(252); + for magnitude in [ + 0.0, + f64::from_bits(1), + f64::MIN_POSITIVE, + 1e-90, + 1e-80, + f64::from_bits(minimum.to_bits() - 1), + ] { + for value in [magnitude, -magnitude] { + assert_eq!(encode_real8(value).unwrap(), 0, "{value:e}"); + } + } + let largest = f64::from_bits(maximum_exclusive.to_bits() - 1); + assert_eq!(encode_real8(largest).unwrap(), 0x7fff_ffff_ffff_fff8); + assert_eq!(decode_real8(encode_real8(largest).unwrap()), largest); + for value in [ + maximum_exclusive, + -maximum_exclusive, + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] { + assert!(encode_real8(value).is_err(), "{value}"); + let mut buffer = [0x55; 8]; + assert!(pack_real8(&mut buffer, value).is_err()); + assert_eq!(buffer, [0x55; 8]); + } +} diff --git a/tests/record_validation.rs b/tests/record_validation.rs new file mode 100644 index 0000000..5fd55d3 --- /dev/null +++ b/tests/record_validation.rs @@ -0,0 +1,122 @@ +mod common; + +use pheon::{ + basic::{ErrType, parse_datetime}, + record::{ + ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair, + Record, RecordData, + }, + records, +}; + +#[test] +fn fixed_record_sizes_are_checked_by_every_read_entry_point() { + let payload = [0; 4]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_HEADER, &payload); + let mut prefixed = Vec::new(); + common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip"); + prefixed.extend_from_slice(&bytes); + + for result in [ + records::HEADER::read(&bytes), + records::HEADER::skip_and_read(&prefixed), + records::HEADER::read_data(&payload, 4), + ] { + let (remaining, reason) = result.unwrap_err(); + assert_eq!(remaining, payload); + assert!(matches!(reason, ErrType::Failed(message) + if message == "Expected record size 2, got 4")); + } + + // The underlying integer-array payload is aligned, but COLROW needs two values. + let payload = [0; 2]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_COLROW, &payload); + assert!(records::COLROW::read(&bytes).is_err()); + assert!(records::COLROW::skip_and_read(&bytes).is_err()); + assert!(records::COLROW::read_data(&payload, 2).is_err()); +} + +#[test] +fn timestamp_year_conversion_checks_the_int16_boundaries() { + for stored_year in [i16::MIN, -1900, 0, i16::MAX - 1900] { + let mut payload = [0; 12]; + payload[..2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, date) = parse_datetime(&payload).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(i32::from(date[0]), i32::from(stored_year) + 1900); + } + for stored_year in [i16::MAX - 1899, i16::MAX] { + for timestamp in [0, 1] { + let mut payload = [0; 24]; + let offset = timestamp * 12; + payload[offset..offset + 2].copy_from_slice(&stored_year.to_be_bytes()); + let (remaining, error) = DateTimePair::read(&payload, 24).unwrap_err(); + assert_eq!(remaining.as_ptr(), payload[offset..].as_ptr()); + assert!( + matches!(error, ErrType::Failed(message) if message.contains("Timestamp year")) + ); + } + } +} + +#[test] +fn metadata_size_constraints_apply_to_every_read_entry_point() { + macro_rules! check { + ($record:ident, $tag:ident) => {{ + let payload = [b'x'; 44]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + let mut prefixed = common::library_header(); + prefixed.extend_from_slice(&bytes); + assert_eq!(records::$record::read(&bytes).unwrap().1, payload); + assert_eq!( + records::$record::skip_and_read(&prefixed).unwrap().1, + payload + ); + assert_eq!( + records::$record::read_data(&payload, 44).unwrap().1, + payload + ); + + let payload = [b'x'; 46]; + let mut bytes = Vec::new(); + common::record(&mut bytes, records::$tag, &payload); + assert!(records::$record::read(&bytes).is_err()); + assert!(records::$record::skip_and_read(&bytes).is_err()); + assert!(records::$record::read_data(&payload, 46).is_err()); + }}; + } + + check!(REFLIBS, RTAG_REFLIBS); + check!(FONTS, RTAG_FONTS); + check!(ATTRTABLE, RTAG_ATTRTABLE); +} + +#[test] +fn malformed_payload_sizes_return_errors_without_panicking() { + let payload = [0; 32]; + assert!(BitArray::read(&payload, 4).is_err()); + assert!(Int2::read(&payload, 4).is_err()); + assert!(Int4::read(&payload, 2).is_err()); + assert!(Real8::read(&payload, 4).is_err()); + assert!(Real8Pair::read(&payload, 8).is_err()); + assert!(DateTimePair::read(&payload, 12).is_err()); + assert!(Empty::read(&payload, 2).is_err()); + assert!(Int2Array::read(&payload, 3).is_err()); + assert!(Int4Array::read(&payload, 6).is_err()); + + let mut bytes = Vec::new(); + common::record(&mut bytes, records::RTAG_XY, &payload[..6]); + assert!(records::XY::read(&bytes).is_err()); + assert!(records::XY::skip_and_read(&bytes).is_err()); + assert!(records::XY::read_data(&payload, 6).is_err()); + + // Valid sizes still reject a truncated buffer; an empty string remains valid. + assert!(Int4::read(&payload[..2], 4).is_err()); + assert!(Real8Pair::read(&payload[..8], 16).is_err()); + assert!(DateTimePair::read(&payload[..12], 24).is_err()); + assert!(Int4Array::read(&payload[..4], 8).is_err()); + assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::::new())); +} diff --git a/tests/review.rs b/tests/review.rs new file mode 100644 index 0000000..ad625bb --- /dev/null +++ b/tests/review.rs @@ -0,0 +1,2526 @@ +mod common; +#[path = "review/reports.rs"] +mod reports; + +use common::{library_header, record}; +use pheon::{ + records::*, + review::{ + Comparison, Document, TextComparison, TextReportOptions, names::LAYER_SENTINEL, + selection::DisplayFilter, + }, +}; +use std::{fs, path::PathBuf, process::Command}; + +fn ascii(bytes: &mut Vec, tag: u16, value: &str) { + let mut value = value.as_bytes().to_vec(); + if value.len() % 2 == 1 { + value.push(0); + } + record(bytes, tag, &value); +} + +fn xy(bytes: &mut Vec, points: &[[i32; 2]]) { + record( + bytes, + RTAG_XY, + &points + .iter() + .flatten() + .flat_map(|v| v.to_be_bytes()) + .collect::>(), + ); +} + +fn layout(cells: &[(&str, Vec>)]) -> Vec { + let mut bytes = library_header(); + for (name, objects) in cells { + record(&mut bytes, RTAG_BGNSTR, &[0; 24]); + ascii(&mut bytes, RTAG_STRNAME, name); + for object in objects { + bytes.extend_from_slice(object); + } + record(&mut bytes, RTAG_ENDSTR, &[]); + } + record(&mut bytes, RTAG_ENDLIB, &[]); + bytes +} + +fn gzip(data: &[u8]) -> Vec { + use std::io::Write; + + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +fn polygon(points: &[[i32; 2]]) -> Vec { + ring(RTAG_BOUNDARY, RTAG_DATATYPE, 1, points) +} + +fn ring(kind: u16, type_tag: u16, layer: u16, points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, kind, &[]); + record(&mut bytes, RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, type_tag, &0_u16.to_be_bytes()); + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn square() -> Vec { + polygon(&[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]) +} + +fn triangle() -> Vec { + polygon(&[[0, 0], [10, 0], [0, 10], [0, 0]]) +} + +fn reference(target: &str, x: i32, count: i16) -> Vec { + let mut bytes = Vec::new(); + record( + &mut bytes, + if count == 1 { RTAG_SREF } else { RTAG_AREF }, + &[], + ); + ascii(&mut bytes, RTAG_SNAME, target); + if count == 1 { + xy(&mut bytes, &[[x, 0]]); + } else { + let counts: Vec<_> = [count, 1].into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, RTAG_COLROW, &counts); + xy(&mut bytes, &[[x, 0], [x + 100, 0], [x, 0]]); + } + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn label(carrier: (u16, u16), text: &str) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_TEXT, &[]); + record(&mut bytes, RTAG_LAYER, &carrier.0.to_be_bytes()); + record(&mut bytes, RTAG_TEXTTYPE, &carrier.1.to_be_bytes()); + xy(&mut bytes, &[[0, 0]]); + ascii(&mut bytes, RTAG_STRING, text); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn path(width: Option, points: &[[i32; 2]]) -> Vec { + let mut bytes = Vec::new(); + record(&mut bytes, RTAG_PATH, &[]); + record(&mut bytes, RTAG_LAYER, &[0, 1]); + record(&mut bytes, RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, RTAG_WIDTH, &width.to_be_bytes()); + } + xy(&mut bytes, points); + record(&mut bytes, RTAG_ENDEL, &[]); + bytes +} + +fn with_property(mut object: Vec, value: &str) -> Vec { + object.truncate(object.len() - 4); + record(&mut object, RTAG_PROPATTR, &17_u16.to_be_bytes()); + ascii(&mut object, RTAG_PROPVALUE, value); + record(&mut object, RTAG_ENDEL, &[]); + object +} + +fn shared(geometry: Vec) -> Vec { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 20, 1)], + ), + ("left", vec![reference("leaf", 0, 2)]), + ("right", vec![reference("leaf", 0, 3)]), + ("leaf", vec![geometry]), + ]) +} + +struct Pair { + name: &'static str, + old: Vec, + new: Vec, + changed_cells: usize, + removed: usize, + added: usize, + metadata: usize, +} + +// Ten deliberately small fixture pairs specify the first object profile. +fn corpus() -> Vec { + let base = layout(&[("CHIP", vec![square(), label((2, 0), "port")])]); + let mut timestamps = base.clone(); + timestamps[10] = 1; // BGNLIB payload, ignored by the object profile. + let meta = |name| { + layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), name), + ], + )]) + }; + vec![ + Pair { + name: "record reordering", + old: base.clone(), + new: layout(&[("CHIP", vec![label((2, 0), "port"), square()])]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "timestamp edit", + old: base, + new: timestamps, + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "polygon start and winding", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[( + "CHIP", + vec![polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])], + )]), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 0, + }, + Pair { + name: "duplicate shape", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![square(), square()])]), + changed_cells: 1, + removed: 0, + added: 1, + metadata: 0, + }, + Pair { + name: "same bounds different geometry", + old: layout(&[("CHIP", vec![square()])]), + new: layout(&[("CHIP", vec![triangle()])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "moved reference", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 5, 1)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "array count", + old: layout(&[ + ("CHIP", vec![reference("leaf", 0, 2)]), + ("leaf", vec![square()]), + ]), + new: layout(&[ + ("CHIP", vec![reference("leaf", 0, 3)]), + ("leaf", vec![square()]), + ]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "element property", + old: layout(&[("CHIP", vec![with_property(square(), "old")])]), + new: layout(&[("CHIP", vec![with_property(square(), "new")])]), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + Pair { + name: "layer name only", + old: meta("1:0:WG"), + new: meta("1:0:CORE:WG"), + changed_cells: 0, + removed: 0, + added: 0, + metadata: 1, + }, + Pair { + name: "shared changed descendant", + old: shared(square()), + new: shared(triangle()), + changed_cells: 1, + removed: 1, + added: 1, + metadata: 0, + }, + ] +} + +#[test] +fn ten_fixture_pairs_have_explicit_expected_changes() { + for pair in corpus() { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert!( + diff.diagnostics.is_empty(), + "{}: {:?}", + pair.name, + diff.diagnostics + ); + assert_eq!(diff.cells.len(), pair.changed_cells, "{}", pair.name); + assert_eq!( + diff.cells.iter().map(|c| c.removed.len()).sum::(), + pair.removed, + "{}", + pair.name + ); + assert_eq!( + diff.cells.iter().map(|c| c.added.len()).sum::(), + pair.added, + "{}", + pair.name + ); + assert_eq!(diff.layer_names.len(), pair.metadata, "{}", pair.name); + let equal = pair.changed_cells + pair.metadata == 0; + assert_eq!(diff.outcome().is_equal(), equal, "{}", pair.name); + assert_eq!( + old.canonical_dump() == new.canonical_dump(), + equal, + "{}", + pair.name + ); + assert_eq!(diff.outcome().exit_code(), if equal { 0 } else { 1 }); + } +} + +#[test] +fn review_identifies_gzip_and_oasis_input() { + for (bytes, expected) in [ + (&b"\x1f\x8b\x08\x00"[..], "gzip-compressed input"), + (&b"%SEMI-OASIS\r\n"[..], "OASIS input"), + ] { + let error = Document::parse(bytes).unwrap_err(); + assert!( + error.starts_with(&format!("at byte 0: {expected}")), + "{error}" + ); + assert_eq!(pheon::review::dump_records(bytes).unwrap_err(), error); + } +} + +#[test] +fn canonical_polygon_dump_is_readable_and_offsets_are_separate() { + let bytes = layout(&[("CHIP", vec![square()])]); + let doc = Document::parse(&bytes).unwrap(); + let dump = doc.canonical_dump(); + assert!( + dump.ends_with("cell \"CHIP\"\n 1/0: rectangle (0, 0) to (10, 10)\n"), + "{dump}" + ); + assert!(!dump.contains("BGNSTR")); + assert!( + doc.record_dump() + .contains("0000000000 0002 HEADER size=6 [600]") + ); + assert!(doc.record_dump().contains("BGNSTR")); + let notched = polygon(&[[0, 0], [0, 3], [2, 3], [2, 2], [3, 2], [3, 0], [0, 0]]); + let bytes = layout(&[("CHIP", vec![notched])]); + let dump = Document::parse(&bytes).unwrap().canonical_dump(); + assert!( + dump.ends_with(concat!( + " 1/0: polygon xy=[(0, 0), (0, 3), (2, 3), (2, 2),\n", + " (3, 2), (3, 0), (0, 0)]\n", + )), + "{dump}" + ); +} + +#[test] +fn canonical_dump_counts_duplicates_by_normalized_object_within_each_cell() { + let test_layout = |copies: usize| { + let mut objects = vec![square(); copies - 1]; + objects.push(polygon(&[[10, 10], [10, 0], [0, 0], [0, 10], [10, 10]])); + objects.push(with_property(square(), "marked")); + layout(&[("A", objects), ("B", vec![square()])]) + }; + let a = test_layout(3); + let b = test_layout(2); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let dump = a.canonical_dump(); + assert!( + dump.ends_with(concat!( + "cell \"A\"\n", + " 3x 1/0: rectangle (0, 0) to (10, 10)\n", + " 1/0: rectangle (0, 0) to (10, 10) property[17]=\"marked\"\n", + "cell \"B\"\n", + " 1/0: rectangle (0, 0) to (10, 10)\n", + )), + "{dump}" + ); + assert!(b.canonical_dump().contains(" 2x 1/0: rectangle")); + assert_ne!(dump, b.canonical_dump()); + assert_eq!(Comparison::between(&a, &b).outcome().exit_code(), 1); +} + +#[test] +fn canonical_dump_and_hierarchy_match_versioned_goldens() { + let bytes = layout(&[( + "CHIP", + vec![ + square(), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + )]); + assert_eq!( + Document::parse(&bytes).unwrap().canonical_dump(), + include_str!("golden/review-canonical.txt") + ); + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + diff.hierarchy(false, 8), + include_str!("golden/review-hierarchy.txt") + ); + assert_eq!( + diff.hierarchy(true, 8), + include_str!("golden/review-hierarchy.mmd") + ); +} + +#[test] +fn misplaced_library_fields_and_nul_names_are_rejected() { + let mut late = layout(&[("CHIP", vec![])]); + late.truncate(late.len() - 4); + ascii(&mut late, RTAG_LIBNAME, "late"); + record(&mut late, RTAG_ENDLIB, &[]); + assert!( + Document::parse(&late) + .unwrap_err() + .contains("library field outside") + ); + assert!( + Document::parse(&layout(&[("CH\0IP", vec![])])) + .unwrap_err() + .contains("NUL") + ); +} + +#[test] +fn cell_and_property_order_do_not_affect_comparison() { + let a = with_property(with_property(square(), "a"), "b"); + let b = with_property(with_property(square(), "b"), "a"); + let a = layout(&[("z", vec![a]), ("a", vec![])]); + let b = layout(&[("a", vec![]), ("z", vec![b])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + assert_ne!(a.cells["z"].offset, b.cells["z"].offset); + assert!(Comparison::between(&a, &b).outcome().is_equal()); + assert_eq!(a.canonical_dump(), b.canonical_dump()); +} + +#[test] +fn metadata_requires_a_sentinel_in_the_same_cell_and_carrier_pair() { + let bytes = layout(&[ + ( + "CHIP", + vec![ + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "2:0:ordinary text"), + ], + ), + ("OTHER", vec![label((900, 7), "3:0:ordinary text")]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!(doc.layer_names.len(), 1); + assert_eq!(doc.layer_names.get(&(1, 0)).unwrap(), "WG"); + assert!(doc.canonical_dump().contains("2:0:ordinary text")); + assert!(doc.canonical_dump().contains("3:0:ordinary text")); + assert!(!doc.canonical_dump().contains(LAYER_SENTINEL)); + let conflicts = layout(&[( + "CHIP", + vec![ + label((9, 0), LAYER_SENTINEL), + label((9, 0), "1:0:A"), + label((9, 0), "1:0:B"), + ], + )]); + assert!( + Document::parse(&conflicts) + .unwrap_err() + .contains("conflicting names") + ); + assert!( + pheon::review::dump_records(&conflicts) + .unwrap() + .contains("1:0:B") + ); + let future = layout(&[("CHIP", vec![label((9, 0), "PHEON_LAYER_NAMES_V2")])]); + let future = Document::parse(&future).unwrap(); + assert_eq!( + Comparison::between(&future, &future).outcome().exit_code(), + 2 + ); +} + +#[test] +fn unknown_records_never_produce_a_successful_equality_claim() { + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![unknown])]); + let doc = Document::parse(&bytes).unwrap(); + let diff = Comparison::between(&doc, &doc); + assert!(!diff.outcome().is_equal()); + assert_eq!(diff.outcome().exit_code(), 2); + assert!(doc.record_dump().contains("7f00")); + let text = diff.text(0); + assert!(text.starts_with("Incomplete\nDiagnostics\n")); + assert!(text.contains("unsupported record 0x7f00")); +} + +#[test] +fn shared_hierarchy_reports_ancestors_and_each_leaf_placement_count() { + let a = shared(square()); + let b = shared(triangle()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + diff.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "CHIP [affected]\n", + " left x1 [affected]\n", + " * leaf x2\n", + " right x1 [affected]\n", + " * leaf x3\n", + ) + ); + let graph = diff.hierarchy(true, 8); + assert_eq!(graph.matches("leaf [cell changed]").count(), 1); + assert!(graph.contains("x2")); + assert!(graph.contains("x3")); + let limited = diff.hierarchy(false, 1); + assert!(!limited.contains("leaf")); + assert!(limited.contains("children omitted")); + assert_eq!( + diff.hierarchy(false, 0), + "Hierarchy\nCHIP [affected] [children omitted]\n" + ); +} + +#[test] +fn ascii_diff_folds_single_placement_paths_up_to_branches_and_changed_edges() { + let with_branch = |count, shape| { + layout(&[ + ("CHIP", vec![reference("block", 0, 1)]), + ("block", vec![reference("routing", 0, 1)]), + ("routing", vec![reference("fanout", 0, 1)]), + ( + "fanout", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("target", 0, count)]), + ("target", vec![]), + ("right", vec![reference("unit", 0, 1)]), + ("unit", vec![reference("leaf", 0, 1)]), + ("leaf", vec![shape]), + ]) + }; + let old = with_branch(2, square()); + let new = with_branch(3, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "CHIP → block → routing [affected]\n", + " fanout x1 [affected]\n", + " * left x1\n", + " target x2 -> x3\n", + " right x1 → unit → * leaf\n", + ) + ); + assert_eq!( + diff.hierarchy(false, 2), + "Hierarchy\nCHIP → block → routing [affected] [children omitted]\n" + ); + assert!(!diff.hierarchy(true, 8).contains('→')); +} + +#[test] +fn ascii_hierarchy_marks_repeated_subtrees_and_depth_limits() { + let bytes = layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ("left", vec![reference("branch", 0, 2)]), + ("right", vec![reference("branch", 0, 3)]), + ("branch", vec![reference("leaf", 0, 1)]), + ("leaf", vec![]), + ]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.hierarchy(false, 8), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2\n", + " \"leaf\" x1\n", + " \"right\" x1\n", + " \"branch\" x3 [subtree elided]\n", + ) + ); + assert_eq!( + doc.hierarchy(false, 2), + concat!( + "Hierarchy\n", + "\"CHIP\"\n", + " \"left\" x1\n", + " \"branch\" x2 [children omitted]\n", + " \"right\" x1\n", + " \"branch\" x3 [children omitted]\n", + ) + ); +} + +#[test] +fn reference_changes_keep_unchanged_targets_and_old_edges_visible() { + for pair in corpus() + .into_iter() + .filter(|p| p.name == "moved reference" || p.name == "array count") + { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + let tree = diff.hierarchy(false, 8); + let edge = if pair.name == "array count" { + " leaf x2 -> x3\n" + } else { + " leaf x1 [reference changed]\n" + }; + assert!(tree.ends_with(edge), "{tree}"); + } + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ]); + let b = layout(&[("CHIP", vec![])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + for (old, new, edge) in [ + (&a, &b, " - leaf x1 -> x0\n"), + (&b, &a, " + leaf x0 -> x1\n"), + ] { + let tree = Comparison::between(old, new).hierarchy(false, 8); + assert!(tree.ends_with(edge), "{tree}"); + } +} + +#[test] +fn same_bounds_changes_and_truncated_reports_remain_differences() { + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new label")])]); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!( + diff.cells[0].removed[0].bounds, + diff.cells[0].added[0].bounds + ); + let limited = diff.text(1); + assert!(limited.contains("Omitted · 2 object changes over the limit")); + assert!(limited.contains("+ 2/0: label")); + assert!(!limited.contains("1/0:")); + assert_eq!(diff.outcome().exit_code(), 1); + assert!(diff.cells[0].removed[0].offset > 0); +} + +#[test] +fn unit_changes_report_values_without_claiming_geometry_equivalence() { + let a = layout(&[("CHIP", vec![square()])]); + let units = Document::parse(&a) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + let mut b = a.clone(); + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + assert_eq!(diff.outcome().exit_code(), 1); + assert!(matches!( + diff.library_changes.as_slice(), + [pheon::review::LibraryChange::Record { + tag: RTAG_UNITS, + .. + }] + )); + let text = diff.text(0); + assert!(text.contains("library UNITS [0.001, 1e-9] -> [0.001, 2e-9]")); + assert!(text.contains("Coordinates use each file's database units")); + assert!(!text.contains("raw=")); + assert!(diff.cells.is_empty()); + assert!(diff.hierarchy(false, 8).is_empty()); +} + +#[test] +fn dangling_references_cycles_and_duplicate_cell_names_are_reported() { + let missing = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let missing = Document::parse(&missing).unwrap(); + assert!( + missing + .diagnostics + .iter() + .any(|d| d.code == "unresolved_reference") + ); + let cycle = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let cycle = Document::parse(&cycle).unwrap(); + assert!( + cycle + .diagnostics + .iter() + .any(|d| d.code == "hierarchy_cycle") + ); + assert!(cycle.hierarchy(false, 8).len() < 200); + let duplicate = layout(&[("a", vec![]), ("a", vec![])]); + assert!( + Document::parse(&duplicate) + .unwrap_err() + .contains("duplicate cell") + ); +} + +#[test] +fn absolute_transform_flags_are_not_discarded() { + let a = reference("leaf", 0, 1); + let mut b = Vec::new(); + record(&mut b, RTAG_SREF, &[]); + ascii(&mut b, RTAG_SNAME, "leaf"); + record(&mut b, RTAG_STRANS, &[0, 2]); + xy(&mut b, &[[0, 0]]); + record(&mut b, RTAG_ENDEL, &[]); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + assert_eq!( + Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .outcome() + .exit_code(), + 1 + ); +} + +#[test] +fn polygon_normalization_handles_repeated_minimum_vertices() { + let ring = [[0, 0], [4, 0], [4, 4], [0, 0], [-2, 2], [-2, 4], [0, 0]]; + let baseline = layout(&[("CHIP", vec![polygon(&ring)])]); + let baseline = Document::parse(&baseline).unwrap(); + let mut open = ring[..ring.len() - 1].to_vec(); + for _ in 0..2 { + for start in 0..open.len() { + let mut rotated = open[start..] + .iter() + .chain(&open[..start]) + .copied() + .collect::>(); + rotated.push(rotated[0]); + let candidate = layout(&[("CHIP", vec![polygon(&rotated)])]); + let candidate = Document::parse(&candidate).unwrap(); + assert!( + Comparison::between(&baseline, &candidate) + .outcome() + .is_equal() + ); + } + open.reverse(); + } +} + +#[test] +fn malformed_lengths_fields_and_units_return_errors_without_panicking() { + let good = layout(&[("CHIP", vec![square()])]); + for length in 0..good.len() { + assert!( + Document::parse(&good[..length]).is_err(), + "accepted truncated length {length}" + ); + } + let mut bad = good.clone(); + bad[0..2].copy_from_slice(&3_u16.to_be_bytes()); + assert!(Document::parse(&bad).is_err()); + let mut bad = good.clone(); + let units = Document::parse(&good) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + bad[units + 4..units + 20].fill(0); + assert!(Document::parse(&bad).unwrap_err().contains("UNITS")); + let mut polygon = square(); + polygon.truncate(polygon.len() - 4); + record(&mut polygon, RTAG_LAYER, &[0, 1]); + record(&mut polygon, RTAG_ENDEL, &[]); + assert!( + Document::parse(&layout(&[("CHIP", vec![polygon])])) + .unwrap_err() + .contains("duplicate element field") + ); +} + +#[test] +fn cli_uses_documented_exit_codes_and_accepts_paths_with_spaces() { + let directory = std::env::temp_dir().join(format!("pheon review {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old file.gds"); + let new = directory.join("new file.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let equal = command() + .arg("diff") + .arg(&old) + .arg(&old) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); + assert_eq!(equal.status.code(), Some(0)); + assert_eq!(equal.stdout, b"Equal\n"); + let diff = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "mermaid", "--max-changes", "0"]) + .output() + .unwrap(); + assert_eq!(diff.status.code(), Some(1)); + let text = String::from_utf8(diff.stdout).unwrap(); + assert!(text.contains("Omitted · 2 object changes over the limit")); + assert!(text.contains("Hierarchy\n```mermaid\nflowchart TD")); + assert!(!text.contains("scope top=all")); + let dump = command() + .arg("dump") + .arg(&old) + .args(["--records"]) + .output() + .unwrap(); + assert_eq!(dump.status.code(), Some(0)); + assert!(String::from_utf8(dump.stdout).unwrap().contains("BGNSTR")); + let graph = command() + .arg("hierarchy") + .arg(&old) + .args(["--format", "mermaid"]) + .output() + .unwrap(); + assert_eq!(graph.status.code(), Some(0)); + assert!( + String::from_utf8(graph.stdout) + .unwrap() + .starts_with("flowchart TD") + ); + let with_reference = |x| layout(&[("CHIP", vec![reference("leaf", x, 1)]), ("leaf", vec![])]); + fs::write(&old, with_reference(0)).unwrap(); + fs::write(&new, with_reference(5)).unwrap(); + let moved = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii"]) + .output() + .unwrap(); + assert_eq!(moved.status.code(), Some(1)); + assert_eq!( + moved.stdout, + b"Different\nCells:\n* CHIP\n ~ reference \"leaf\" moved by (5, 0) dbu\n" + ); + let scoped = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(["--hierarchy", "ascii", "--cell", "CHIP"]) + .output() + .unwrap(); + assert_eq!(scoped.status.code(), Some(1)); + assert!( + String::from_utf8(scoped.stdout) + .unwrap() + .ends_with("Hierarchy\n* CHIP\n leaf x1 [reference changed] [outside scope]\n") + ); + fs::write(&new, b"invalid").unwrap(); + assert_eq!( + command() + .arg("diff") + .arg(&old) + .arg(&new) + .output() + .unwrap() + .status + .code(), + Some(2) + ); +} + +#[test] +fn cli_color_preserves_plain_output_and_machine_formats() { + let directory = std::env::temp_dir().join(format!("pheon color {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let with_changes = |name, shape, positions: std::ops::Range| { + let objects = positions + .map(|x| reference("leaf", x, 1)) + .chain([shape]) + .collect(); + layout(&[ + ("top", vec![reference("route", 0, 1)]), + ("route", vec![reference("CHIP", 0, 1)]), + ("CHIP", objects), + ("leaf", vec![]), + (name, vec![]), + ]) + }; + fs::write(&old, with_changes("obsolete", square(), 0..2)).unwrap(); + fs::write(&new, with_changes("new", triangle(), 10..13)).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + let output = command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + String::from_utf8(output.stdout).unwrap() + }; + let plain = diff(&["--hierarchy", "mermaid"]); + let colored = diff(&["--hierarchy", "mermaid", "--color=always"]); + for span in [ + "\x1b[31m- obsolete", + "\x1b[32m+ new", + "\x1b[33m* CHIP", + "\x1b[31m-2 refs", + "\x1b[32m+3 refs", + "\x1b[33m1 polygon changed", + ] { + assert!(colored.contains(span), "{span:?} in {colored:?}"); + } + let strip_ansi = |text: &str| { + let mut parts = text.split("\x1b["); + let mut plain = parts.next().unwrap().to_string(); + for part in parts { + let (code, text) = part.split_once('m').unwrap(); + assert!(code.bytes().all(|b| b.is_ascii_digit() || b == b';')); + plain.push_str(text); + } + plain + }; + assert_eq!(strip_ansi(&colored), plain); + assert_eq!(diff(&["--hierarchy", "mermaid", "--color", "never"]), plain); + assert_eq!( + colored.split_once("```mermaid\n").unwrap().1, + plain.split_once("```mermaid\n").unwrap().1 + ); + assert_eq!( + diff(&["--color", "always", "--format", "json"]), + diff(&["--format", "json"]) + ); + let ascii = diff(&["--hierarchy", "ascii", "--color=always"]); + let folded = ascii + .lines() + .find(|line| line.starts_with("top → route → ")) + .unwrap(); + assert!(folded.contains("\x1b[33m* CHIP")); + assert_eq!(strip_ansi(&ascii), diff(&["--hierarchy", "ascii"])); + + for (options, token) in [ + (&[][..], "\x1b[36mcell \"CHIP\""), + (&["--records"][..], "\x1b[36mBGNSTR"), + ] { + let dump = |choice| { + let output = command() + .arg("dump") + .arg(&old) + .args(options) + .args(["--color", choice]) + .output() + .unwrap(); + assert!(output.status.success()); + String::from_utf8(output.stdout).unwrap() + }; + let colored = dump("always"); + assert!(colored.contains(token), "{colored:?}"); + assert_eq!(strip_ansi(&colored), dump("auto")); + assert_eq!(dump("auto"), dump("never")); + } +} + +#[test] +fn cli_details_show_coordinates_and_properties_within_filters_and_limits() { + let directory = std::env::temp_dir().join(format!("pheon-details-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let common_ref = reference("leaf", 0, 1); + let additional_ref = reference("leaf", 5, 1); + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let shifted = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + fs::write( + &old, + layout(&[ + ( + "CHIP", + vec![with_property(square(), "old"), common_ref.clone()], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "before other")]), + ("removed", vec![label((2, 0), "payload in removed cell")]), + ]), + ) + .unwrap(); + fs::write( + &new, + layout(&[ + ( + "CHIP", + vec![ + with_property(shifted, "new"), + with_property(path(Some(8), &points), "path property"), + common_ref, + additional_ref.clone(), + additional_ref, + ], + ), + ("leaf", vec![]), + ("other", vec![label((2, 0), "after other")]), + ("added", vec![label((2, 0), "payload in added cell")]), + ]), + ) + .unwrap(); + let diff = |options: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + }; + let detailed = diff(&["--show-cell", "CHIP", "--details", "--hierarchy", "ascii"]); + assert!(detailed.contains(concat!( + " ~ 1/0: rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10) property[17]=\"old\"\n", + " + 1/0: rectangle (2, -1) to (12, 9) property[17]=\"new\"\n", + ))); + assert!(detailed.contains(concat!( + " + 1/0: path width=[8] xy=[(0, 0), (4, 0), (4, -5), (13, -5),\n", + " (13, 9)] property[17]=\"path property\"\n", + ))); + assert!(detailed.contains(" + 2× reference \"leaf\" xy=(5, 0) · placements 1 -> 3\n")); + assert!(!detailed.contains("Hierarchy\n")); + assert!(!detailed.contains("after other")); + let limited = diff(&[ + "--show-cell", + "CHIP", + "--details", + "--max-changes", + "1", + "--hierarchy", + "ascii", + ]); + assert!(limited.contains("4 object changes over the limit")); + assert!(!limited.contains("~ 1/0: rectangle")); + assert!(!limited.contains("property[17]=\"old\"")); + assert!(limited.ends_with("Hierarchy\n* CHIP\n leaf x1 -> x3\n")); + + let all = diff(&["--details"]); + assert!(all.contains("Cells:\n- removed\n+ added\n* CHIP\n")); + assert!(!all.contains("payload in")); + assert_eq!( + diff(&["--details", "--format", "json"]), + diff(&["--format", "json"]) + ); +} + +#[test] +fn cli_decodes_gzip_by_header_and_checks_the_trailer() { + let directory = std::env::temp_dir().join(format!("pheon gzip {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let plain = directory.join("plain.gds"); + let compressed = directory.join("compressed.gds"); + let data = shared(square()); + // Splitting a record between gzip members must preserve the decompressed layout. + let mut encoded = [gzip(&data[..7]), gzip(&data[7..])].concat(); + fs::write(&plain, &data).unwrap(); + fs::write(&compressed, &encoded).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let dump = |path: &PathBuf| { + let output = command() + .arg("dump") + .arg(path) + .arg("--records") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout + }; + assert_eq!(dump(&plain), dump(&compressed)); + let compare = || { + let output = command() + .arg("diff") + .arg(&plain) + .arg(&compressed) + .args(["--format", "json"]) + .output() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + (output.status.code(), report) + }; + let (code, report) = compare(); + assert_eq!(code, Some(0)); + assert_eq!(report["result"], "equal"); + + // Corrupt the last CRC after all the GDSII bytes have already been decoded. + let crc = encoded.len() - 8; + encoded[crc] ^= 0xff; + fs::write(&compressed, encoded).unwrap(); + let (code, report) = compare(); + assert_eq!(code, Some(2)); + assert_eq!(report["status"], "failed"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "invalid_input"); + assert_eq!(diagnostic["side"], "new"); + assert!( + diagnostic["message"] + .as_str() + .unwrap() + .contains("gzip decompression failed") + ); +} + +#[test] +fn metadata_writer_preserves_unrelated_bytes_and_consolidates_updates() { + use pheon::review::names::write_layer_names; + use std::collections::BTreeMap; + let mut bytes = shared(with_property(square(), "keep this")); + bytes.extend_from_slice(&[0; 32]); + let updates = BTreeMap::from([ + ((1, 0), "WG".to_string()), + ((2, 3), "HEATER:draw".to_string()), + ]); + let output = write_layer_names(&bytes, "CHIP", (900, 7), &updates).unwrap(); + let doc = Document::parse(&output).unwrap(); + assert_eq!(doc.layer_names, updates); + let meta: Vec<_> = doc.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .collect(); + let start = meta.iter().map(|o| o.offset).min().unwrap(); + let end = meta.iter().map(|o| o.end_offset).max().unwrap(); + assert_eq!([&output[..start], &output[end..]].concat(), bytes); + let updated = write_layer_names( + &output, + "CHIP", + (900, 7), + &BTreeMap::from([((1, 0), "CORE".to_string())]), + ) + .unwrap(); + let parsed = Document::parse(&updated).unwrap(); + assert_eq!(parsed.layer_names[&(1, 0)], "CORE"); + assert_eq!(parsed.layer_names[&(2, 3)], "HEATER:draw"); + assert_eq!( + parsed.cells["CHIP"] + .objects + .iter() + .filter(|o| o.layer == Some((900, 7))) + .count(), + 3 + ); + let diff = Comparison::between(&Document::parse(&output).unwrap(), &parsed); + assert!(diff.cells.is_empty()); + assert_eq!(diff.layer_names.len(), 1); + assert_eq!( + write_layer_names(&updated, "CHIP", (900, 7), &BTreeMap::new()).unwrap(), + updated + ); +} + +#[test] +fn metadata_writer_rejects_ambiguous_carriers_versions_and_bad_names() { + use pheon::review::names::{parse_layer_name, write_layer_names}; + use std::collections::BTreeMap; + let bytes = layout(&[("CHIP", vec![label((900, 7), "ordinary label")])]); + assert!( + write_layer_names(&bytes, "CHIP", (900, 7), &BTreeMap::new()) + .unwrap_err() + .contains("ordinary design") + ); + assert!( + write_layer_names(&bytes, "missing", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("does not exist") + ); + let bytes = layout(&[("CHIP", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")])]); + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::new()) + .unwrap_err() + .contains("unsupported") + ); + let bytes = layout(&[("CHIP", vec![square()])]); + for value in ["".to_string(), "bad\0name".to_string(), "x".repeat(65536)] { + assert!( + write_layer_names(&bytes, "CHIP", (901, 7), &BTreeMap::from([((1, 0), value)])) + .is_err() + ); + } + for text in ["+1:0:name", "1:-1:name", "1:0:", "65536:0:name"] { + assert!(parse_layer_name(text).is_err()); + } +} + +#[test] +fn metadata_cli_creates_a_separate_file_and_refuses_existing_output() { + let directory = std::env::temp_dir().join(format!("pheon-metadata-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let source = directory.join("source.gds.gz"); + let output = directory.join("output.gds"); + let bytes = gzip(&shared(square())); + fs::write(&source, &bytes).unwrap(); + let _ = fs::remove_file(&output); + let mut command = Command::new(env!("CARGO_BIN_EXE_pheon")); + command.arg("name-layers").arg(&source).arg(&output).args([ + "--cell", + "CHIP", + "--carrier", + "900/7", + "--name", + "1:0:WG", + ]); + let result = command.output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(fs::read(&source).unwrap(), bytes); + let generated = fs::read(&output).unwrap(); + assert_eq!( + Document::parse(&generated).unwrap().layer_names[&(1, 0)], + "WG" + ); + assert_eq!(command.output().unwrap().status.code(), Some(2)); + assert_eq!(fs::read(&output).unwrap(), generated); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn scope_selection_distinguishes_reachable_subtrees_from_exact_cells() { + use pheon::review::selection::Scope; + let a = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![square()]), + ("unused", vec![square()]), + ]); + let b = layout(&[ + ("CHIP", vec![reference("leaf", 0, 1)]), + ("leaf", vec![triangle()]), + ("unused", vec![triangle()]), + ]); + let top = Scope { + top: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&top, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&top, false).unwrap(); + assert_eq!( + old.cells.keys().map(String::as_str).collect::>(), + ["CHIP", "leaf"] + ); + assert_eq!(Comparison::between(&old, &new).cells.len(), 1); + assert!(old.canonical_dump().contains("Scope top=\"CHIP\"\n")); + let cell = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + let old = Document::parse(&a).unwrap().selected(&cell, false).unwrap(); + let new = Document::parse(&b).unwrap().selected(&cell, false).unwrap(); + assert!(old.diagnostics.is_empty()); + assert!(Comparison::between(&old, &new).outcome().is_equal()); + let missing = Scope { + top: Some("missing".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&a) + .unwrap() + .selected(&missing, false) + .is_err() + ); + let wrong = Scope { + top: Some("CHIP".to_string()), + cell: Some("unused".to_string()), + layer: None, + }; + assert!( + wrong + .validate_pair(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()) + .is_err() + ); +} + +#[test] +fn layer_scope_filters_geometry_and_references_by_descendant_coverage() { + use pheon::review::selection::Scope; + let mut other_layer = square(); + other_layer[8..10].copy_from_slice(&2_u16.to_be_bytes()); + let a = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 0, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer.clone()]), + ]); + let b = layout(&[ + ( + "CHIP", + vec![reference("wg", 0, 1), reference("heater", 5, 1)], + ), + ("wg", vec![square()]), + ("heater", vec![other_layer]), + ]); + for (layer, expected) in [((1, 0), 0), ((2, 0), 1)] { + let scope = Scope { + layer: Some(layer), + ..Scope::default() + }; + let old = Document::parse(&a) + .unwrap() + .selected(&scope, false) + .unwrap(); + let new = Document::parse(&b) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!( + Comparison::between(&old, &new).outcome().exit_code(), + expected + ); + assert!(old.diagnostics.is_empty()); + } +} + +#[test] +fn scoped_issues_do_not_hide_relevant_unsupported_objects_or_cycles() { + use pheon::review::selection::Scope; + let mut unknown = square(); + unknown.truncate(unknown.len() - 4); + record(&mut unknown, 0x7f00, &[]); + record(&mut unknown, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![square()]), ("unused", vec![unknown])]); + let scope = Scope { + cell: Some("CHIP".to_string()), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .diagnostics + .is_empty() + ); + let scope = Scope { + cell: Some("unused".to_string()), + ..Scope::default() + }; + assert!( + !Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .diagnostics + .is_empty() + ); + let bytes = layout(&[ + ("a", vec![reference("b", 0, 1)]), + ("b", vec![reference("a", 0, 1)]), + ]); + let scope = Scope { + layer: Some((999, 0)), + ..Scope::default() + }; + assert!( + Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap() + .diagnostics + .iter() + .any(|d| d.code == "hierarchy_cycle") + ); +} + +#[test] +fn display_filters_never_change_equality_or_totals() { + use pheon::review::selection::DisplayFilter; + let a = layout(&[("CHIP", vec![square()])]); + let b = layout(&[("CHIP", vec![triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text_filtered( + 100, + &DisplayFilter { + cell: None, + layer: Some((42, 0)), + }, + ); + assert!(text.starts_with("Different · 1 cell modified\n")); + assert!(text.contains("Display filter layer=42/0\n")); + assert!(text.contains("Omitted · 2 object changes filtered out")); + assert!(!text.contains("Cells:\n")); + assert_eq!(diff.outcome().exit_code(), 1); +} + +#[test] +fn text_hides_empty_headings_and_counts_filtered_and_limited_changes() { + use pheon::review::selection::DisplayFilter; + + let mut attributes = Vec::new(); + record(&mut attributes, RTAG_STRCLASS, &[0, 1]); + let a = layout(&[ + ("changed", vec![square()]), + ("other", vec![square()]), + ( + "gone", + vec![square(), triangle(), reference("target", 0, 1)], + ), + ("header", vec![]), + ("target", vec![]), + ]); + let b = layout(&[ + ("changed", vec![triangle()]), + ("other", vec![triangle()]), + ("new", vec![square(), reference("target", 0, 1)]), + ("empty", vec![]), + ("header", vec![attributes]), + ("target", vec![]), + ]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let text = diff.text(0); + assert!(text.starts_with("Different · 3 cells modified, 2 added, 1 removed\n")); + assert!(text.ends_with(concat!( + "Cells:\n", + "- gone\n", + "+ empty\n", + "+ new\n", + "* header (attributes changed)\n", + ))); + assert!(text.contains("Omitted · 4 object changes over the limit")); + + let text = diff.text_filtered( + 1, + &DisplayFilter { + cell: Some("changed".to_string()), + layer: Some((1, 0)), + }, + ); + assert!(text.contains(concat!( + "Display filter cell=\"changed\" layer=1/0\n", + "Omitted · 2 object changes filtered out · 2 object changes over the limit", + " · 4 cell-level changes filtered out\n", + ))); + assert!(!text.contains("Cells:\n")); +} + +#[test] +fn cell_list_quotes_names_with_whitespace() { + let old = layout(&[("old name", vec![]), ("same\nname", vec![square()])]); + let new = layout(&[("new_name", vec![]), ("same\nname", vec![triangle()])]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\n", + "Cells:\n", + "- \"old name\"\n", + "+ new_name\n", + "* \"same\\nname\"\n", + " 1/0: 1 polygon changed\n", + ) + ); + assert!(diff.hierarchy(false, 8).is_empty()); +} + +#[test] +fn reference_summaries_group_repeated_targets_and_keep_full_placements_in_dumps() { + let empty = layout(&[("leaf", vec![]), ("other", vec![])]); + let with_chip = layout(&[ + ( + "CHIP", + vec![ + square(), + label((2, 0), "chip label"), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "3:0:WG"), + reference("leaf", 0, 1), + reference("leaf", 5, 1), + reference("leaf", 2, 2), + reference("other", 10, 1), + ], + ), + ("leaf", vec![]), + ("other", vec![]), + ]); + let existing_chip = layout(&[("CHIP", vec![]), ("leaf", vec![]), ("other", vec![])]); + let empty = Document::parse(&empty).unwrap(); + let with_chip = Document::parse(&with_chip).unwrap(); + let existing_chip = Document::parse(&existing_chip).unwrap(); + + for (old, new, sign) in [(&empty, &with_chip, '+'), (&with_chip, &empty, '-')] { + let diff = Comparison::between(old, new); + let text_diff = TextComparison::between(old, new); + let text = diff.text(100); + assert!(text.ends_with(&format!("Cells:\n{sign} CHIP\n"))); + assert!(!text.contains("\n ")); + assert!(!text.contains("1/0")); + assert!(!text.contains("2/0")); + assert!(!text.contains("Omitted")); + assert_eq!(diff.text(0), text); + assert_eq!(text_diff.outcome(), diff.outcome()); + for mermaid in [false, true] { + assert_eq!(text_diff.hierarchy(mermaid, 8), diff.hierarchy(mermaid, 8)); + } + for show in [ + DisplayFilter::default(), + DisplayFilter { + cell: Some("other".into()), + layer: None, + }, + DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ] { + for max in [0, 2, 100] { + assert_eq!( + text_diff.render_text(max, &show, true), + diff.text_filtered(max, &show) + ); + } + } + } + + for (old, new, sign) in [ + (&existing_chip, &with_chip, "+"), + (&with_chip, &existing_chip, "-"), + ] { + let text = Comparison::between(old, new).text(100); + assert!(text.contains(&format!( + " {sign} 2 refs, 1 array ref to leaf (4 placements)\n" + ))); + assert!(!text.contains("\"leaf\" xy=")); + assert!(text.contains(&format!(" {sign} reference \"other\" xy=(10, 0)\n"))); + } + let dump = with_chip.canonical_dump(); + assert!(dump.contains(" reference \"leaf\" xy=(5, 0)\n")); + assert!(dump.contains(" array reference \"leaf\" (2 × 1) xy=[(2, 0), (102, 0), (2, 0)]\n")); +} + +#[test] +fn reference_summaries_combine_visible_changes_to_a_target() { + let make_layout = |positions: std::ops::Range, moving_x| { + let mut refs: Vec<_> = positions + .chain(20..27) + .map(|x| reference("leaf", x, 1)) + .collect(); + refs.push(reference("moving", moving_x, 1)); + refs.push(reference("moving", 99, 1)); + layout(&[("CHIP", refs), ("leaf", vec![]), ("moving", vec![])]) + }; + let old = make_layout(0..3, 0); + let new = make_layout(10..16, 5); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " ~ reference \"moving\" moved by (5, 0) dbu · 2 total placements\n", + " leaf · -3 refs, +6 refs · placements 10 -> 13\n", + ) + ); + let ascii = + |max, show: &DisplayFilter| diff.render_text_with_ascii_hierarchy(max, show, true, 8); + assert_eq!(ascii(100, &Default::default()), diff.text(100)); + assert!(diff.text(5).ends_with(" - 3 refs to leaf\n")); + let limited = diff.text(6); + assert!( + limited.starts_with( + "Different · 1 cell modified\nOmitted · 5 object changes over the limit\n" + ) + ); + assert!(limited.ends_with(" leaf · -3 refs, +1 ref\n")); + let limited_ascii = ascii(6, &Default::default()); + assert!(limited_ascii.starts_with(&limited)); + assert!(limited_ascii.ends_with("Hierarchy\n* CHIP\n leaf x10 -> x13\n")); + + let filtered = ascii( + 100, + &DisplayFilter { + cell: None, + layer: Some((1, 0)), + }, + ); + assert!(filtered.contains("11 object changes filtered out")); + assert!(filtered.ends_with(" leaf x10 -> x13\n moving x2 [reference changed]\n")); +} + +#[test] +fn combined_ascii_hierarchy_preserves_paths_to_changed_and_shared_cells() { + let make_layout = |x, shape| { + layout(&[ + ( + "CHIP", + vec![reference("left", 0, 1), reference("right", 0, 1)], + ), + ( + "left", + vec![ + reference("shared", x, 1), + reference("changed", x, 1), + reference("local", x, 1), + ], + ), + ("right", vec![reference("shared", x, 1)]), + ("shared", vec![]), + ("changed", vec![shape]), + ("local", vec![]), + ]) + }; + let old = make_layout(0, square()); + let new = make_layout(5, triangle()); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let text = diff.render_text_with_ascii_hierarchy(100, &Default::default(), true, 8); + let (cells, hierarchy) = text.split_once("Hierarchy\n").unwrap(); + assert!(cells.contains("reference \"local\" moved by (5, 0) dbu")); + assert_eq!( + hierarchy, + concat!( + "CHIP [affected]\n", + " * left x1\n", + " * changed x1 [reference changed]\n", + " shared x1 [reference changed]\n", + " * right x1\n", + " shared x1 [reference changed]\n", + ) + ); +} + +#[test] +fn combined_reference_summaries_keep_array_counts_and_placements() { + let old = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 0, 1), + reference("leaf", 10, 2), + reference("leaf", 20, 3), + ], + ), + ("leaf", vec![]), + ]); + let new = layout(&[ + ( + "CHIP", + vec![ + reference("leaf", 50, 1), + reference("leaf", 60, 1), + reference("leaf", 70, 2), + reference("leaf", 80, 4), + ], + ), + ("leaf", vec![]), + ]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\nCells:\n* CHIP\n", + " leaf · -1 ref, -2 array refs (6 placements), +2 refs, +2 array refs (8 placements)\n", + ) + ); +} + +#[test] +fn git_external_driver_handles_changes_additions_deletions_and_errors() { + let directory = std::env::temp_dir().join(format!( + "pheon-git-review-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let inherited_path = std::env::var_os("PATH").unwrap_or_default(); + let git_path = std::env::join_paths( + std::iter::once( + PathBuf::from(env!("CARGO_BIN_EXE_pheon")) + .parent() + .unwrap() + .to_path_buf(), + ) + .chain(std::env::split_paths(&inherited_path)), + ) + .unwrap(); + let git = |args: &[&str]| { + let result = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(args) + .output() + .unwrap(); + assert!( + result.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&result.stderr) + ); + result + }; + git(&["init", "-q"]); + git(&["config", "user.name", "Review Fixture"]); + git(&["config", "user.email", "fixture@example.invalid"]); + git(&["config", "commit.gpgsign", "false"]); + git(&["config", "core.hooksPath", "/dev/null"]); + git(&[ + "config", + "include.path", + concat!(env!("CARGO_MANIFEST_DIR"), "/contrib/git/config"), + ]); + assert_eq!( + String::from_utf8(git(&["config", "--get", "diff.pheon.trustExitCode"]).stdout) + .unwrap() + .trim(), + "false" + ); + fs::write( + directory.join(".gitattributes"), + include_str!("../contrib/git/attributes"), + ) + .unwrap(); + let path = directory.join("chip layout.gds"); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + git(&["add", "."]); + git(&["commit", "-qm", "baseline"]); + fs::write(&path, layout(&[("CHIP", vec![triangle()])])).unwrap(); + let changed = git(&["diff", "--", "chip layout.gds"]); + assert!( + String::from_utf8(changed.stdout) + .unwrap() + .contains("Different\nCells:\n* CHIP\n") + ); + fs::write(&path, b"invalid").unwrap(); + let invalid = Command::new("git") + .env("PATH", &git_path) + .arg("-C") + .arg(&directory) + .args(["diff", "--", "chip layout.gds"]) + .output() + .unwrap(); + assert!(!invalid.status.success()); + assert!(String::from_utf8_lossy(&invalid.stderr).contains("pheon:")); + fs::write(&path, layout(&[("CHIP", vec![square()])])).unwrap(); + let mode_change = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["git-diff", "--hierarchy", "ascii", "--", "chip layout.gds"]) + .arg(&path) + .args(["old", "100644"]) + .arg(&path) + .args(["new", "100755"]) + .output() + .unwrap(); + assert!(mode_change.status.success()); + assert_eq!( + mode_change.stdout, + b"layout \"chip layout.gds\"\nmode \"100644\" -> \"100755\"\nEqual\n" + ); + let added = directory.join("added layout.gds"); + fs::write(&added, shared(square())).unwrap(); + git(&["add", "added layout.gds"]); + let result = git(&["diff", "--cached", "--", "added layout.gds"]); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.starts_with("layout \"added layout.gds\" added\nDifferent · 4 cells added\n")); + assert!(text.contains("Cells:\n+ CHIP\n+ leaf\n+ left\n+ right\nHierarchy\n")); + assert!(!text.contains("\nmode ")); + assert!(!text.contains("\nlibrary ")); + fs::remove_file(&path).unwrap(); + git(&["add", "-u"]); + let result = git(&["diff", "--cached", "--", "chip layout.gds"]); + let text = String::from_utf8(result.stdout).unwrap(); + assert_eq!( + text, + "layout \"chip layout.gds\" deleted\nDifferent\nCells:\n- CHIP\n" + ); + // Textconv is an alternative to the external driver, with a separate cache. + git(&["commit", "-qm", "replace fixture"]); + git(&["mv", "added layout.gds", "renamed layout.gds"]); + let renamed = directory.join("renamed layout.gds"); + fs::write(&renamed, shared(triangle())).unwrap(); + git(&["add", "renamed layout.gds"]); + let result = git(&["diff", "--cached", "--find-renames"]); + let text = String::from_utf8(result.stdout).unwrap(); + assert!(text.contains("renamed layout.gds"), "{text}"); + assert!(text.contains("Different\nCells:\n* leaf\n"), "{text}"); + git(&["commit", "-qm", "rename fixture"]); + fs::write(directory.join(".gitattributes"), "*.gds diff=pheon-text\n").unwrap(); + fs::write(&renamed, shared(square())).unwrap(); + let result = git(&["diff", "--", "renamed layout.gds"]); + assert!( + String::from_utf8(result.stdout) + .unwrap() + .contains("- 1/0: polygon") + ); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn replacement_explanations_are_unique_and_keep_full_evidence() { + for pair in corpus().into_iter().filter(|p| { + [ + "same bounds different geometry", + "moved reference", + "array count", + ] + .contains(&p.name) + }) { + let old = Document::parse(&pair.old).unwrap(); + let new = Document::parse(&pair.new).unwrap(); + let diff = Comparison::between(&old, &new); + assert_eq!(diff.cells[0].replacements.len(), 1); + let text = diff.text(100); + match pair.name { + "moved reference" => { + assert!(text.contains(" ~ reference \"leaf\" moved by (5, 0) dbu\n")) + } + "array count" => { + assert!(text.contains(" ~ array reference \"leaf\" columns/rows 2 x 1 -> 3 x 1\n")) + } + _ => assert_eq!(text, include_str!("golden/review-diff.txt")), + } + assert!(!text.contains(" - ")); + assert!(!text.contains(" + ")); + assert!(!text.contains("bbox=")); + assert!(!diff.text(0).contains(" ~ ")); + assert_eq!( + diff.text(1), + "Different · 1 cell modified\nOmitted · 2 object changes over the limit\n" + ); + } + let a = layout(&[("CHIP", vec![square(), square()])]); + let b = layout(&[("CHIP", vec![triangle(), triangle()])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!(diff.cells[0].replacements.is_empty()); + let text = diff.text(100); + assert!(text.contains(" 1/0: 2 rectangles removed · 2 polygons added\n")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); +} + +#[test] +fn layer_only_shape_replacements_keep_evidence_and_respect_selection() { + let rect = [[10, 0], [0, 0], [0, 10], [10, 10], [10, 0]]; + let on_core = ring(RTAG_BOUNDARY, RTAG_DATATYPE, 2, &rect); + let named = |mut objects: Vec>, names: &[&str]| { + objects.push(label((900, 7), LAYER_SENTINEL)); + objects.extend(names.iter().map(|name| label((900, 7), name))); + layout(&[("CHIP", objects)]) + }; + let old = named(vec![square()], &["1:0:WG", "2:0:CORE"]); + let new = named(vec![on_core.clone()], &["1:0:WG", "2:0:CORE"]); + let old = Document::parse(&old).unwrap(); + let diff = Comparison::between(&old, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(2), + "Different\nCells:\n* CHIP\n ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n" + ); + assert!( + diff.render_text_with_options( + 2, + &Default::default(), + true, + TextReportOptions { + details: true, + ..Default::default() + }, + ) + .ends_with(concat!( + " ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n", + " - 1/0: rectangle (0, 0) to (10, 10)\n", + " + 2/0: rectangle (0, 0) to (10, 10)\n", + )) + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(!limited.contains("Cells:")); + for (pair, suffix) in [ + ((1, 0), " 1/0 (WG): 1 rectangle removed\n"), + ((2, 0), " 2/0 (CORE): 1 rectangle added\n"), + ] { + let filtered = diff.text_filtered( + 1, + &DisplayFilter { + cell: None, + layer: Some(pair), + }, + ); + assert!(filtered.contains("1 object change filtered out")); + assert!(filtered.ends_with(suffix), "{filtered}"); + } + let json = diff.json_report(&Default::default(), &Default::default()); + let replacement = &json["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_layer_change"); + assert_eq!(replacement["field_changes"].as_array().unwrap().len(), 1); + assert_eq!(replacement["field_changes"][0]["tag"], RTAG_LAYER); + assert_eq!(json["summary"]["removed_objects"], "1"); + assert_eq!(json["summary"]["added_objects"], "1"); + + let old = named(vec![square()], &["1:0:WG"]); + let new = named(vec![on_core], &["2:0:CORE"]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!( + diff.text(2) + .ends_with(" ~ 1/0 (WG) → 2/0 (CORE) · rectangle\n") + ); +} + +#[test] +fn layer_only_matching_requires_unique_shapes_and_identical_payloads() { + let core_square = ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ); + for (old, new) in [ + (vec![square(), square()], vec![core_square.clone()]), + ( + vec![square()], + vec![core_square.clone(), core_square.clone()], + ), + (vec![square()], vec![with_property(core_square, "new")]), + ( + vec![square()], + vec![ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + )], + ), + ] { + let old = layout(&[("CHIP", old)]); + let new = layout(&[("CHIP", new)]); + let diff = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + assert!(diff.cells[0].replacements.is_empty()); + assert!(diff.text(100).contains(" removed\n")); + assert!(diff.text(100).contains(" added\n")); + } +} + +#[test] +fn datatype_only_path_changes_pair_on_the_same_layer() { + let old = layout(&[("CHIP", vec![path(Some(8), &[[0, 0], [10, 10]])])]); + let old_doc = Document::parse(&old).unwrap(); + let at = old_doc + .records + .iter() + .find(|record| record.tag == RTAG_DATATYPE) + .unwrap() + .offset + + 4; + let mut new = old.clone(); + new[at..at + 2].copy_from_slice(&3_u16.to_be_bytes()); + let diff = Comparison::between(&old_doc, &Document::parse(&new).unwrap()); + assert!(diff.text(2).ends_with(" ~ 1/0 → 1/3 · path\n")); + assert_eq!( + diff.cells[0].replacements[0].match_rule, + "unique_layer_change" + ); +} + +#[test] +fn polygon_diff_reports_single_displacements_and_groups_bulk_moves() { + let before = layout(&[("CHIP", vec![with_property(square(), "old")])]); + let before = Document::parse(&before).unwrap(); + for (shape, label) in [ + ( + polygon(&[[-5, 2], [5, 2], [5, 12], [-5, 12], [-5, 2]]), + "1 rectangle changed", + ), + ( + polygon(&[[0, 0], [20, 0], [20, 10], [0, 10], [0, 0]]), + "1 rectangle changed", + ), + ( + polygon(&[[5, 0], [10, 5], [5, 10], [0, 5], [5, 0]]), + "1 polygon changed", + ), + (square(), "1 rectangle changed"), + ] { + let after = layout(&[("CHIP", vec![with_property(shape, "new")])]); + let after = Document::parse(&after).unwrap(); + let diff = Comparison::between(&before, &after); + let text = diff.text(100); + assert!(text.contains(&format!(" 1/0: {label}\n")), "{text}"); + assert!(!text.contains("dbu")); + assert!(!text.contains("bbox=")); + assert!(!text.contains("xy=")); + } + + let moved_triangle = polygon(&[[2, -1], [12, -1], [2, 9], [2, -1]]); + let moved_square = polygon(&[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]]); + let named_layout = |mut objects: Vec>| { + objects.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", objects)]) + }; + for (shape, shifted, name) in [ + (triangle(), moved_triangle, "polygon"), + (square(), moved_square.clone(), "rectangle"), + ( + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], + ), + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[2, -1], [12, -1], [12, 9], [2, 9], [2, -1]], + ), + "box", + ), + ] { + let before = named_layout(vec![shape]); + let after = named_layout(vec![shifted]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!( + diff.text(100) + .contains(&format!(" 1/0 (WG): {name} moved by (2, -1) dbu\n")) + ); + } + + let before = named_layout(vec![square()]); + let after = named_layout(vec![ + moved_square, + ring( + RTAG_BOX, + RTAG_BOXTYPE, + 1, + &[[20, 0], [30, 0], [30, 10], [20, 10], [20, 0]], + ), + ]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert!( + diff.text(100) + .contains(" 1/0 (WG): 1 rectangle moved · 1 box added\n") + ); + assert!( + diff.text(2) + .contains(" 1/0 (WG): rectangle moved by (2, -1) dbu\n") + ); +} + +#[test] +fn polygon_diff_groups_shapes_by_layer_across_dimensions() { + let shapes = [ + square(), + polygon(&[[30, 0], [50, 0], [50, 5], [30, 5], [30, 0]]), + polygon(&[[70, 0], [80, 0], [71, 9], [70, 0]]), + ring( + RTAG_BOUNDARY, + RTAG_DATATYPE, + 2, + &[[0, 0], [10, 0], [0, 10], [0, 0]], + ), + ]; + let layout_with_property = |value| { + let objects = shapes + .iter() + .map(|shape| with_property(shape.clone(), value)) + .collect(); + layout(&[("CHIP", objects)]) + }; + let before = layout_with_property("old"); + let after = layout_with_property("new"); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + assert_eq!( + diff.text(100), + concat!( + "Different\n", + "Cells:\n* CHIP\n", + " 1/0: 2 rectangles, 1 polygon changed\n", + " 2/0: 1 polygon changed\n", + ) + ); +} + +#[test] +fn polygon_diff_distinguishes_boxes_from_rectangular_boundaries() { + let before = layout(&[("CHIP", vec![square()])]); + let points = [[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]; + let after = layout(&[("CHIP", vec![ring(RTAG_BOX, RTAG_BOXTYPE, 1, &points)])]); + let diff = Comparison::between( + &Document::parse(&before).unwrap(), + &Document::parse(&after).unwrap(), + ); + let text = diff.text(100); + assert!(text.contains(" 1/0: 1 rectangle removed · 1 box added\n")); +} + +#[test] +fn path_width_explanation_preserves_absent_and_zero() { + let points = [[0, 0], [10, 10]]; + let a = layout(&[("CHIP", vec![path(None, &points)])]); + let b = layout(&[("CHIP", vec![path(Some(0), &points)])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert!( + diff.text(100) + .contains(" ~ 1/0: path width absent -> 0 dbu\n") + ); +} + +#[test] +fn unmatched_paths_summarize_coordinates_that_would_wrap() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let empty = layout(&[("CHIP", vec![])]); + let paths = layout(&[( + "CHIP", + vec![ + path(None, &points), + path(Some(-8), &points), + path(Some(0), &points[..4]), + ], + )]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let text = Comparison::between(old, new).text(100); + for width in ["absent", "-8 dbu"] { + assert!(text.contains(&format!( + " {sign} 1/0: path · width {width} · 5 vertices · bounds (0, -5) to (13, 9) dbu\n" + ))); + } + assert_eq!( + text.lines().filter(|line| line.starts_with(" ")).count(), + 3, + "{text}" + ); + assert!(text.contains("xy=[(0, 0), (4, 0), (4, -5), (13, -5)]")); + } + assert!(paths.canonical_dump().contains("\n (13, 9)]")); + let report = + Comparison::between(&empty, &paths).json_report(&Default::default(), &Default::default()); + assert!( + report["cell_changes"][0]["added"] + .as_array() + .unwrap() + .iter() + .flat_map(|object| object["fields"].as_array().unwrap()) + .any(|field| field["tag"] == RTAG_XY && field["value"] == serde_json::json!(points)) + ); +} + +#[test] +fn repeated_unmatched_rows_group_only_identical_objects_within_the_limit() { + let points = [[0, 0], [4, 0], [4, -5], [13, -5], [13, 9]]; + let mut other_points = points; + other_points[1][0] = 5; + let named_paths = |mut paths: Vec>| { + paths.extend([label((900, 7), LAYER_SENTINEL), label((900, 7), "1:0:WG")]); + layout(&[("CHIP", paths)]) + }; + let empty = named_paths(vec![]); + let paths = named_paths(vec![ + path(None, &points), + path(None, &other_points), + path(None, &points), + ]); + let empty = Document::parse(&empty).unwrap(); + let paths = Document::parse(&paths).unwrap(); + for (old, new, sign) in [(&empty, &paths, "+"), (&paths, &empty, "-")] { + let diff = Comparison::between(old, new); + let details = "path · width absent · 5 vertices · bounds (0, -5) to (13, 9) dbu"; + let full = diff.text(3); + assert!(full.contains("1/0: name \"WG\"\nCells:\n")); + assert!(full.ends_with(&format!( + " {sign} 2× 1/0: {details}\n {sign} 1/0: {details}\n" + ))); + assert!( + diff.text(2) + .ends_with(&format!(" {sign} 2× 1/0 (WG): {details}\n")) + ); + let limited = diff.text(1); + assert!(limited.contains("2 object changes over the limit")); + assert!(limited.ends_with(&format!(" {sign} 1/0 (WG): {details}\n"))); + } +} + +#[test] +fn label_replacements_show_only_changed_fields() { + let a = layout(&[("CHIP", vec![label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + assert_eq!( + diff.text(100), + "Different\nCells:\n* CHIP\n ~ 2/0: label text \"old\" -> \"new\"\n" + ); +} + +#[test] +fn unchanged_layer_names_explain_geometry_changes_once() { + let named = |geometry, text| { + layout(&[ + ( + "CHIP", + vec![ + geometry, + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + ], + ), + ("leaf", vec![label((1, 0), text)]), + ]) + }; + let old = named(square(), "old"); + let a = Document::parse(&old).unwrap(); + let new = named(triangle(), "old"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + assert_eq!( + diff.text(100), + "Different\nCells:\n* CHIP\n 1/0 (WG): 1 polygon changed\n" + ); + assert!(diff.layer_names.is_empty()); + + let new = named(triangle(), "new"); + let diff = Comparison::between(&a, &Document::parse(&new).unwrap()); + let text = diff.text(100); + assert!(text.contains("1/0: name \"WG\"\nCells:\n")); + assert!(!text.contains("(WG)")); + assert!(diff.text(2).ends_with(" 1/0 (WG): 1 polygon changed\n")); + let show = DisplayFilter { + cell: Some("leaf".into()), + layer: None, + }; + assert!( + diff.text_filtered(100, &show) + .ends_with(" ~ 1/0 (WG): label text \"old\" -> \"new\"\n") + ); + + let names = corpus() + .into_iter() + .find(|case| case.name == "layer name only") + .unwrap(); + let old = Document::parse(&names.old).unwrap(); + let new = Document::parse(&names.new).unwrap(); + assert!( + Comparison::between(&old, &new) + .text(0) + .contains("1/0: name \"WG\" -> \"CORE:WG\"") + ); +} + +#[test] +fn library_presence_changes_distinguish_empty_and_absent_revisions() { + let bytes = layout(&[]); + let doc = Document::parse(&bytes).unwrap(); + let absent = doc.empty_like(); + let added = Comparison::between(&absent, &doc); + assert_eq!(added.outcome().exit_code(), 1); + assert!(added.text(0).contains("library added\n")); + assert!(added.text(0).starts_with("Different · 1 library change\n")); + assert_eq!( + added.render_text(0, &Default::default(), false), + "Different\n" + ); + let removed = Comparison::between(&doc, &absent); + assert_eq!(removed.outcome().exit_code(), 1); + assert!(removed.text(0).contains("library removed\n")); +} + +#[test] +fn cli_scope_and_display_filters_have_distinct_exit_behavior() { + let directory = std::env::temp_dir().join(format!("pheon-scope-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + fs::write(&old, shared(square())).unwrap(); + fs::write(&new, shared(triangle())).unwrap(); + for (options, expected) in [ + (vec!["--cell", "CHIP"], 0), + (vec!["--top", "CHIP"], 1), + (vec!["--layer", "99/0"], 0), + (vec!["--show-cell", "missing"], 1), + (vec!["--show-layer", "99/0"], 1), + (vec!["--top", "missing"], 2), + ] { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(&options) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(expected), + "{options:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + fs::write(&old, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .args(["diff", "--top", "CHIP"]) + .arg(&old) + .arg(&new) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn cli_resolves_layer_names_before_applying_scope_and_display_filters() { + let directory = std::env::temp_dir().join(format!("pheon-named-layer-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let old = directory.join("old.gds"); + let new = directory.join("new.gds"); + let fixture = |shape, text, name| { + layout(&[ + ("CHIP", vec![shape, label((2, 0), text)]), + ( + "legend", + vec![label((900, 7), LAYER_SENTINEL), label((900, 7), name)], + ), + ]) + }; + fs::write(&old, fixture(square(), "before", "1:0:WG")).unwrap(); + fs::write(&new, fixture(triangle(), "after", "1:0:CORE")).unwrap(); + let command = || Command::new(env!("CARGO_BIN_EXE_pheon")); + let diff = |options: &[&str]| { + command() + .arg("diff") + .arg(&old) + .arg(&new) + .args(options) + .output() + .unwrap() + }; + for (option, name) in [("--layer", "WG"), ("--show-layer", "CORE")] { + for format in ["text", "json"] { + let named = diff(&[option, name, "--cell", "CHIP", "--format", format]); + let numeric = diff(&[option, "1/0", "--cell", "CHIP", "--format", format]); + assert_eq!( + named.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&named.stderr) + ); + assert_eq!(named.stdout, numeric.stdout); + } + } + + let dump = |layer| { + let output = command() + .arg("dump") + .arg(&old) + .args(["--layer", layer]) + .output() + .unwrap(); + assert!(output.status.success()); + output.stdout + }; + assert_eq!(dump("WG"), dump("1/0")); + + let unknown = diff(&["--show-layer", "wg", "--format", "json"]); + assert_eq!(unknown.status.code(), Some(2)); + let report: serde_json::Value = serde_json::from_slice(&unknown.stdout).unwrap(); + assert_eq!(report["diagnostics"][0]["code"], "command_error"); + assert_eq!( + report["diagnostics"][0]["message"], + "--show-layer unknown layer name \"wg\"" + ); + + fs::write(&new, fixture(triangle(), "after", "2:0:WG")).unwrap(); + let ambiguous = diff(&["--layer", "WG"]); + assert_eq!(ambiguous.status.code(), Some(2)); + assert_eq!( + String::from_utf8(ambiguous.stderr).unwrap().trim_end(), + "pheon: --layer ambiguous layer name \"WG\" (choose 1/0, 2/0)" + ); + assert_eq!(diff(&["--layer", "1/0"]).status.code(), Some(1)); +} + +#[test] +fn large_polygons_and_billion_instance_arrays_stay_compact() { + let mut points = vec![[0, 0], [0, 1]]; + points.extend((1..8187).map(|x| [x, 1])); + points.extend([[8186, 0], [0, 0]]); + let mut array = Vec::new(); + record(&mut array, RTAG_AREF, &[]); + ascii(&mut array, RTAG_SNAME, "leaf"); + record(&mut array, RTAG_COLROW, &[0x7f, 0xff, 0x7f, 0xff]); + xy(&mut array, &[[0, 0], [32767, 0], [0, 32767]]); + record(&mut array, RTAG_ENDEL, &[]); + let bytes = layout(&[("CHIP", vec![array]), ("leaf", vec![polygon(&points)])]); + let doc = Document::parse(&bytes).unwrap(); + assert_eq!( + doc.edges()[&("CHIP".to_string(), "leaf".to_string())], + 32767 * 32767 + ); + assert_eq!(doc.cells["CHIP"].objects.len(), 1); + assert!(doc.hierarchy(false, 8).len() < 200); + assert!(Comparison::between(&doc, &doc).outcome().is_equal()); +} + +#[test] +fn deep_hierarchy_comparison_and_rendering_are_iterative() { + let names: Vec<_> = (0..2048).map(|i| format!("cell_{i}")).collect(); + let cells = |leaf: Vec| { + names + .iter() + .enumerate() + .map(|(i, name)| { + ( + name.as_str(), + if i + 1 == names.len() { + vec![leaf.clone()] + } else { + vec![reference(&names[i + 1], 0, 1)] + }, + ) + }) + .collect::>() + }; + let a = layout(&cells(square())); + let b = layout(&cells(triangle())); + let a = Document::parse(&a).unwrap(); + let b = Document::parse(&b).unwrap(); + let diff = Comparison::between(&a, &b); + let tree = diff.hierarchy(false, 5); + assert_eq!(diff.cells.len(), 1); + assert!(tree.contains("children omitted")); + assert!(tree.len() < 500); +} diff --git a/tests/review/reports.rs b/tests/review/reports.rs new file mode 100644 index 0000000..71f06cd --- /dev/null +++ b/tests/review/reports.rs @@ -0,0 +1,900 @@ +use super::*; +use pheon::review::{ + Diagnostic, Outcome, RevisionPair, Side, + report::{ReportInputs, ReportOptions, failure_report}, + selection::{DisplayFilter, Scope}, +}; +use serde_json::{Value, json}; + +fn inputs() -> ReportInputs { + ReportInputs { + old: Some("before.gds".into()), + new: Some("after.gds".into()), + } +} + +fn report(old: &[u8], new: &[u8], options: &ReportOptions) -> Value { + let a = Document::parse(old).unwrap(); + let b = Document::parse(new).unwrap(); + Comparison::between(&a, &b).json_report(&inputs(), options) +} + +fn unknown(geometry: Vec) -> Vec { + let mut object = geometry; + object.truncate(object.len() - 4); + record(&mut object, 0x7f02, &[0, 42]); + record(&mut object, RTAG_ENDEL, &[]); + layout(&[("CHIP", vec![object])]) +} + +#[test] +fn adaptive_pair_rebases_diagnostics_and_keeps_metadata_and_graph_state() { + let mut stable = vec![square(); 512]; + stable.extend([ + reference("leaf", 0, 2), + label((900, 7), LAYER_SENTINEL), + label((900, 7), "1:0:WG"), + label((900, 8), "PHEON_LAYER_NAMES_V2"), + ]); + let mut unsupported = square(); + unsupported.truncate(unsupported.len() - 4); + record(&mut unsupported, 0x7f02, &[0, 42]); + record(&mut unsupported, RTAG_ENDEL, &[]); + let old = layout(&[ + ("leaf", vec![]), + ("changed", vec![square()]), + ("stable", stable.clone()), + ("unsupported", vec![unsupported.clone()]), + ]); + let mut new = layout(&[ + ("unsupported", vec![unsupported]), + ("stable", stable), + ("changed", vec![triangle()]), + ]); + let stable_offset = Document::parse(&new).unwrap().cells["stable"].offset; + new[stable_offset + 4..stable_offset + 6].copy_from_slice(&2026_u16.to_be_bytes()); + + let full = Comparison::between( + &Document::parse(&old).unwrap(), + &Document::parse(&new).unwrap(), + ); + let adaptive = RevisionPair::parse(&old, &new).unwrap().compare(); + let options = ReportOptions { + hierarchy: true, + ..ReportOptions::default() + }; + let actual = adaptive.json_report(&inputs(), &options); + assert_eq!(actual, full.json_report(&inputs(), &options)); + assert_eq!(actual["layer_legend"][0]["name"], "WG"); + assert_eq!(actual["status"], "incomplete"); + assert!( + actual["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|issue| { issue["code"] == "unresolved_reference" && issue["side"] == "new" }) + ); + let text = RevisionPair::parse(&old, &new).unwrap().compare_text(); + assert_eq!( + text.render_text(100, &DisplayFilter::default(), true), + full.text(100) + ); + assert_eq!(text.hierarchy(false, 8), full.hierarchy(false, 8)); + assert_eq!(text.hierarchy(true, 8), full.hierarchy(true, 8)); +} + +#[test] +fn adaptive_pair_prioritizes_old_structure_errors_over_new_frame_errors() { + let mut old = layout(&[("CHIP", vec![square()])]); + let name_offset = Document::parse(&old) + .unwrap() + .records + .iter() + .find(|r| r.tag == RTAG_LIBNAME) + .unwrap() + .offset; + old[name_offset + 2] = 0x7f; + let error = RevisionPair::parse(&old, b"bad").unwrap_err(); + assert_eq!(error.side, Some(Side::Old)); + assert!(error.message.contains("LIBNAME and UNITS")); +} + +#[test] +fn comparison_reports_survive_dropped_documents_and_overwritten_inputs() { + let comparison = { + let mut old_bytes = shared(square()); + let mut new_bytes = shared(triangle()); + let old = Document::parse(&old_bytes).unwrap(); + let new = Document::parse(&new_bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + drop(old); + drop(new); + old_bytes.fill(0); + new_bytes.fill(0); + comparison + }; + assert_eq!(comparison.outcome(), Outcome::Different); + assert!(comparison.old.present && comparison.new.present); + assert_eq!(comparison.old.units.as_ref().unwrap().len(), 16); + let mut actual = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + ..Default::default() + }, + ); + actual["producer"]["version"] = json!("test"); + let expected: Value = + serde_json::from_str(include_str!("../golden/report-geometry.json")).unwrap(); + assert_eq!(actual, expected); + assert_eq!( + comparison.hierarchy(false, 8), + include_str!("../golden/review-hierarchy.txt") + ); + assert_eq!( + comparison.hierarchy(true, 8), + include_str!("../golden/review-hierarchy.mmd") + ); + // Display choices affect evidence, not the captured outcome or full graph. + let limited = comparison.json_report( + &inputs(), + &ReportOptions { + hierarchy: true, + max_objects: Some(0), + ..Default::default() + }, + ); + assert_eq!(limited["hierarchy"], actual["hierarchy"]); + assert_eq!(limited["summary"], actual["summary"]); + assert_eq!(limited["exit_code"], 1); + assert!( + comparison + .text(0) + .contains("Omitted · 2 object changes over the limit") + ); +} + +#[test] +fn comparison_diagnostics_capture_both_sides_and_scope_mismatch_once() { + let comparison = { + let bytes = unknown(square()); + let old = Document::parse(&bytes) + .unwrap() + .selected( + &Scope { + cell: Some("CHIP".to_string()), + ..Default::default() + }, + false, + ) + .unwrap(); + let mut new = Document::parse(&bytes).unwrap(); + let comparison = Comparison::between(&old, &new); + // Changes to source documents cannot alter an existing result. + new.scope = old.scope.clone(); + new.diagnostics.clear(); + comparison + }; + assert_eq!( + comparison.outcome(), + Outcome::Incomplete { different: false } + ); + assert_eq!(comparison.diagnostics.len(), 3); + assert_eq!(comparison.diagnostics[0].side, Some(Side::Old)); + assert_eq!(comparison.diagnostics[1].side, Some(Side::New)); + assert_eq!(comparison.diagnostics[2].code, "scope_mismatch"); + assert_eq!(comparison.diagnostics[2].side, None); + let report = comparison.json_report(&inputs(), &ReportOptions::default()); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["result"], "unknown"); + assert_eq!(report["exit_code"], 2); + assert_eq!(report["scope"]["cell"], "CHIP"); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 3); + assert_eq!(report["diagnostics"][0]["side"], "old"); + assert_eq!(report["diagnostics"][1]["side"], "new"); + assert_eq!(report["diagnostics"][2]["code"], "scope_mismatch"); + let text = comparison.text(0); + assert!(text.starts_with("Incomplete\nScope cell=\"CHIP\"\nDiagnostics\n")); + let scope = text.find("\n comparison scopes differ").unwrap(); + let old = text.find("\n old:").unwrap(); + let new = text.find("\n new:").unwrap(); + assert!(scope < old && old < new); +} + +#[test] +fn reports_preserve_fixture_results_totals_and_replacement_links() { + for pair in corpus() { + let r = report(&pair.old, &pair.new, &ReportOptions::default()); + assert_eq!(r["schema_version"], 1); + assert_eq!(r["comparison_profile"], "pheon-structural-v1"); + assert_eq!(r["producer"]["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(r["status"], "complete", "{}", pair.name); + assert_eq!( + r["exit_code"], + if pair.changed_cells + pair.metadata == 0 { + 0 + } else { + 1 + } + ); + assert_eq!( + r["summary"]["changed_cells"], + pair.changed_cells.to_string() + ); + assert_eq!(r["summary"]["added_objects"], pair.added.to_string()); + assert_eq!(r["summary"]["removed_objects"], pair.removed.to_string()); + assert_eq!( + r["summary"]["layer_name_changes"], + pair.metadata.to_string() + ); + assert_eq!(r["report_selection"]["truncated"], false); + for cell in r["cell_changes"].as_array().unwrap() { + for replacement in cell["replacements"].as_array().unwrap() { + assert!( + cell["removed"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["removed"]) + ); + assert!( + cell["added"] + .as_array() + .unwrap() + .iter() + .any(|o| o["id"] == replacement["added"]) + ); + } + } + assert_eq!(r, report(&pair.old, &pair.new, &ReportOptions::default())); + } +} + +#[test] +fn incomplete_reports_distinguish_known_differences_and_include_structured_locations() { + let a = unknown(square()); + let same = report(&a, &a, &ReportOptions::default()); + assert_eq!(same["status"], "incomplete"); + assert_eq!(same["result"], "unknown"); + assert_eq!(same["exit_code"], 2); + assert_eq!(same["summary"]["counts"], "observed"); + let diagnostic = &same["diagnostics"][0]; + assert_eq!(diagnostic["code"], "unsupported_record"); + assert_eq!(diagnostic["side"], "old"); + assert_eq!(diagnostic["cell"], "CHIP"); + assert_eq!(diagnostic["layer"], json!({"number":1,"purpose":0})); + let offset: usize = diagnostic["byte_offset"].as_str().unwrap().parse().unwrap(); + assert_eq!(&a[offset + 2..offset + 4], &0x7f02_u16.to_be_bytes()); + let changed = report(&a, &unknown(triangle()), &ReportOptions::default()); + assert_eq!(changed["status"], "incomplete"); + assert_eq!(changed["result"], "different"); + assert_eq!(changed["exit_code"], 2); + assert_eq!( + changed["cell_changes"][0]["removed"][0]["fields"] + .as_array() + .unwrap() + .last() + .unwrap()["payload_hex"], + "0x002a" + ); +} + +#[test] +fn future_layer_metadata_remains_incomplete_under_geometry_scopes() { + for location in ["CHIP", "legend"] { + let bytes = if location == "CHIP" { + layout(&[( + "CHIP", + vec![square(), label((900, 7), "PHEON_LAYER_NAMES_V2")], + )]) + } else { + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), + ]) + }; + let source = Document::parse(&bytes).unwrap().diagnostics[0].offset; + for scope in [ + Scope::default(), + Scope { + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + layer: Some((999, 0)), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + ..Default::default() + }, + Scope { + top: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + Scope { + cell: Some("CHIP".into()), + layer: Some((1, 0)), + ..Default::default() + }, + ] { + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + assert_eq!(doc.diagnostics.len(), 1, "{scope:?}"); + let diagnostic = &doc.diagnostics[0]; + assert_eq!(diagnostic.code, "unsupported_metadata_version"); + assert_eq!(diagnostic.cell.as_deref(), Some(location)); + assert_eq!(diagnostic.layer, Some((900, 7))); + assert_eq!(diagnostic.offset, source); + let diff = Comparison::between(&doc, &doc); + assert_eq!(diff.outcome().exit_code(), 2); + assert!(!diff.outcome().is_equal()); + assert!(diff.text(0).starts_with("Incomplete\n")); + let json = diff.json_report(&inputs(), &ReportOptions::default()); + assert_eq!(json["status"], "incomplete"); + assert_eq!(json["result"], "unknown"); + assert_eq!(json["exit_code"], 2); + assert_eq!(json["diagnostics"].as_array().unwrap().len(), 2); + } + } +} + +#[test] +fn json_is_unbounded_by_default_and_explicit_filters_do_not_change_totals() { + let a = layout(&[("CHIP", vec![])]); + let b = layout(&[("CHIP", vec![square(); 120])]); + let full = report(&a, &b, &ReportOptions::default()); + assert_eq!( + full["cell_changes"][0]["added"].as_array().unwrap().len(), + 120 + ); + let limited = report( + &a, + &b, + &ReportOptions { + max_objects: Some(1), + ..Default::default() + }, + ); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["exit_code"], 1); + assert_eq!(limited["report_selection"]["truncated"], true); + assert_eq!(limited["report_selection"]["limited_objects"], "119"); + assert_eq!(limited["report_selection"]["emitted_objects"], "1"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: Some("absent".into()), + layer: None, + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["cell_changes"], json!([])); + assert_eq!(filtered["report_selection"]["filtered_cells"], "1"); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + let filtered = report( + &a, + &b, + &ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + ..Default::default() + }, + ); + assert_eq!(filtered["summary"], full["summary"]); + assert_eq!(filtered["report_selection"]["filtered_objects"], "120"); + assert_eq!(filtered["cell_changes"][0]["added"], json!([])); +} + +#[test] +fn streamed_report_keeps_object_ids_and_replacements_across_layer_filter_gaps() { + let a = layout(&[("CHIP", vec![square(), label((2, 0), "old")])]); + let b = layout(&[("CHIP", vec![triangle(), label((2, 0), "new")])]); + let diff = Comparison::between(&Document::parse(&a).unwrap(), &Document::parse(&b).unwrap()); + let options = ReportOptions { + filter: DisplayFilter { + cell: None, + layer: Some((2, 0)), + }, + max_objects: Some(2), + ..Default::default() + }; + let mut bytes = Vec::new(); + diff.write_json_report(&mut bytes, &inputs(), &options) + .unwrap(); + let actual: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(actual, diff.json_report(&inputs(), &options)); + assert_eq!(actual["report_selection"]["filtered_objects"], "2"); + let cell = &actual["cell_changes"][0]; + assert_eq!(cell["added"].as_array().unwrap().len(), 1); + assert_eq!(cell["removed"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"].as_array().unwrap().len(), 1); + assert_eq!(cell["replacements"][0]["removed"], "c0:old:1"); + assert_eq!(cell["replacements"][0]["added"], "c0:new:1"); +} + +#[test] +fn typed_evidence_preserves_defaults_properties_bytes_and_real_encodings() { + let mut a = reference("leaf", 0, 1); + a.truncate(a.len() - 4); + record(&mut a, RTAG_MAG, &0x4110000000000000_u64.to_be_bytes()); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_PROPATTR, &17_u16.to_be_bytes()); + record(&mut a, RTAG_PROPVALUE, &[0xff, 0xfe]); + record(&mut a, RTAG_ENDEL, &[]); + let mut b = a.clone(); + let mag = Document::parse(&layout(&[("CHIP", vec![a.clone()]), ("leaf", vec![])])) + .unwrap() + .records + .into_iter() + .find(|r| r.tag == RTAG_MAG) + .unwrap() + .offset; + // Replace the REAL8 payload within the object, retaining the same decoded value. + let within = a + .windows(8) + .position(|v| v == 0x4110000000000000_u64.to_be_bytes()) + .unwrap(); + b[within..within + 8].copy_from_slice(&0x4201000000000000_u64.to_be_bytes()); + let a = layout(&[("CHIP", vec![a]), ("leaf", vec![])]); + let b = layout(&[("CHIP", vec![b]), ("leaf", vec![])]); + let before = Document::parse(&a).unwrap(); + let after = Document::parse(&b).unwrap(); + assert!(before.canonical_dump().contains("mag=1.0 property")); + assert!( + after + .canonical_dump() + .contains("mag=1.0 raw=0x4201000000000000") + ); + let text = Comparison::between(&before, &after).text(100); + assert!(text.contains("mag 1.0 raw=0x4110000000000000 -> 1.0 raw=0x4201000000000000")); + let omitted = layout(&[("CHIP", vec![reference("leaf", 0, 1)]), ("leaf", vec![])]); + let omitted = Document::parse(&omitted).unwrap(); + let text = Comparison::between(&before, &omitted).text(100); + assert!(text.contains("mag 1.0 -> absent")); + assert!(!text.contains("raw=")); + let r = report(&a, &b, &ReportOptions::default()); + let replacement = &r["cell_changes"][0]["replacements"][0]; + assert_eq!(replacement["match_rule"], "unique_kind_target"); + let change = &replacement["field_changes"][0]; + assert_eq!(change["tag"], RTAG_MAG); + assert_eq!(change["before"]["value"], change["after"]["value"]); + assert_ne!( + change["before"]["payload_hex"], + change["after"]["payload_hex"] + ); + let removed = &r["cell_changes"][0]["removed"][0]; + assert_eq!(removed["properties"].as_array().unwrap().len(), 2); + assert_eq!(removed["properties"][0]["attribute"], 17); + assert_eq!(removed["properties"][0]["value"], Value::Null); + assert_eq!(removed["properties"][0]["payload_hex"], "0xfffe"); + let offset: usize = removed["source"]["byte_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert!(offset < mag); + let end: usize = removed["source"]["end_offset"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert_eq!(&a[end - 4..end], &[0, 4, 17, 0]); + + let mut path = square(); + path[2..4].copy_from_slice(&RTAG_PATH.to_be_bytes()); + let mut width = path.clone(); + width.truncate(width.len() - 4); + record(&mut width, RTAG_WIDTH, &0_i32.to_be_bytes()); + record(&mut width, RTAG_ENDEL, &[]); + let r = report( + &layout(&[("CHIP", vec![path])]), + &layout(&[("CHIP", vec![width])]), + &ReportOptions::default(), + ); + let change = &r["cell_changes"][0]["replacements"][0]["field_changes"][0]; + assert_eq!(change["tag"], RTAG_WIDTH); + assert_eq!(change["before"], Value::Null); + assert_eq!(change["after"]["value"], json!([0])); +} + +#[test] +fn hierarchy_reports_shared_ancestors_and_symbolic_arrays() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let r = report(&shared(square()), &shared(triangle()), &options); + let graph = &r["hierarchy"]; + assert_eq!(graph["nodes"].as_array().unwrap().len(), 4); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "CHIP") + .unwrap()["change"], + "affected" + ); + assert_eq!( + graph["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "leaf") + .unwrap()["change"], + "direct" + ); + let edge = graph["edges"] + .as_array() + .unwrap() + .iter() + .find(|e| e["parent"] == "right") + .unwrap(); + assert_eq!(edge["before_count"], "3"); + assert_eq!(edge["reference_changed"], false); + let bytes = shared(square()); + let scope = Scope { + cell: Some("CHIP".into()), + ..Default::default() + }; + let doc = Document::parse(&bytes) + .unwrap() + .selected(&scope, false) + .unwrap(); + let r = Comparison::between(&doc, &doc).json_report(&inputs(), &options); + assert_eq!(r["scope"]["cell"], "CHIP"); + assert_eq!( + r["hierarchy"]["nodes"] + .as_array() + .unwrap() + .iter() + .find(|n| n["cell"] == "left") + .unwrap()["after"], + "outside_scope" + ); + let dangling = layout(&[("CHIP", vec![reference("missing", 0, 1)])]); + let r = report(&dangling, &dangling, &options); + assert_eq!(r["diagnostics"][0]["code"], "unresolved_reference"); + assert_eq!(r["diagnostics"][0]["target"], "missing"); + assert_eq!(r["hierarchy"]["nodes"][1]["after"], "unresolved"); + let cycle = layout(&[("CHIP", vec![reference("CHIP", 0, 1)])]); + assert_eq!( + report(&cycle, &cycle, &options)["diagnostics"][0]["code"], + "hierarchy_cycle" + ); +} + +#[test] +fn library_attributes_empty_cells_units_and_absence_are_explicit() { + let a = layout(&[]); + let b = layout(&[("empty", vec![])]); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["cell_changes"][0]["before"], Value::Null); + assert_eq!(r["cell_changes"][0]["after"]["attributes"], json!([])); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); + let removed = report(&b, &a, &ReportOptions::default()); + assert_eq!( + removed["cell_changes"][0]["before"]["attributes"], + json!([]) + ); + assert_eq!(removed["cell_changes"][0]["after"], Value::Null); + assert_eq!(removed["cell_changes"][0]["attributes_changed"], true); + let mut b = b; + let end = Document::parse(&b).unwrap().cells["empty"].end_offset; + let mut attribute = Vec::new(); + record(&mut attribute, RTAG_STRCLASS, &[0, 1]); + b.splice(end..end, attribute); + let r = report(&layout(&[("empty", vec![])]), &b, &ReportOptions::default()); + assert_eq!( + r["cell_changes"][0]["after"]["attributes"][0]["value"], + json!([1]) + ); + assert_eq!(r["cell_changes"][0]["attributes_changed"], true); + assert_eq!(r["inputs"]["old"]["units"]["encoding"], "real8"); + let doc = Document::parse(&a).unwrap(); + let absent = doc.empty_like(); + let r = Comparison::between(&absent, &doc).json_report(&inputs(), &ReportOptions::default()); + assert_eq!(r["inputs"]["old"]["present"], false); + assert_eq!(r["inputs"]["old"]["units"], Value::Null); + assert_eq!( + r["library_changes"], + json!([{"field":"presence","tag":null,"before":false,"after":true}]) + ); + let mut b = a.clone(); + let units = doc + .records + .iter() + .find(|r| r.tag == RTAG_UNITS) + .unwrap() + .offset; + b[units + 12..units + 20] + .copy_from_slice(&pheon::basic::encode_real8(2e-9).unwrap().to_be_bytes()); + let r = report(&a, &b, &ReportOptions::default()); + assert_eq!(r["library_changes"][0]["tag"], RTAG_UNITS); + assert_ne!(r["inputs"]["old"]["units"], r["inputs"]["new"]["units"]); +} + +#[test] +fn reports_keep_extreme_coordinates_unsigned_layers_and_timestamp_equality() { + let a = layout(&[("quoted\"\\\n日本", vec![])]); + let mut shape = polygon(&[ + [i32::MIN, i32::MIN], + [i32::MAX, i32::MIN], + [i32::MAX, i32::MAX], + [i32::MIN, i32::MAX], + [i32::MIN, i32::MIN], + ]); + shape[8..10].copy_from_slice(&u16::MAX.to_be_bytes()); + let b = layout(&[("quoted\"\\\n日本", vec![shape])]); + let r = report(&a, &b, &ReportOptions::default()); + let object = &r["cell_changes"][0]["added"][0]; + assert_eq!(object["layer"]["number"], u16::MAX); + assert_eq!( + object["bounds"], + json!([i32::MIN, i32::MIN, i32::MAX, i32::MAX]) + ); + let encoded = serde_json::to_string(&r).unwrap(); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), r); + let mut times = b.clone(); + let doc = Document::parse(&b).unwrap(); + for record in doc + .records + .iter() + .filter(|r| matches!(r.tag, RTAG_BGNLIB | RTAG_BGNSTR)) + { + times[record.offset + 4..record.offset + 28].fill(1); + } + let r = report(&b, ×, &ReportOptions::default()); + assert_eq!(r["result"], "equal"); + assert_eq!(r["exit_code"], 0); +} + +#[cfg(unix)] +#[test] +fn cli_preserves_non_utf8_path_bytes_and_literal_option_names() { + use std::os::unix::ffi::OsStringExt; + let directory = std::env::temp_dir().join(format!("pheon json paths {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let path = directory.join(std::ffi::OsString::from_vec(b"file\xff.gds".to_vec())); + fs::write(&path, layout(&[])).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .args(["--format", "json", "--"]) + .arg(&path) + .arg(&path) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + let r: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(r["inputs"]["old"]["path"], Value::Null); + let expected = "0x".to_string() + + &path + .as_os_str() + .as_encoded_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + assert_eq!(r["inputs"]["old"]["path_encoded_hex"], expected); + for name in ["--format", "json"] { + fs::write(directory.join(name), layout(&[])).unwrap(); + } + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .current_dir(&directory) + .args(["diff", "--", "--format", "json"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(0)); + assert_eq!(output.stdout, b"Equal\n"); +} + +#[test] +fn machine_reports_match_versioned_goldens() { + let options = ReportOptions { + hierarchy: true, + ..Default::default() + }; + let equal = layout(&[("CHIP", vec![square()])]); + let metadata = corpus() + .into_iter() + .find(|p| p.name == "layer name only") + .unwrap(); + let samples = [ + ("equal", report(&equal, &equal, &ReportOptions::default())), + ( + "geometry", + report(&shared(square()), &shared(triangle()), &options), + ), + ( + "metadata", + report(&metadata.old, &metadata.new, &ReportOptions::default()), + ), + ( + "incomplete", + report( + &unknown(square()), + &unknown(square()), + &ReportOptions::default(), + ), + ), + ( + "failed", + failure_report( + &inputs(), + &Scope::default(), + &ReportOptions::default(), + &Diagnostic::new("invalid_input", "truncated record", Some(Side::New)), + ), + ), + ]; + for (name, mut actual) in samples { + actual["producer"]["version"] = json!("test"); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join(format!("tests/golden/report-{name}.json")); + if std::env::var_os("PHEON_UPDATE_REPORT_GOLDENS").is_some() { + fs::write(&path, serde_json::to_string_pretty(&actual).unwrap() + "\n").unwrap(); + } + let expected: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(actual, expected, "{name}"); + } +} + +#[test] +fn cli_emits_json_for_success_input_errors_scope_errors_and_argument_errors() { + let directory = std::env::temp_dir().join(format!("pheon json {}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + let _cleanup = Cleanup(directory.clone()); + let a = directory.join("before file.gds"); + let b = directory.join("after file.gds"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); + let command = |extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(["--format", "json"]) + .args(extra) + .output() + .unwrap(); + assert!(output.stderr.is_empty(), "{:?}", output.stderr); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + output.status.code().unwrap(), + report["exit_code"].as_i64().unwrap() as i32 + ); + report + }; + let check_text_failure = |report: &Value, extra: &[&str]| { + let output = Command::new(env!("CARGO_BIN_EXE_pheon")) + .arg("diff") + .arg(&a) + .arg(&b) + .args(extra) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + format!( + "pheon: {}\n", + report["diagnostics"][0]["message"].as_str().unwrap() + ) + ); + }; + let full = command(&["--hierarchy", "graph"]); + assert_eq!(full["result"], "different"); + assert!(full["hierarchy"].is_object()); + let large = |mut bytes: Vec| { + bytes.resize(3 * 1024 * 1024, 0); + bytes + }; + let mut large_old = large(shared(square())); + let mut large_new = large(shared(triangle())); + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&["--hierarchy", "graph"]), full); + *large_old.last_mut().unwrap() = 1; + *large_new.last_mut().unwrap() = 1; + fs::write(&a, &large_old).unwrap(); + fs::write(&b, &large_new).unwrap(); + assert_eq!(command(&[])["diagnostics"][0]["side"], "old"); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, shared(triangle())).unwrap(); + let limited = command(&["--max-changes", "0"]); + assert_eq!(limited["summary"], full["summary"]); + assert_eq!(limited["report_selection"]["emitted_objects"], "0"); + for extra in [ + &["--top", "missing"][..], + &["--unknown"], + &["--max-changes", "x"], + &["--hierarchy", "mermaid"], + &["--depth", "3"], + ] { + let failure = command(extra); + assert_eq!(failure["status"], "failed"); + assert_eq!(failure["result"], "unknown"); + assert_eq!(failure["summary"], Value::Null); + if extra.first() == Some(&"--top") { + assert_eq!(failure["diagnostics"][0]["code"], "invalid_scope"); + assert_eq!(failure["diagnostics"][0]["side"], Value::Null); + check_text_failure(&failure, extra); + } + } + fs::write(&b, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "new"); + assert_eq!(invalid["inputs"]["new"]["path"], b.to_str().unwrap()); + check_text_failure(&invalid, &[]); + fs::write(&a, b"invalid").unwrap(); + let invalid = command(&[]); + assert_eq!(invalid["diagnostics"][0]["code"], "invalid_input"); + assert_eq!(invalid["diagnostics"][0]["side"], "old"); + check_text_failure(&invalid, &[]); + fs::write(&a, shared(square())).unwrap(); + fs::remove_file(&b).unwrap(); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "new"); + check_text_failure(&missing, &[]); + fs::remove_file(&a).unwrap(); + let missing = command(&[]); + assert_eq!(missing["diagnostics"][0]["code"], "io_error"); + assert_eq!(missing["diagnostics"][0]["side"], "old"); + check_text_failure(&missing, &[]); + fs::write(&a, shared(square())).unwrap(); + fs::write(&b, fs::read(&a).unwrap()).unwrap(); + assert_eq!(command(&[])["result"], "equal"); + fs::write(&b, unknown(square())).unwrap(); + assert_eq!(command(&[])["status"], "incomplete"); + fs::write( + &b, + layout(&[ + ("CHIP", vec![square()]), + ("legend", vec![label((900, 7), "PHEON_LAYER_NAMES_V2")]), + ]), + ) + .unwrap(); + for flags in [ + &["--layer", "1/0"][..], + &["--top", "CHIP"], + &["--cell", "CHIP", "--layer", "1/0"], + ] { + let report = command(flags); + assert_eq!(report["status"], "incomplete"); + assert_eq!(report["exit_code"], 2); + assert_eq!( + report["diagnostics"][0]["code"], + "unsupported_metadata_version" + ); + } +} diff --git a/tests/selective_reads.rs b/tests/selective_reads.rs new file mode 100644 index 0000000..cd06348 --- /dev/null +++ b/tests/selective_reads.rs @@ -0,0 +1,798 @@ +mod common; + +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi}; +use arrow_array::{ + Array, BooleanArray, Float64Array, Int16Array, Int32Array, ListArray, StringArray, StructArray, + UInt32Array, UInt64Array, +}; +use common::{library_header, record}; +use pheon::{ + basic::encode_real8, + last_error_message, + library::{read_library, read_selected_library, scan_library}, + read_cells_bytes, read_path, records, scan_bytes, +}; +use std::{ + ffi::{CStr, CString}, + ptr, +}; + +fn xy(values: &[i32]) -> Vec { + values.iter().flat_map(|v| v.to_be_bytes()).collect() +} + +fn fixture() -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_XY, &xy(&[3, -4])); + record(&mut bytes, records::RTAG_PROPATTR, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, b"value\0"); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + record(&mut bytes, records::RTAG_STRANS, &[0x80, 0]); + record( + &mut bytes, + records::RTAG_MAG, + &encode_real8(2.0).unwrap().to_be_bytes(), + ); + record( + &mut bytes, + records::RTAG_ANGLE, + &encode_real8(90.0).unwrap().to_be_bytes(), + ); + record(&mut bytes, records::RTAG_COLROW, &[0, 2, 0, 3]); + record(&mut bytes, records::RTAG_XY, &xy(&[-5, -7, 15, -7, -5, 23])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &1_i16.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &0_i16.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +fn list_struct(array: &StructArray, name: &str, row: usize) -> StructArray { + let list = array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + list.value(row) + .as_any() + .downcast_ref::() + .unwrap() + .clone() +} + +fn aref_fixture(origin: [i32; 2], column: [i32; 2], row: [i32; 2], counts: [i16; 2]) -> Vec { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + record(&mut bytes, records::RTAG_AREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"leaf"); + let counts: Vec = counts.into_iter().flat_map(i16::to_be_bytes).collect(); + record(&mut bytes, records::RTAG_COLROW, &counts); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[origin[0], origin[1], column[0], column[1], row[0], row[1]]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + bytes +} + +#[test] +fn aref_pitches_use_wide_coordinate_differences_in_full_reads_and_scans() { + let bytes = aref_fixture( + [i32::MIN, i32::MAX], + [i32::MAX - 1, i32::MIN + 1], + [i32::MIN, i32::MIN + 1], + [2, 2], + ); + let full_cells = list_struct(&read_library(&bytes).unwrap().1, "cells", 0); + let scan_cells = list_struct(&scan_library(&bytes).unwrap().1, "cells", 0); + for refs in [ + list_struct(&full_cells, "arefs", 0), + list_struct(&scan_cells, "refs", 0), + ] { + let column = u64_value(&refs, "xy0", 0); + let row = u64_value(&refs, "xy1", 0); + assert_eq!((column >> 32) as i32, i32::MAX); + assert_eq!(column as i32, -i32::MAX); + assert_eq!((row >> 32) as i32, 0); + assert_eq!(row as i32, -i32::MAX); + } +} + +#[test] +fn aref_invalid_counts_and_unrepresentable_pitches_return_errors() { + let check_array_error = |bytes: &[u8], message: &str| { + let ranges = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for result in [ + read_library(bytes), + scan_library(bytes), + read_selected_library(bytes, &ranges), + ] { + let (_, error) = result.unwrap_err(); + assert!(matches!(error, pheon::basic::ErrType::Failed(reason) + if reason.contains(message))); + } + assert!( + pheon::review::Document::parse(bytes) + .unwrap_err() + .contains(message) + ); + }; + for counts in [[0, 1], [-1, 1], [1, 0], [1, -1]] { + let bytes = aref_fixture([0, 0], [10, 0], [0, 10], counts); + check_array_error(&bytes, "AREF column and row counts must be positive"); + } + + let mut bytes = aref_fixture([0, 0], [10, 0], [0, 10], [1, 2]); + let xy_offset = pheon::review::Document::parse(&bytes) + .unwrap() + .records + .iter() + .find(|record| record.tag == records::RTAG_XY) + .unwrap() + .offset; + let mut replacement = Vec::new(); + record(&mut replacement, records::RTAG_XY, &[0; 16]); + bytes.splice(xy_offset..xy_offset + 28, replacement); + check_array_error(&bytes, "AREF needs exactly three XY coordinate pairs"); + + let bytes = aref_fixture([i32::MIN, 0], [i32::MAX, 0], [i32::MIN, 10], [1, 2]); + assert!(read_library(&bytes).is_err()); + assert!(scan_library(&bytes).is_err()); +} + +#[test] +fn fractional_aref_pitches_are_rejected_by_all_arrow_readers() { + for (column, row, counts) in [ + ([10, 0], [0, 20], [3, 2]), + ([-10, 0], [0, 20], [3, 2]), + ([0, 10], [20, 0], [3, 2]), + ([20, 0], [0, 10], [2, 3]), + ([20, 0], [-10, 0], [2, 3]), + ] { + let bytes = aref_fixture([0, 0], column, row, counts); + let range = [(library_header().len() as u64, bytes.len() as u64 - 4)]; + for error in [ + read_library(&bytes).unwrap_err().1, + scan_library(&bytes).unwrap_err().1, + read_selected_library(&bytes, &range).unwrap_err().1, + ] { + assert!(format!("{error:?}").contains("not an integer database-unit increment")); + } + } + + let bytes = aref_fixture([0, 0], [10, 0], [0, 20], [3, 2]); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + -1 + ); + let range = [library_header().len() as u64, bytes.len() as u64 - 4]; + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + range.as_ptr(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + let path = + std::env::temp_dir().join(format!("pheon-fractional-aref-{}.gds", std::process::id())); + struct Cleanup(std::path::PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + let _cleanup = Cleanup(path.clone()); + std::fs::write(&path, &bytes).unwrap(); + let path = CString::new(path.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { read_path(path.as_ptr(), &mut array, &mut schema) }, + -1 + ); + let message = unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap(); + assert!(message.contains("not an integer database-unit increment")); + assert!(array.is_released()); + + // The object reviewer keeps the exact endpoints, including fractional pitches. + let mut bytes = bytes; + bytes.truncate(bytes.len() - 4); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"leaf"); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let doc = pheon::review::Document::parse(&bytes).unwrap(); + assert_eq!( + pheon::review::Comparison::between(&doc, &doc).outcome().exit_code(), + 0 + ); +} + +#[test] +fn rectangle_detection_handles_extreme_coordinates_and_preserves_other_polygons() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + let wide = [ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MAX, + i32::MIN, + i32::MIN, + ]; + let reversed = [ + i32::MIN, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MIN, + ]; + let diagonal = [0, 0, 10, 10, 0, 10, 10, 0, 0, 0]; + let degenerate = [0, 0, 0, 0, 10, 0, 10, 0, 0, 0]; + for points in [&wide, &reversed, &diagonal, °enerate] { + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + record(&mut bytes, records::RTAG_XY, &xy(points)); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let full = read_library(&bytes).unwrap().1; + let scan = scan_library(&bytes).unwrap().1; + assert_eq!( + full, + read_selected_library(&bytes, &ranges(&scan)).unwrap().1 + ); + let cells = list_struct(&full, "cells", 0); + let rects = list_struct(&cells, "rect_batches", 0); + let values = rects + .column_by_name("rects") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + &[ + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX, + i32::MIN, + i32::MIN, + i32::MAX, + i32::MAX + ] + ); + let boundaries = list_struct(&cells, "boundary_batches", 0); + let values = boundaries + .column_by_name("vertices") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let values = values.as_any().downcast_ref::().unwrap(); + assert_eq!( + values.values().as_ref(), + [&diagonal[..8], °enerate[..8]].concat() + ); +} + +#[test] +fn paths_preserve_missing_explicit_zero_and_negative_widths() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for width in [None, Some(0_i32), Some(-100)] { + record(&mut bytes, records::RTAG_PATH, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + if let Some(width) = width { + record(&mut bytes, records::RTAG_WIDTH, &width.to_be_bytes()); + } + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0, 10, 10])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, full) = read_library(&bytes).unwrap(); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + let paths = list_struct(&cells, "paths", 0); + let widths = paths + .column_by_name("width") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + widths.iter().collect::>(), + [None, Some(0), Some(-100)] + ); +} + +fn u64_value(array: &StructArray, name: &str, row: usize) -> u64 { + array + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row) +} + +fn ranges(scan: &StructArray) -> Vec<(u64, u64)> { + let cells = list_struct(scan, "cells", 0); + (0..cells.len()) + .map(|i| { + ( + u64_value(&cells, "struct_start_offset", i), + u64_value(&cells, "struct_end_offset", i), + ) + }) + .collect() +} + +#[test] +fn scanned_ranges_reproduce_the_full_library_including_offsets() { + let bytes = fixture(); + let (_, full) = read_library(&bytes).unwrap(); + let (scan_remaining, scan) = scan_library(&bytes).unwrap(); + assert!(scan_remaining.is_empty()); + let ranges = ranges(&scan); + assert_eq!(ranges.len(), 2); + for &(start, end) in &ranges { + assert_eq!(&bytes[start as usize..start as usize + 4], &[0, 28, 5, 2]); + assert_eq!(&bytes[end as usize - 4..end as usize], &[0, 4, 7, 0]); + } + let (remaining, selected) = read_selected_library(&bytes, &ranges).unwrap(); + assert_eq!(remaining, bytes); // Random access leaves the caller's cursor unchanged. + assert_eq!(full, selected); + let cells = list_struct(&full, "cells", 0); + assert_eq!(u64_value(&cells, "file_offset", 0), ranges[0].0 + 28 + 8); +} + +#[test] +fn scan_preserves_reference_transforms_and_skips_geometry() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let cells = list_struct(&scan, "cells", 0); + let refs = list_struct(&cells, "refs", 0); + assert_eq!(refs.len(), 2); + assert_eq!(list_struct(&cells, "refs", 1).len(), 0); + let target = refs + .column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(target.values().as_ref(), &[1, 1]); + assert_eq!( + u64_value(&refs, "xy", 0), + (3_u64 << 32) | (-4_i32 as u32 as u64) + ); + assert_eq!(u64_value(&refs, "xy0", 0), 0); + assert_eq!(u64_value(&refs, "xy1", 0), 0); + let counts = refs + .column_by_name("counts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(counts.values().as_ref(), &[(1 << 16) | 1, (2 << 16) | 3]); + assert!( + refs.column_by_name("invert_y") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1) + ); + assert_eq!( + refs.column_by_name("scale") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + 2.0 + ); + assert_eq!( + refs.column_by_name("angle_rad") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(1), + std::f64::consts::FRAC_PI_2 + ); + assert_eq!(u64_value(&refs, "xy0", 1), 10_u64 << 32); + assert_eq!(u64_value(&refs, "xy1", 1), 10); +} + +#[test] +fn selections_support_subsets_reordering_and_empty_results() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (_, leaf) = read_selected_library(&bytes, &ranges[1..]).unwrap(); + let leaf_cells = list_struct(&leaf, "cells", 0); + assert_eq!(leaf_cells.len(), 1); + let leaf_names = leaf + .column_by_name("cell_names") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!( + leaf_names + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "leaf" + ); + let (_, reordered) = read_selected_library(&bytes, &[ranges[1], ranges[0]]).unwrap(); + let reordered_cells = list_struct(&reordered, "cells", 0); + assert_eq!( + u64_value(&reordered_cells, "file_offset", 0), + u64_value(&leaf_cells, "file_offset", 0) + ); + let refs = list_struct(&reordered_cells, "sref_props", 1); + assert_eq!( + refs.column_by_name("target") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + let (_, empty) = read_selected_library(&bytes, &[]).unwrap(); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); +} + +#[test] +fn id_tables_preserve_encounter_order_and_selection_local_ids() { + let mut bytes = library_header(); + for (name, layer, datatype) in [("zeta", 9_i16, 4_i16), ("middle", 2, 8), ("alpha\0", 9, 4)] { + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, name.as_bytes()); + if name == "zeta" { + record(&mut bytes, records::RTAG_SREF, &[]); + record(&mut bytes, records::RTAG_SNAME, b"alpha\0"); + record(&mut bytes, records::RTAG_XY, &xy(&[0, 0])); + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_BOUNDARY, &[]); + record(&mut bytes, records::RTAG_LAYER, &layer.to_be_bytes()); + record(&mut bytes, records::RTAG_DATATYPE, &datatype.to_be_bytes()); + record( + &mut bytes, + records::RTAG_XY, + &xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]), + ); + record(&mut bytes, records::RTAG_ENDEL, &[]); + record(&mut bytes, records::RTAG_ENDSTR, &[]); + } + record(&mut bytes, records::RTAG_ENDLIB, &[]); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, full) = read_library(&bytes).unwrap(); + let cell_ranges = ranges(&scan); + + let check = |library: &StructArray, names: &[&str], layers: &[u32], ids: &[u32]| { + let values = |name| { + library + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + }; + let name_values = values("cell_names"); + assert_eq!( + name_values + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(Option::unwrap) + .collect::>(), + names + ); + let cells = list_struct(library, "cells", 0); + assert_eq!( + cells + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + ids + ); + if library.column_by_name("layers").is_some() { + let layer_values = values("layers"); + assert_eq!( + layer_values + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + layers + ); + } + }; + let layer_a = (9 << 16) | 4; + let layer_b = (2 << 16) | 8; + check( + &full, + &["zeta", "alpha", "middle"], + &[layer_a, layer_b], + &[0, 2, 1], + ); + check(&scan, &["zeta", "alpha", "middle"], &[], &[0, 2, 1]); + let (_, all_selected) = read_selected_library(&bytes, &cell_ranges).unwrap(); + assert_eq!(all_selected, full); + let (_, subset) = read_selected_library(&bytes, &cell_ranges[1..]).unwrap(); + check(&subset, &["middle", "alpha"], &[layer_b, layer_a], &[0, 1]); + let (_, reversed) = + read_selected_library(&bytes, &[cell_ranges[2], cell_ranges[1], cell_ranges[0]]).unwrap(); + check( + &reversed, + &["alpha", "middle", "zeta"], + &[layer_a, layer_b], + &[0, 1, 2], + ); +} + +#[test] +fn streamed_and_buffered_properties_preserve_duplicates_order_and_empty_lists() { + let mut bytes = library_header(); + record(&mut bytes, records::RTAG_BGNSTR, &[0; 24]); + record(&mut bytes, records::RTAG_STRNAME, b"top\0"); + for (kind, properties) in [ + (records::RTAG_BOUNDARY, true), + (records::RTAG_PATH, true), + (records::RTAG_PATH, false), + ] { + record(&mut bytes, kind, &[]); + record(&mut bytes, records::RTAG_LAYER, &[0, 1]); + record(&mut bytes, records::RTAG_DATATYPE, &[0, 0]); + let vertices = if kind == records::RTAG_BOUNDARY { + xy(&[0, 0, 10, 0, 10, 10, 0, 10, 0, 0]) + } else { + xy(&[0, 0, 10, 10]) + }; + record(&mut bytes, records::RTAG_XY, &vertices); + if properties { + for (key, value) in [(7_i16, b"first\0".as_slice()), (7, b"second"), (9, b"")] { + record(&mut bytes, records::RTAG_PROPATTR, &key.to_be_bytes()); + record(&mut bytes, records::RTAG_PROPVALUE, value); + } + } + record(&mut bytes, records::RTAG_ENDEL, &[]); + } + record(&mut bytes, records::RTAG_ENDSTR, &[]); + record(&mut bytes, records::RTAG_ENDLIB, &[]); + + let (remaining, full) = read_library(&bytes).unwrap(); + assert!(remaining.is_empty()); + let (_, scan) = scan_library(&bytes).unwrap(); + let (_, selected) = read_selected_library(&bytes, &ranges(&scan)).unwrap(); + assert_eq!(selected, full); + let cells = list_struct(&full, "cells", 0); + let boundaries = list_struct(&cells, "boundary_props", 0); + let paths = list_struct(&cells, "paths", 0); + assert_eq!(boundaries.len(), 1); + assert_eq!(paths.len(), 2); + for elements in [&boundaries, &paths] { + let properties = list_struct(elements, "properties", 0); + let keys = properties + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(keys.values().as_ref(), &[7, 7, 9]); + let values = properties + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + values.iter().collect::>(), + [Some("first"), Some("second"), Some("")] + ); + } + let empty_properties = paths + .column_by_name("properties") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!empty_properties.is_null(1)); + assert_eq!(empty_properties.value_length(1), 0); +} + +#[test] +fn selected_ranges_must_cover_exactly_one_complete_cell() { + let bytes = fixture(); + let (_, scan) = scan_library(&bytes).unwrap(); + let ranges = ranges(&scan); + let (start, end) = ranges[0]; + for range in [ + (end, start), + (start, u64::MAX), + (start, end - 1), + (start + 4, end), + (start, end + 4), + (start, ranges[1].1), + ] { + assert!( + read_selected_library(&bytes, &[range]).is_err(), + "accepted {range:?}" + ); + } +} + +#[test] +fn ffi_accepts_null_empty_ranges_and_exports_both_schemas() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 0, + &mut array, + &mut schema, + ) + }, + 0 + ); + let empty = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(list_struct(&empty, "cells", 0).len(), 0); + + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { scan_bytes(bytes.as_ptr(), bytes.len(), &mut array, &mut schema) }, + 0 + ); + let scan = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + let ranges: Vec = ranges(&scan) + .into_iter() + .flat_map(|(a, b)| [a, b]) + .collect(); + let mut array = FFI_ArrowArray::empty(); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ranges.as_ptr(), + ranges.len() / 2, + &mut array, + &mut schema, + ) + }, + 0 + ); + let selected = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(selected, read_library(&bytes).unwrap().1); + assert!(last_error_message().is_null()); +} + +#[test] +fn ffi_rejects_null_nonempty_inputs_and_overflowing_range_counts() { + let bytes = fixture(); + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 0, &mut array, &mut schema) }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("scan failed") + ); + assert_eq!( + unsafe { scan_bytes(ptr::null(), 1, &mut array, &mut schema) }, + -1 + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + 1, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("ranges pointer was null") + ); + assert_eq!( + unsafe { + read_cells_bytes( + bytes.as_ptr(), + bytes.len(), + ptr::null(), + usize::MAX, + &mut array, + &mut schema, + ) + }, + -1 + ); + assert!( + unsafe { CStr::from_ptr(last_error_message()) } + .to_str() + .unwrap() + .contains("range count is too large") + ); + assert!(array.is_released()); +} diff --git a/tests/writing.rs b/tests/writing.rs new file mode 100644 index 0000000..dc07994 --- /dev/null +++ b/tests/writing.rs @@ -0,0 +1,192 @@ +use arrow_array::Array; +use pheon::{ + basic::{pack_bitarray, pack_int2, pack_int4, pack_real8}, + library::{FileHeader, read_library}, + record::{Int2Array, Int4Array, Record, RecordData, RecordHeader}, + records, +}; +use std::io::{self, Write}; + +#[test] +fn record_headers_include_their_own_size() { + let mut bytes = Vec::new(); + assert_eq!(records::HEADER::write(&mut bytes, &600).unwrap(), 6); + assert_eq!(bytes, [0, 6, 0, 2, 2, 88]); + let (remaining, value) = records::HEADER::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(value, 600); + let maximum = RecordHeader { + data_size: u16::MAX - 4, + tag: 0x1003, + }; + assert_eq!(maximum.pack_into().unwrap(), [0xff, 0xff, 0x10, 0x03]); +} + +#[test] +fn integer_array_records_round_trip() { + let mut bytes = Vec::new(); + records::COLROW::write(&mut bytes, &[2, 3]).unwrap(); + let (remaining, values) = records::COLROW::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(values.collect::>(), [2, 3]); + + let mut bytes = Vec::new(); + let values = [i32::MIN, -1, 0, i32::MAX]; + records::XY::write(&mut bytes, &values).unwrap(); + let (remaining, parsed) = records::XY::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed.collect::>(), values); +} + +#[test] +fn scalar_packers_write_only_their_prefix_and_check_short_buffers() { + type Case = (fn(&mut [u8]), &'static [u8]); + let cases: &[Case] = &[ + (|bytes| pack_bitarray(bytes, &[true; 16]), &[0xff, 0xff]), + (|bytes| pack_int2(bytes, -2), &[0xff, 0xfe]), + (|bytes| pack_int4(bytes, -2), &[0xff, 0xff, 0xff, 0xfe]), + ( + |bytes| pack_real8(bytes, 1.0).unwrap(), + &[0x41, 0x10, 0, 0, 0, 0, 0, 0], + ), + ]; + for (pack, expected) in cases { + let mut bytes = [0xa5; 16]; + pack(&mut bytes); + assert_eq!(&bytes[..expected.len()], *expected); + assert!(bytes[expected.len()..].iter().all(|&byte| byte == 0xa5)); + let mut short = vec![0xa5; expected.len() - 1]; + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| pack(&mut short))).is_err() + ); + assert!(short.iter().all(|&byte| byte == 0xa5)); + } + // Invalid REAL8 values are rejected before accessing the output buffer. + assert!(pack_real8(&mut [], f64::INFINITY).is_err()); +} + +#[test] +fn array_packers_reject_nonexact_buffers_before_writing() { + for width in [2, 4] { + let required = 2 * width; + for size in [required - 1, required + 1, required + width] { + let mut bytes = vec![0xa5; size]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if width == 2 { + Int2Array::pack_into(&mut bytes, &[-1, 1]); + } else { + Int4Array::pack_into(&mut bytes, &[-1, 1]); + } + })); + assert!(result.is_err()); + assert!(bytes.iter().all(|&byte| byte == 0xa5)); + } + } +} + +#[test] +fn string_records_pad_only_when_needed_and_support_empty_strings() { + for value in [b"".as_slice(), b"odd", b"even"] { + let mut bytes = Vec::new(); + let expected_size = 4 + value.len() + value.len() % 2; + assert_eq!( + records::LIBNAME::write(&mut bytes, value).unwrap(), + expected_size + ); + assert_eq!(bytes.len(), expected_size); + let (remaining, parsed) = records::LIBNAME::read(&bytes).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(parsed, value); + if value.len() % 2 == 1 { + assert_eq!(bytes.last(), Some(&0)); + } + } +} + +struct ShortWriter(Vec); + +impl Write for ShortWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let size = bytes.len().min(1); + self.0.extend_from_slice(&bytes[..size]); + Ok(size) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn library_header_round_trips_through_a_short_writer() { + let header = FileHeader::new(b"odd", 1e-9, 0.001); + let mut writer = ShortWriter(Vec::new()); + let mut written = header.write(&mut writer).unwrap(); + written += records::ENDLIB::write(&mut writer, &()).unwrap(); + assert_eq!(written, writer.0.len()); + let (remaining, library) = read_library(&writer.0).unwrap(); + assert!(remaining.is_empty()); + assert_eq!(library.len(), 1); +} + +#[test] +fn oversized_records_fail_before_writing() { + let mut bytes = Vec::new(); + let error = records::XY::write(&mut bytes, &vec![0; 16383]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(bytes.is_empty()); + + // Header packing and direct header writes reject every overflowing size too. + for data_size in u16::MAX - 3..=u16::MAX { + let header = RecordHeader { + tag: records::RTAG_XY, + data_size, + }; + assert_eq!( + header.pack_into().unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + header.write(&mut bytes).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(bytes.is_empty()); + } +} + +#[test] +fn invalid_record_sizes_preserve_the_output() { + let mut bytes = vec![0xa5]; + for count in [0, 1, 3] { + let error = records::COLROW::write(&mut bytes, &vec![2; count]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } + let error = records::COLROW::write_header(&mut bytes, 2).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + + macro_rules! check_metadata { + ($record:ident) => { + for count in [43, 44, 45] { + let payload = vec![b'x'; count]; + let mut bytes = vec![0xa5]; + let result = records::$record::write(&mut bytes, &payload); + if count == 45 { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + } else { + // Validation uses the padded record size, not the input length. + assert_eq!(result.unwrap(), 48); + assert_eq!(records::$record::read(&bytes[1..]).unwrap().1, payload); + } + } + let error = records::$record::write_header(&mut bytes, 46).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(bytes, [0xa5]); + }; + } + check_metadata!(REFLIBS); + check_metadata!(FONTS); + check_metadata!(ATTRTABLE); +} diff --git a/tools/build_manylinux.py b/tools/build_manylinux.py new file mode 100644 index 0000000..c90c7da --- /dev/null +++ b/tools/build_manylinux.py @@ -0,0 +1,104 @@ +"""Build Linux release wheels using Podman storage in this checkout.""" + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +CACHE = ROOT / ".cache/wheels" + + +def local_environment() -> dict[str, str]: + podman = CACHE / "podman" + scratch = CACHE / "tmp" + podman.mkdir(parents=True, exist_ok=True) + scratch.mkdir(exist_ok=True) + storage = podman / "storage.conf" + graphroot = json.dumps(str(podman / "storage"), ensure_ascii=False) + runroot = json.dumps(str(podman / "run"), ensure_ascii=False) + config = f"[storage]\ngraphroot = {graphroot}\nrunroot = {runroot}\n" + # Replace the config atomically so simultaneous Podman commands can read it. + if not storage.exists() or storage.read_text(encoding="utf-8") != config: + temporary = podman / f"storage.{os.getpid()}.conf" + temporary.write_text(config, encoding="utf-8") + temporary.replace(storage) + + env = dict(os.environ, CONTAINERS_STORAGE_CONF=str(storage), TMPDIR=str(scratch)) + for key in ("CONTAINER_HOST", "CONTAINER_CONNECTION"): + env.pop(key, None) + return env + + +def build(sdist: Path) -> None: + env = local_environment() + image = (ROOT / "contrib/wheels/manylinux-image.txt").read_text(encoding="utf-8").strip() + + # Use the same Podman invocation as cibuildwheel to check the active store. + graphroot = subprocess.check_output( + ["podman", "info", "--format", "{{.Store.GraphRoot}}"], env=env, text=True, + ).strip() + if Path(graphroot).resolve() != (CACHE / "podman/storage").resolve(): + raise SystemExit(f"Expected the checkout's Podman store, found {graphroot}") + + subprocess.run( + ["cargo", "fetch", "--locked", "--target", "x86_64-unknown-linux-gnu"], + cwd=ROOT, env=env, check=True, + ) + toolchain = subprocess.check_output( + ["rustc", "--print", "sysroot"], cwd=ROOT, env=env, text=True, + ).strip() + registry = Path(env.get("CARGO_HOME", Path.home() / ".cargo")).resolve() / "registry" + for directory in ("cargo/registry", "pip", "target", "cibuildwheel"): + (CACHE / directory).mkdir(parents=True, exist_ok=True) + mounts = [ + f"{toolchain}:/rust-toolchain:ro", + f"{CACHE / 'cargo'}:/pheon-cache/cargo", + f"{registry}:/pheon-cache/cargo/registry:ro", + f"{CACHE / 'pip'}:/pheon-cache/pip", + f"{CACHE / 'target'}:/pheon-cache/target", + ] + create_args = shlex.join([arg for mount in mounts for arg in ("--volume", mount)]) + env.update( + CIBW_CONTAINER_ENGINE=f"podman; disable_host_mount: true; create_args: {create_args}", + CIBW_MANYLINUX_X86_64_IMAGE=image, + CIBW_ENVIRONMENT_LINUX=( + "PATH=/rust-toolchain/bin:$PATH CARGO_HOME=/pheon-cache/cargo " + "CARGO_TARGET_DIR=/pheon-cache/target CARGO_NET_OFFLINE=true " + "PIP_CACHE_DIR=/pheon-cache/pip" + ), + CIBW_CACHE_PATH=str(CACHE / "cibuildwheel"), + CIBW_BUILD="cp31{1,2,3,4}-manylinux_x86_64", + CIBW_TEST_COMMAND="python -I {project}/tools/check_installed.py", + ) + env.pop("PHEON_LIBRARY", None) + subprocess.run( + [sys.executable, "-m", "cibuildwheel", "--platform", "linux", "--archs", "x86_64", + "--output-dir", str(sdist.parent), str(sdist)], + cwd=ROOT, env=env, check=True, + ) + + +def main() -> int: + if sys.argv[1:2] == ["podman"]: + return subprocess.run( + ["podman", "--remote=false", *sys.argv[2:]], env=local_environment(), + ).returncode + + parser = argparse.ArgumentParser( + description=__doc__, epilog="Use 'podman ARGS' to inspect the local store.", + ) + parser.add_argument("sdist", type=Path, help="Pheon source archive") + args = parser.parse_args() + if not args.sdist.is_file(): + parser.error(f"source archive does not exist: {args.sdist}") + build(args.sdist.resolve()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/check_gds_corpus.py b/tools/check_gds_corpus.py new file mode 100644 index 0000000..8331145 --- /dev/null +++ b/tools/check_gds_corpus.py @@ -0,0 +1,167 @@ +"""Check a pinned open-source layout corpus against Pheon's installed readers.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tomllib +from urllib.parse import quote + +from pheon import arrow + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_input(path, url, digest, fetch): + if path.is_file(): + data = path.read_bytes() + elif fetch: + data = subprocess.check_output( + ["curl", "-fLsS", "--retry", "2", "--max-time", "60", url] + ) + else: + raise FileNotFoundError(f"{path} is missing. Rerun with --fetch.") + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError(f"SHA-256 mismatch for {path}") + if not path.is_file(): + path.write_bytes(data) + return data + + +def inspect_layout(cli, path, data): + results = {} + reads = { + "full": lambda: arrow.read_path(path), + "scan": lambda: arrow.scan_bytes(data), + } + layouts = {} + for name, read in reads.items(): + try: + layout = read() + layouts[name] = layout + results[name] = { + "status": "ok", + "cells": len(layout.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results[name] = {"status": "failed", "message": str(error)} + + if "scan" in layouts: + cells = layouts["scan"].field("cells")[0].values + ranges = list( + zip( + cells.field("struct_start_offset").to_pylist(), + cells.field("struct_end_offset").to_pylist(), + strict=True, + ) + ) + try: + selected = arrow.read_cells_bytes(data, ranges) + if "full" in layouts and not selected.equals(layouts["full"]): + raise ValueError( + "Reading all scanned cell ranges differs from the full read" + ) + results["selected"] = { + "status": "ok", + "cells": len(selected.field("cells")[0].values), + } + except (ValueError, OSError, TypeError) as error: + results["selected"] = {"status": "failed", "message": str(error)} + else: + results["selected"] = {"status": "skipped"} + + try: + run = subprocess.run( + [cli, "diff", path, path, "--format", "json"], + capture_output=True, + timeout=60, + ) + except (subprocess.TimeoutExpired, OSError) as error: + results["cli"] = {"status": "failed", "message": str(error)} + return results + try: + report = json.loads(run.stdout) + results["cli"] = { + "status": report["status"], + "result": report["result"], + "exit_code": run.returncode, + "diagnostics": report["diagnostics"], + } + if (report["status"], report["result"], run.returncode) not in { + ("complete", "equal", 0), + ("incomplete", "unknown", 2), + ("failed", "unknown", 2), + }: + results["cli"]["status"] = "failed" + except (ValueError, KeyError, TypeError) as error: + results["cli"] = { + "status": "failed", + "message": str(error), + "exit_code": run.returncode, + "stderr": run.stderr[-4096:].decode("utf-8", errors="replace"), + } + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fetch", action="store_true", help="download missing inputs using curl" + ) + binary = "pheon.exe" if os.name == "nt" else "pheon" + parser.add_argument("--cli", type=Path, default=ROOT / "target/release" / binary) + parser.add_argument("--cache", type=Path, default=ROOT / ".cache/gds-corpus") + args = parser.parse_args() + cli = args.cli.resolve() + if not cli.is_file(): + parser.error(f"{cli} is missing. Build the CLI or pass --cli.") + + for name in ("files", "licenses"): + (args.cache / name).mkdir(parents=True, exist_ok=True) + sources = tomllib.loads( + (ROOT / "contrib/gds-corpus.toml").read_text(encoding="utf-8") + ) + all_results = {} + for key, source in sources.items(): + base = ( + f"https://raw.githubusercontent.com/{source['repo']}/{source['revision']}" + ) + read_input( + args.cache / "licenses" / f"{key}-LICENSE", + base + "/LICENSE", + source["license_sha256"], + args.fetch, + ) + for upstream, digest in source["layouts"].items(): + name = f"{key}-{Path(upstream).name}" + path = args.cache / "files" / name + raw = read_input(path, f"{base}/{quote(upstream)}", digest, args.fetch) + data = gzip.decompress(raw) if path.suffix == ".gz" else raw + results = inspect_layout(cli, path, data) + all_results[name] = results + print( + name, + *( + f"{operation}={result['status']}" + for operation, result in results.items() + ), + ) + + report = args.cache / "report.json" + report.write_text(json.dumps(all_results, indent=2) + "\n", encoding="utf-8") + failures = sum( + any(result["status"] == "failed" for result in case.values()) + for case in all_results.values() + ) + print( + f"{len(all_results)} layouts checked, {failures} with failures. See {report}." + ) + return int(failures != 0) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_installed.py b/tools/check_installed.py new file mode 100644 index 0000000..437aadb --- /dev/null +++ b/tools/check_installed.py @@ -0,0 +1,26 @@ +"""Run outside the checkout, without PHEON_LIBRARY, after installing a wheel.""" +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +# A separate interpreter checks that PyArrow stays unloaded during packing. +subprocess.run( + [sys.executable, "-I", "-c", """ +import sys +from importlib.metadata import version +import pheon +from pheon._native import library +assert pheon.__version__ == version('pheon') +assert library.parent == __import__('pathlib').Path(pheon.__file__).parent +assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') +assert 'pyarrow' not in sys.modules +"""], + check=True, +) +suite = unittest.defaultTestLoader.discover(str(ROOT / "tests/python")) +if not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(): + raise SystemExit(1)