From 953dc4202bfd60611e9db2ae66e87bb40a527ddf Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 19 Nov 2026 19:19:28 -0800 Subject: [PATCH] [gdsii.arrow] use published `pheon` library for GDSII arrow read/write --- MIGRATION.md | 9 +- README.md | 11 ++ masque/file/gdsii/arrow.py | 180 +++------------------------ masque/file/gdsii/lazy_arrow.py | 60 ++++----- masque/test/test_gdsii.py | 26 ++++ masque/test/test_gdsii_arrow.py | 23 +++- masque/test/test_gdsii_lazy_arrow.py | 38 ++++-- pyproject.toml | 2 +- 8 files changed, 136 insertions(+), 213 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index e723821..ced6c1d 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -768,11 +768,10 @@ finally: the context-manager protocol. The old `full_load` and `postprocess` arguments are gone; materialize/copy the desired cells and post-process them explicitly. -An optional Arrow/native backend is available through -`masque.file.gdsii.arrow` and `masque.file.gdsii.lazy_arrow`; install the new -`arrow` extra to use it. These modules are additive and are not a transparent -replacement unless their additional dependencies and native library are -available. +Install `masque[arrow]` to use the Pheon-backed `masque.file.gdsii.arrow` and +`masque.file.gdsii.lazy_arrow` readers. For local native builds, set +`PHEON_LIBRARY` to the Pheon shared library in place of the former +`KLAMATH_RS_EXT_LIB` setting. GDS writing now has one entry point for eager and lazy libraries. Use `masque.file.gdsii.write()` or `masque.file.gdsii.writefile()` regardless of diff --git a/README.md b/README.md index 7060cd7..02f3318 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Requirements: * klamath (used for GDSII i/o) Optional requirements: +* `arrow` (eager and lazy Arrow GDSII reads): pheon * `ezdxf` (DXF i/o): ezdxf * `oasis` (OASIS i/o): fatamorgana * `svg` (SVG output): svgwrite @@ -31,6 +32,16 @@ Install with pip: pip install 'masque[oasis,dxf,svg,visualization,text]' ``` +Install `masque[arrow]` for the eager `masque.file.gdsii.arrow` and lazy +`masque.file.gdsii.lazy_arrow` readers. Pheon's Python package supplies PyArrow +and loads the native library bundled in its platform wheel. Installing Pheon +from source requires Rust and a native linker. For local development, +`PHEON_LIBRARY` can point the Python package to a Cargo-built shared library. + +Pheon's Arrow schema requires array reference pitches to be exact signed 32-bit +integers in GDSII database units. Use `masque.file.gdsii.readfile()` or +`masque.file.gdsii.lazy.readfile()` for layouts with fractional pitches. + ## Overview A layout consists of a hierarchy of `Pattern`s stored in a single `Library`. diff --git a/masque/file/gdsii/arrow.py b/masque/file/gdsii/arrow.py index 6a2a42c..e9e9185 100644 --- a/masque/file/gdsii/arrow.py +++ b/masque/file/gdsii/arrow.py @@ -1,6 +1,6 @@ # ruff: noqa: ARG001 """ -GDSII file format readers and writers using the `TODO` library. +GDSII file format readers using the `pheon` library. Note that GDSII references follow the same convention as `masque`, with this order of operations: @@ -17,39 +17,34 @@ Notes: * PLEX is not supported * ELFLAGS are not supported * GDS does not support library- or structure-level annotations - * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. - * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) - TODO writing TODO warn on boxes, nodes """ from __future__ import annotations from typing import TYPE_CHECKING, Any -from functools import cache -from importlib.machinery import EXTENSION_SUFFIXES -import importlib.util import logging -import os -import pathlib -import gzip -import sys -import tempfile from klamath.basic import KlamathError import numpy -import pyarrow -from pyarrow.cffi import ffi -from ..utils import is_gzipped from ... import Pattern, Ref, PatternError, Label from ...shapes import Polygon, Path, PolyCollection, RectCollection from ...repetition import Grid from ...library import Library +try: + from pheon import arrow as pheon_arrow +except ModuleNotFoundError as error: + if error.name == 'pheon': + raise ImportError('Install masque[arrow] to use the Pheon GDSII reader') from error + raise + if TYPE_CHECKING: from collections.abc import Callable import mmap + import pathlib + import pyarrow from numpy.typing import NDArray @@ -58,22 +53,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -ffi.cdef( - """ - const char* last_error_message(void); - int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); - int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); - int read_cells_bytes( - uint8_t* data, - size_t size, - uint64_t* ranges, - size_t range_count, - struct ArrowArray* array, - struct ArrowSchema* schema - ); - """ -) - _PATH_CAP_MAP = { 0: Path.Cap.Flush, 1: Path.Cap.Circle, @@ -100,80 +79,11 @@ def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDAr return numpy.stack((xx, yy), axis=-1) -def _local_library_filename() -> str: - if sys.platform.startswith('linux'): - return 'libklamath_rs_ext.so' - if sys.platform == 'darwin': - return 'libklamath_rs_ext.dylib' - if sys.platform == 'win32': - return 'klamath_rs_ext.dll' - raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}') - - -def _installed_library_candidates() -> list[pathlib.Path]: - candidates: list[pathlib.Path] = [] - +def _native_read(reader: Callable[..., pyarrow.StructArray], *args: Any) -> pyarrow.StructArray: try: - spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext') - except ModuleNotFoundError: - spec = None - if spec is not None and spec.origin is not None: - candidates.append(pathlib.Path(spec.origin)) - - try: - pkg_spec = importlib.util.find_spec('klamath_rs_ext') - except ModuleNotFoundError: - pkg_spec = None - if pkg_spec is not None and pkg_spec.submodule_search_locations is not None: - for location in pkg_spec.submodule_search_locations: - pkg_dir = pathlib.Path(location) - for suffix in EXTENSION_SUFFIXES: - candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}'))) - - return candidates - - -def _repo_library_candidates() -> list[pathlib.Path]: - repo_root = pathlib.Path(__file__).resolve().parents[3] - library_name = _local_library_filename() - return [ - repo_root / 'klamath-rs' / 'target' / 'release' / library_name, - repo_root / 'klamath-rs' / 'target' / 'debug' / library_name, - ] - - -def _find_klamath_rs_library() -> pathlib.Path | None: - env_path = os.environ.get('KLAMATH_RS_EXT_LIB') - if env_path: - candidate = pathlib.Path(env_path).expanduser() - if candidate.exists(): - return candidate.resolve() - - seen: set[pathlib.Path] = set() - for candidate in _installed_library_candidates() + _repo_library_candidates(): - resolved = candidate.expanduser() - if resolved in seen: - continue - seen.add(resolved) - if resolved.exists(): - return resolved.resolve() - return None - - -def is_available() -> bool: - return _find_klamath_rs_library() is not None - - -@cache -def _get_clib() -> Any: - lib_path = _find_klamath_rs_library() - if lib_path is None: - raise ImportError( - 'Could not locate klamath_rs_ext shared library. ' - 'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` ' - 'or set KLAMATH_RS_EXT_LIB to the built library path.' - ) - return ffi.dlopen(str(lib_path)) + return reader(*args) + except pheon_arrow.PheonError as error: + raise KlamathError(str(error)) from error def _read_annotations( @@ -188,69 +98,19 @@ def _read_annotations( return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} -def _read_to_arrow( - filename: str | pathlib.Path, - ) -> pyarrow.Array: - path = pathlib.Path(filename).expanduser().resolve() - ptr_array = ffi.new('struct ArrowArray[]', 1) - ptr_schema = ffi.new('struct ArrowSchema[]', 1) - if is_gzipped(path): - with gzip.open(path, mode='rb') as src: - data = src.read() - with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream: - tmp_stream.write(data) - tmp_name = tmp_stream.name - try: - _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path') - finally: - pathlib.Path(tmp_name).unlink(missing_ok=True) - else: - _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path') - return _import_arrow_array(ptr_array, ptr_schema) +def _read_to_arrow(filename: str | pathlib.Path) -> pyarrow.StructArray: + return _native_read(pheon_arrow.read_path, filename) -def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: - iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) - iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - return pyarrow.Array._import_from_c(iptr_array, iptr_schema) - - -def _call_native(status: int, action: str) -> None: - if status == 0: - return - - err_ptr = _get_clib().last_error_message() - if err_ptr == ffi.NULL: - raise KlamathError(f'{action} failed') - - message = ffi.string(err_ptr).decode(errors='replace') - raise KlamathError(message) - - -def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: - ptr_array = ffi.new('struct ArrowArray[]', 1) - ptr_schema = ffi.new('struct ArrowSchema[]', 1) - buf_view = memoryview(buffer) - cbuf = ffi.from_buffer('uint8_t[]', buf_view) - _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes') - return _import_arrow_array(ptr_array, ptr_schema) +def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.StructArray: + return _native_read(pheon_arrow.scan_bytes, buffer) def _read_selected_cells_to_arrow( buffer: bytes | mmap.mmap | memoryview, ranges: NDArray[numpy.uint64], - ) -> pyarrow.Array: - ptr_array = ffi.new('struct ArrowArray[]', 1) - ptr_schema = ffi.new('struct ArrowSchema[]', 1) - buf_view = memoryview(buffer) - cbuf = ffi.from_buffer('uint8_t[]', buf_view) - flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) - cranges = ffi.from_buffer('uint64_t[]', flat_ranges) - _call_native( - _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema), - 'read_cells_bytes', - ) - return _import_arrow_array(ptr_array, ptr_schema) + ) -> pyarrow.StructArray: + return _native_read(pheon_arrow.read_cells_bytes, buffer, ranges) def readfile( diff --git a/masque/file/gdsii/lazy_arrow.py b/masque/file/gdsii/lazy_arrow.py index d686bfa..6520025 100644 --- a/masque/file/gdsii/lazy_arrow.py +++ b/masque/file/gdsii/lazy_arrow.py @@ -1,5 +1,5 @@ """ -Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. +Lazy GDSII reader backed by Pheon's Arrow scan and selected-cell reads. This module is intentionally separate from `gdsii.arrow` so the eager read path keeps its current behavior and performance profile. @@ -7,7 +7,7 @@ keeps its current behavior and performance profile. from __future__ import annotations from dataclasses import dataclass -from typing import IO, TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any import gzip import logging import mmap @@ -16,7 +16,6 @@ import pathlib import numpy from . import arrow -from ..utils import is_gzipped from ...library import ( ILibraryView, IMaterializable, @@ -41,16 +40,6 @@ class _StructRange: end: int -@dataclass -class _SourceBuffer: - path: pathlib.Path - data: bytes | mmap.mmap - handle: IO[bytes] | None = None - - def raw_slice(self, start: int, end: int) -> bytes: - return self.data[start:end] - - @dataclass class _ScanRefs: offsets: NDArray[numpy.integer[Any]] @@ -82,15 +71,17 @@ class _ScanPayload: cells: dict[str, _CellScan] refs: _ScanRefs -def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: - if is_gzipped(path): - with gzip.open(path, mode='rb') as stream: - data = stream.read() - return _SourceBuffer(path=path, data=data) - handle = path.open(mode='rb', buffering=0) - mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) - return _SourceBuffer(path=path, data=mapped, handle=handle) +def _open_source_buffer(path: pathlib.Path) -> bytes | mmap.mmap: + with path.open(mode='rb') as stream: + magic = stream.read(2) + if not magic: + return b'' + if magic == b'\x1f\x8b': + stream.seek(0) + with gzip.GzipFile(fileobj=stream) as source: + return source.read() + return mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: @@ -204,7 +195,7 @@ class ArrowLibrary(ILibraryView, IMaterializable): *, path: pathlib.Path, payload: _ScanPayload, - source: _SourceBuffer, + source: bytes | mmap.mmap, ) -> None: self.path = path self.library_info = payload.library_info @@ -217,10 +208,15 @@ class ArrowLibrary(ILibraryView, IMaterializable): def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: path = pathlib.Path(filename).expanduser().resolve() source = _open_source_buffer(path) - scan_arr = arrow._scan_buffer_to_arrow(source.data) - assert len(scan_arr) == 1 - payload = _extract_scan_payload(scan_arr[0]) - return cls(path=path, payload=payload, source=source) + try: + scan_arr = arrow._scan_buffer_to_arrow(source) + assert len(scan_arr) == 1 + payload = _extract_scan_payload(scan_arr[0]) + return cls(path=path, payload=payload, source=source) + except BaseException: + if isinstance(source, mmap.mmap): + source.close() + raise def __getitem__(self, key: str) -> Pattern: return self.materialize(key, persist=True) @@ -239,7 +235,7 @@ class ArrowLibrary(ILibraryView, IMaterializable): def raw_struct_bytes(self, name: str) -> bytes: struct_range = self._payload.cells[name].struct_range - return self._source.raw_slice(struct_range.start, struct_range.end) + return self._source[struct_range.start:struct_range.end] def can_copy_raw_struct(self, name: str) -> bool: return name not in self._cache @@ -285,7 +281,7 @@ class ArrowLibrary(ILibraryView, IMaterializable): ], dtype=numpy.uint64, ) - arrow_arr = arrow._read_selected_cells_to_arrow(self._source.data, ranges) + arrow_arr = arrow._read_selected_cells_to_arrow(self._source, ranges) assert len(arrow_arr) == 1 selected_lib, _info = arrow.read_arrow(arrow_arr[0]) for name in uncached: @@ -349,12 +345,8 @@ class ArrowLibrary(ILibraryView, IMaterializable): return rows def close(self) -> None: - data = self._source.data - if isinstance(data, mmap.mmap): - data.close() - if self._source.handle is not None: - self._source.handle.close() - self._source.handle = None + if isinstance(self._source, mmap.mmap): + self._source.close() def __enter__(self) -> ArrowLibrary: return self diff --git a/masque/test/test_gdsii.py b/masque/test/test_gdsii.py index 5f655df..21e807d 100644 --- a/masque/test/test_gdsii.py +++ b/masque/test/test_gdsii.py @@ -1,5 +1,9 @@ from pathlib import Path from typing import cast +import subprocess +import sys +import textwrap + import numpy import pytest from numpy.testing import assert_equal, assert_allclose @@ -11,6 +15,28 @@ from ..file import gdsii from ..shapes import Path as MPath, Polygon +def test_gdsii_import_requires_pheon_only_for_arrow_readers() -> None: + script = textwrap.dedent(''' + from importlib import import_module + import sys + + sys.modules['pheon'] = None + from masque.file import gdsii + + assert 'masque.file.gdsii.arrow' not in sys.modules + assert 'masque.file.gdsii.lazy_arrow' not in sys.modules + for reader in ('arrow', 'lazy_arrow'): + try: + import_module(f'{gdsii.__name__}.{reader}') + except ImportError as error: + assert 'Install masque[arrow]' in str(error), error + else: + raise AssertionError(f'{reader} imported without Pheon') + ''') + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + + def test_gdsii_roundtrip(tmp_path: Path) -> None: lib = Library() diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 0a33453..a819731 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -1,4 +1,6 @@ +from importlib.util import find_spec from pathlib import Path +import os import subprocess import sys import textwrap @@ -7,20 +9,31 @@ import klamath import numpy import pytest -pytest.importorskip('pyarrow') - from .. import Ref, Label, PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection from ..file import gdsii -from ..file.gdsii import arrow as gdsii_arrow from tools.generate_gds_perf import write_fixture -if not gdsii_arrow.is_available(): - pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) +if find_spec('pheon') is None: + pytest.skip('pheon package is not available', allow_module_level=True) + +from ..file.gdsii import arrow as gdsii_arrow # noqa: E402 + + +def test_arrow_import_reports_unloadable_native_library(tmp_path: Path) -> None: + result = subprocess.run( + [sys.executable, '-c', 'from masque.file.gdsii import arrow'], + env={**os.environ, 'PHEON_LIBRARY': str(tmp_path / 'missing-library')}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + assert 'Could not load Pheon library' in result.stderr def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None: diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 927f35b..f010a11 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -1,28 +1,30 @@ +from importlib.util import find_spec from pathlib import Path +import mmap import subprocess import sys import textwrap import klamath +from klamath.basic import KlamathError import numpy import pytest -pytest.importorskip('pyarrow') - from .. import PatternError, LibraryError from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView from ..pattern import Pattern from ..repetition import Grid from ..file import gdsii from ..file.utils import preflight_source_aware -from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow -from ..file.gdsii import arrow as gdsii_arrow from ..file.gdsii import writer as gdsii_writer from tools.generate_gds_perf import write_fixture -if not gdsii_arrow.is_available(): - pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) +if find_spec('pheon') is None: + pytest.skip('pheon package is not available', allow_module_level=True) + +from ..file.gdsii import arrow as gdsii_arrow # noqa: E402 +from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow # noqa: E402 @pytest.mark.parametrize('cached', [False, True]) @@ -268,9 +270,10 @@ def test_gdsii_lazy_arrow_detached_batch_preserves_native_batching( assert not mapped._cache -def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: +@pytest.mark.parametrize('content', [b'not-a-gds', b'']) +def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path, content: bytes) -> None: gds_file = tmp_path / 'invalid.gds' - gds_file.write_bytes(b'not-a-gds') + gds_file.write_bytes(content) script = textwrap.dedent(f""" from masque.file.gdsii import lazy_arrow as gdsii_lazy_arrow @@ -290,6 +293,25 @@ def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> assert 'KlamathError' in result.stdout +def test_gdsii_lazy_arrow_closes_mapping_when_scan_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + scan = gdsii_arrow._scan_buffer_to_arrow + mappings: list[mmap.mmap] = [] + + def record_mapping(source: bytes | mmap.mmap | memoryview) -> object: + assert isinstance(source, mmap.mmap) + mappings.append(source) + return scan(source) + + monkeypatch.setattr(gdsii_arrow, '_scan_buffer_to_arrow', record_mapping) + with pytest.raises(KlamathError): + gdsii_lazy_arrow.readfile(gds_file) + + assert len(mappings) == 1 + assert mappings[0].closed + + def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None: gds_file = tmp_path / 'invalid_path_type.gds' _write_invalid_path_type_fixture(gds_file) diff --git a/pyproject.toml b/pyproject.toml index 6ace09f..ae4a5a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ packages = ["masque"] path = "masque/__init__.py" [project.optional-dependencies] -arrow = ["pyarrow", "cffi"] +arrow = ["pheon>=0.3,<0.4"] oasis = ["fatamorgana>=0.14"] dxf = ["ezdxf~=1.4", "pyclipper"] svg = ["svgwrite"]