[gdsii.arrow] use published pheon library for GDSII arrow read/write

This commit is contained in:
Jan Petykiewicz 2026-11-19 19:19:28 -08:00
commit 953dc4202b
8 changed files with 136 additions and 213 deletions

View file

@ -768,11 +768,10 @@ finally:
the context-manager protocol. The old `full_load` and `postprocess` arguments
are gone; materialize/copy the desired cells and post-process them explicitly.
An optional Arrow/native backend is available through
`masque.file.gdsii.arrow` and `masque.file.gdsii.lazy_arrow`; install the new
`arrow` extra to use it. These modules are additive and are not a transparent
replacement unless their additional dependencies and native library are
available.
Install `masque[arrow]` to use the Pheon-backed `masque.file.gdsii.arrow` and
`masque.file.gdsii.lazy_arrow` readers. For local native builds, set
`PHEON_LIBRARY` to the Pheon shared library in place of the former
`KLAMATH_RS_EXT_LIB` setting.
GDS writing now has one entry point for eager and lazy libraries. Use
`masque.file.gdsii.write()` or `masque.file.gdsii.writefile()` regardless of

View file

@ -19,6 +19,7 @@ Requirements:
* klamath (used for GDSII i/o)
Optional requirements:
* `arrow` (eager and lazy Arrow GDSII reads): pheon
* `ezdxf` (DXF i/o): ezdxf
* `oasis` (OASIS i/o): fatamorgana
* `svg` (SVG output): svgwrite
@ -31,6 +32,16 @@ Install with pip:
pip install 'masque[oasis,dxf,svg,visualization,text]'
```
Install `masque[arrow]` for the eager `masque.file.gdsii.arrow` and lazy
`masque.file.gdsii.lazy_arrow` readers. Pheon's Python package supplies PyArrow
and loads the native library bundled in its platform wheel. Installing Pheon
from source requires Rust and a native linker. For local development,
`PHEON_LIBRARY` can point the Python package to a Cargo-built shared library.
Pheon's Arrow schema requires array reference pitches to be exact signed 32-bit
integers in GDSII database units. Use `masque.file.gdsii.readfile()` or
`masque.file.gdsii.lazy.readfile()` for layouts with fractional pitches.
## Overview
A layout consists of a hierarchy of `Pattern`s stored in a single `Library`.

View file

@ -1,6 +1,6 @@
# ruff: noqa: ARG001
"""
GDSII file format readers and writers using the `TODO` library.
GDSII file format readers using the `pheon` library.
Note that GDSII references follow the same convention as `masque`,
with this order of operations:
@ -17,39 +17,34 @@ Notes:
* PLEX is not supported
* ELFLAGS are not supported
* GDS does not support library- or structure-level annotations
* GDS creation/modification/access times are set to 1900-01-01 for reproducibility.
* Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01)
TODO writing
TODO warn on boxes, nodes
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from functools import cache
from importlib.machinery import EXTENSION_SUFFIXES
import importlib.util
import logging
import os
import pathlib
import gzip
import sys
import tempfile
from klamath.basic import KlamathError
import numpy
import pyarrow
from pyarrow.cffi import ffi
from ..utils import is_gzipped
from ... import Pattern, Ref, PatternError, Label
from ...shapes import Polygon, Path, PolyCollection, RectCollection
from ...repetition import Grid
from ...library import Library
try:
from pheon import arrow as pheon_arrow
except ModuleNotFoundError as error:
if error.name == 'pheon':
raise ImportError('Install masque[arrow] to use the Pheon GDSII reader') from error
raise
if TYPE_CHECKING:
from collections.abc import Callable
import mmap
import pathlib
import pyarrow
from numpy.typing import NDArray
@ -58,22 +53,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
ffi.cdef(
"""
const char* last_error_message(void);
int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema);
int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema);
int read_cells_bytes(
uint8_t* data,
size_t size,
uint64_t* ranges,
size_t range_count,
struct ArrowArray* array,
struct ArrowSchema* schema
);
"""
)
_PATH_CAP_MAP = {
0: Path.Cap.Flush,
1: Path.Cap.Circle,
@ -100,80 +79,11 @@ def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDAr
return numpy.stack((xx, yy), axis=-1)
def _local_library_filename() -> str:
if sys.platform.startswith('linux'):
return 'libklamath_rs_ext.so'
if sys.platform == 'darwin':
return 'libklamath_rs_ext.dylib'
if sys.platform == 'win32':
return 'klamath_rs_ext.dll'
raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}')
def _installed_library_candidates() -> list[pathlib.Path]:
candidates: list[pathlib.Path] = []
def _native_read(reader: Callable[..., pyarrow.StructArray], *args: Any) -> pyarrow.StructArray:
try:
spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext')
except ModuleNotFoundError:
spec = None
if spec is not None and spec.origin is not None:
candidates.append(pathlib.Path(spec.origin))
try:
pkg_spec = importlib.util.find_spec('klamath_rs_ext')
except ModuleNotFoundError:
pkg_spec = None
if pkg_spec is not None and pkg_spec.submodule_search_locations is not None:
for location in pkg_spec.submodule_search_locations:
pkg_dir = pathlib.Path(location)
for suffix in EXTENSION_SUFFIXES:
candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}')))
return candidates
def _repo_library_candidates() -> list[pathlib.Path]:
repo_root = pathlib.Path(__file__).resolve().parents[3]
library_name = _local_library_filename()
return [
repo_root / 'klamath-rs' / 'target' / 'release' / library_name,
repo_root / 'klamath-rs' / 'target' / 'debug' / library_name,
]
def _find_klamath_rs_library() -> pathlib.Path | None:
env_path = os.environ.get('KLAMATH_RS_EXT_LIB')
if env_path:
candidate = pathlib.Path(env_path).expanduser()
if candidate.exists():
return candidate.resolve()
seen: set[pathlib.Path] = set()
for candidate in _installed_library_candidates() + _repo_library_candidates():
resolved = candidate.expanduser()
if resolved in seen:
continue
seen.add(resolved)
if resolved.exists():
return resolved.resolve()
return None
def is_available() -> bool:
return _find_klamath_rs_library() is not None
@cache
def _get_clib() -> Any:
lib_path = _find_klamath_rs_library()
if lib_path is None:
raise ImportError(
'Could not locate klamath_rs_ext shared library. '
'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` '
'or set KLAMATH_RS_EXT_LIB to the built library path.'
)
return ffi.dlopen(str(lib_path))
return reader(*args)
except pheon_arrow.PheonError as error:
raise KlamathError(str(error)) from error
def _read_annotations(
@ -188,69 +98,19 @@ def _read_annotations(
return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)}
def _read_to_arrow(
filename: str | pathlib.Path,
) -> pyarrow.Array:
path = pathlib.Path(filename).expanduser().resolve()
ptr_array = ffi.new('struct ArrowArray[]', 1)
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
if is_gzipped(path):
with gzip.open(path, mode='rb') as src:
data = src.read()
with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream:
tmp_stream.write(data)
tmp_name = tmp_stream.name
try:
_call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path')
finally:
pathlib.Path(tmp_name).unlink(missing_ok=True)
else:
_call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path')
return _import_arrow_array(ptr_array, ptr_schema)
def _read_to_arrow(filename: str | pathlib.Path) -> pyarrow.StructArray:
return _native_read(pheon_arrow.read_path, filename)
def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array:
iptr_schema = int(ffi.cast('uintptr_t', ptr_schema))
iptr_array = int(ffi.cast('uintptr_t', ptr_array))
return pyarrow.Array._import_from_c(iptr_array, iptr_schema)
def _call_native(status: int, action: str) -> None:
if status == 0:
return
err_ptr = _get_clib().last_error_message()
if err_ptr == ffi.NULL:
raise KlamathError(f'{action} failed')
message = ffi.string(err_ptr).decode(errors='replace')
raise KlamathError(message)
def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array:
ptr_array = ffi.new('struct ArrowArray[]', 1)
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
buf_view = memoryview(buffer)
cbuf = ffi.from_buffer('uint8_t[]', buf_view)
_call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes')
return _import_arrow_array(ptr_array, ptr_schema)
def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.StructArray:
return _native_read(pheon_arrow.scan_bytes, buffer)
def _read_selected_cells_to_arrow(
buffer: bytes | mmap.mmap | memoryview,
ranges: NDArray[numpy.uint64],
) -> pyarrow.Array:
ptr_array = ffi.new('struct ArrowArray[]', 1)
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
buf_view = memoryview(buffer)
cbuf = ffi.from_buffer('uint8_t[]', buf_view)
flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED'))
cranges = ffi.from_buffer('uint64_t[]', flat_ranges)
_call_native(
_get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema),
'read_cells_bytes',
)
return _import_arrow_array(ptr_array, ptr_schema)
) -> pyarrow.StructArray:
return _native_read(pheon_arrow.read_cells_bytes, buffer, ranges)
def readfile(

View file

@ -1,5 +1,5 @@
"""
Lazy GDSII readers and writers backed by native Arrow scan/materialize paths.
Lazy GDSII reader backed by Pheon's Arrow scan and selected-cell reads.
This module is intentionally separate from `gdsii.arrow` so the eager read path
keeps its current behavior and performance profile.
@ -7,7 +7,7 @@ keeps its current behavior and performance profile.
from __future__ import annotations
from dataclasses import dataclass
from typing import IO, TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any
import gzip
import logging
import mmap
@ -16,7 +16,6 @@ import pathlib
import numpy
from . import arrow
from ..utils import is_gzipped
from ...library import (
ILibraryView,
IMaterializable,
@ -41,16 +40,6 @@ class _StructRange:
end: int
@dataclass
class _SourceBuffer:
path: pathlib.Path
data: bytes | mmap.mmap
handle: IO[bytes] | None = None
def raw_slice(self, start: int, end: int) -> bytes:
return self.data[start:end]
@dataclass
class _ScanRefs:
offsets: NDArray[numpy.integer[Any]]
@ -82,15 +71,17 @@ class _ScanPayload:
cells: dict[str, _CellScan]
refs: _ScanRefs
def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer:
if is_gzipped(path):
with gzip.open(path, mode='rb') as stream:
data = stream.read()
return _SourceBuffer(path=path, data=data)
handle = path.open(mode='rb', buffering=0)
mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ)
return _SourceBuffer(path=path, data=mapped, handle=handle)
def _open_source_buffer(path: pathlib.Path) -> bytes | mmap.mmap:
with path.open(mode='rb') as stream:
magic = stream.read(2)
if not magic:
return b''
if magic == b'\x1f\x8b':
stream.seek(0)
with gzip.GzipFile(fileobj=stream) as source:
return source.read()
return mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ)
def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload:
@ -204,7 +195,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
*,
path: pathlib.Path,
payload: _ScanPayload,
source: _SourceBuffer,
source: bytes | mmap.mmap,
) -> None:
self.path = path
self.library_info = payload.library_info
@ -217,10 +208,15 @@ class ArrowLibrary(ILibraryView, IMaterializable):
def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary:
path = pathlib.Path(filename).expanduser().resolve()
source = _open_source_buffer(path)
scan_arr = arrow._scan_buffer_to_arrow(source.data)
try:
scan_arr = arrow._scan_buffer_to_arrow(source)
assert len(scan_arr) == 1
payload = _extract_scan_payload(scan_arr[0])
return cls(path=path, payload=payload, source=source)
except BaseException:
if isinstance(source, mmap.mmap):
source.close()
raise
def __getitem__(self, key: str) -> Pattern:
return self.materialize(key, persist=True)
@ -239,7 +235,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
def raw_struct_bytes(self, name: str) -> bytes:
struct_range = self._payload.cells[name].struct_range
return self._source.raw_slice(struct_range.start, struct_range.end)
return self._source[struct_range.start:struct_range.end]
def can_copy_raw_struct(self, name: str) -> bool:
return name not in self._cache
@ -285,7 +281,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
],
dtype=numpy.uint64,
)
arrow_arr = arrow._read_selected_cells_to_arrow(self._source.data, ranges)
arrow_arr = arrow._read_selected_cells_to_arrow(self._source, ranges)
assert len(arrow_arr) == 1
selected_lib, _info = arrow.read_arrow(arrow_arr[0])
for name in uncached:
@ -349,12 +345,8 @@ class ArrowLibrary(ILibraryView, IMaterializable):
return rows
def close(self) -> None:
data = self._source.data
if isinstance(data, mmap.mmap):
data.close()
if self._source.handle is not None:
self._source.handle.close()
self._source.handle = None
if isinstance(self._source, mmap.mmap):
self._source.close()
def __enter__(self) -> ArrowLibrary:
return self

View file

@ -1,5 +1,9 @@
from pathlib import Path
from typing import cast
import subprocess
import sys
import textwrap
import numpy
import pytest
from numpy.testing import assert_equal, assert_allclose
@ -11,6 +15,28 @@ from ..file import gdsii
from ..shapes import Path as MPath, Polygon
def test_gdsii_import_requires_pheon_only_for_arrow_readers() -> None:
script = textwrap.dedent('''
from importlib import import_module
import sys
sys.modules['pheon'] = None
from masque.file import gdsii
assert 'masque.file.gdsii.arrow' not in sys.modules
assert 'masque.file.gdsii.lazy_arrow' not in sys.modules
for reader in ('arrow', 'lazy_arrow'):
try:
import_module(f'{gdsii.__name__}.{reader}')
except ImportError as error:
assert 'Install masque[arrow]' in str(error), error
else:
raise AssertionError(f'{reader} imported without Pheon')
''')
result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False)
assert result.returncode == 0, result.stderr
def test_gdsii_roundtrip(tmp_path: Path) -> None:
lib = Library()

View file

@ -1,4 +1,6 @@
from importlib.util import find_spec
from pathlib import Path
import os
import subprocess
import sys
import textwrap
@ -7,20 +9,31 @@ import klamath
import numpy
import pytest
pytest.importorskip('pyarrow')
from .. import Ref, Label, PatternError
from ..library import Library
from ..pattern import Pattern
from ..repetition import Grid
from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection
from ..file import gdsii
from ..file.gdsii import arrow as gdsii_arrow
from tools.generate_gds_perf import write_fixture
if not gdsii_arrow.is_available():
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
if find_spec('pheon') is None:
pytest.skip('pheon package is not available', allow_module_level=True)
from ..file.gdsii import arrow as gdsii_arrow # noqa: E402
def test_arrow_import_reports_unloadable_native_library(tmp_path: Path) -> None:
result = subprocess.run(
[sys.executable, '-c', 'from masque.file.gdsii import arrow'],
env={**os.environ, 'PHEON_LIBRARY': str(tmp_path / 'missing-library')},
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert 'Could not load Pheon library' in result.stderr
def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None:

View file

@ -1,28 +1,30 @@
from importlib.util import find_spec
from pathlib import Path
import mmap
import subprocess
import sys
import textwrap
import klamath
from klamath.basic import KlamathError
import numpy
import pytest
pytest.importorskip('pyarrow')
from .. import PatternError, LibraryError
from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView
from ..pattern import Pattern
from ..repetition import Grid
from ..file import gdsii
from ..file.utils import preflight_source_aware
from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow
from ..file.gdsii import arrow as gdsii_arrow
from ..file.gdsii import writer as gdsii_writer
from tools.generate_gds_perf import write_fixture
if not gdsii_arrow.is_available():
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
if find_spec('pheon') is None:
pytest.skip('pheon package is not available', allow_module_level=True)
from ..file.gdsii import arrow as gdsii_arrow # noqa: E402
from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow # noqa: E402
@pytest.mark.parametrize('cached', [False, True])
@ -268,9 +270,10 @@ def test_gdsii_lazy_arrow_detached_batch_preserves_native_batching(
assert not mapped._cache
def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None:
@pytest.mark.parametrize('content', [b'not-a-gds', b''])
def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path, content: bytes) -> None:
gds_file = tmp_path / 'invalid.gds'
gds_file.write_bytes(b'not-a-gds')
gds_file.write_bytes(content)
script = textwrap.dedent(f"""
from masque.file.gdsii import lazy_arrow as gdsii_lazy_arrow
@ -290,6 +293,25 @@ def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) ->
assert 'KlamathError' in result.stdout
def test_gdsii_lazy_arrow_closes_mapping_when_scan_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
gds_file = tmp_path / 'invalid.gds'
gds_file.write_bytes(b'not-a-gds')
scan = gdsii_arrow._scan_buffer_to_arrow
mappings: list[mmap.mmap] = []
def record_mapping(source: bytes | mmap.mmap | memoryview) -> object:
assert isinstance(source, mmap.mmap)
mappings.append(source)
return scan(source)
monkeypatch.setattr(gdsii_arrow, '_scan_buffer_to_arrow', record_mapping)
with pytest.raises(KlamathError):
gdsii_lazy_arrow.readfile(gds_file)
assert len(mappings) == 1
assert mappings[0].closed
def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None:
gds_file = tmp_path / 'invalid_path_type.gds'
_write_invalid_path_type_fixture(gds_file)

View file

@ -81,7 +81,7 @@ packages = ["masque"]
path = "masque/__init__.py"
[project.optional-dependencies]
arrow = ["pyarrow", "cffi"]
arrow = ["pheon>=0.3,<0.4"]
oasis = ["fatamorgana>=0.14"]
dxf = ["ezdxf~=1.4", "pyclipper"]
svg = ["svgwrite"]