[gdsii.arrow] use published pheon library for GDSII arrow read/write
This commit is contained in:
parent
31b853be38
commit
953dc4202b
8 changed files with 136 additions and 213 deletions
|
|
@ -768,11 +768,10 @@ finally:
|
||||||
the context-manager protocol. The old `full_load` and `postprocess` arguments
|
the context-manager protocol. The old `full_load` and `postprocess` arguments
|
||||||
are gone; materialize/copy the desired cells and post-process them explicitly.
|
are gone; materialize/copy the desired cells and post-process them explicitly.
|
||||||
|
|
||||||
An optional Arrow/native backend is available through
|
Install `masque[arrow]` to use the Pheon-backed `masque.file.gdsii.arrow` and
|
||||||
`masque.file.gdsii.arrow` and `masque.file.gdsii.lazy_arrow`; install the new
|
`masque.file.gdsii.lazy_arrow` readers. For local native builds, set
|
||||||
`arrow` extra to use it. These modules are additive and are not a transparent
|
`PHEON_LIBRARY` to the Pheon shared library in place of the former
|
||||||
replacement unless their additional dependencies and native library are
|
`KLAMATH_RS_EXT_LIB` setting.
|
||||||
available.
|
|
||||||
|
|
||||||
GDS writing now has one entry point for eager and lazy libraries. Use
|
GDS writing now has one entry point for eager and lazy libraries. Use
|
||||||
`masque.file.gdsii.write()` or `masque.file.gdsii.writefile()` regardless of
|
`masque.file.gdsii.write()` or `masque.file.gdsii.writefile()` regardless of
|
||||||
|
|
|
||||||
11
README.md
11
README.md
|
|
@ -19,6 +19,7 @@ Requirements:
|
||||||
* klamath (used for GDSII i/o)
|
* klamath (used for GDSII i/o)
|
||||||
|
|
||||||
Optional requirements:
|
Optional requirements:
|
||||||
|
* `arrow` (eager and lazy Arrow GDSII reads): pheon
|
||||||
* `ezdxf` (DXF i/o): ezdxf
|
* `ezdxf` (DXF i/o): ezdxf
|
||||||
* `oasis` (OASIS i/o): fatamorgana
|
* `oasis` (OASIS i/o): fatamorgana
|
||||||
* `svg` (SVG output): svgwrite
|
* `svg` (SVG output): svgwrite
|
||||||
|
|
@ -31,6 +32,16 @@ Install with pip:
|
||||||
pip install 'masque[oasis,dxf,svg,visualization,text]'
|
pip install 'masque[oasis,dxf,svg,visualization,text]'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Install `masque[arrow]` for the eager `masque.file.gdsii.arrow` and lazy
|
||||||
|
`masque.file.gdsii.lazy_arrow` readers. Pheon's Python package supplies PyArrow
|
||||||
|
and loads the native library bundled in its platform wheel. Installing Pheon
|
||||||
|
from source requires Rust and a native linker. For local development,
|
||||||
|
`PHEON_LIBRARY` can point the Python package to a Cargo-built shared library.
|
||||||
|
|
||||||
|
Pheon's Arrow schema requires array reference pitches to be exact signed 32-bit
|
||||||
|
integers in GDSII database units. Use `masque.file.gdsii.readfile()` or
|
||||||
|
`masque.file.gdsii.lazy.readfile()` for layouts with fractional pitches.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
A layout consists of a hierarchy of `Pattern`s stored in a single `Library`.
|
A layout consists of a hierarchy of `Pattern`s stored in a single `Library`.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# ruff: noqa: ARG001
|
# ruff: noqa: ARG001
|
||||||
"""
|
"""
|
||||||
GDSII file format readers and writers using the `TODO` library.
|
GDSII file format readers using the `pheon` library.
|
||||||
|
|
||||||
Note that GDSII references follow the same convention as `masque`,
|
Note that GDSII references follow the same convention as `masque`,
|
||||||
with this order of operations:
|
with this order of operations:
|
||||||
|
|
@ -17,39 +17,34 @@ Notes:
|
||||||
* PLEX is not supported
|
* PLEX is not supported
|
||||||
* ELFLAGS are not supported
|
* ELFLAGS are not supported
|
||||||
* GDS does not support library- or structure-level annotations
|
* GDS does not support library- or structure-level annotations
|
||||||
* GDS creation/modification/access times are set to 1900-01-01 for reproducibility.
|
|
||||||
* Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01)
|
|
||||||
|
|
||||||
TODO writing
|
|
||||||
TODO warn on boxes, nodes
|
TODO warn on boxes, nodes
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
from functools import cache
|
|
||||||
from importlib.machinery import EXTENSION_SUFFIXES
|
|
||||||
import importlib.util
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import gzip
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
from klamath.basic import KlamathError
|
from klamath.basic import KlamathError
|
||||||
import numpy
|
import numpy
|
||||||
import pyarrow
|
|
||||||
from pyarrow.cffi import ffi
|
|
||||||
|
|
||||||
from ..utils import is_gzipped
|
|
||||||
from ... import Pattern, Ref, PatternError, Label
|
from ... import Pattern, Ref, PatternError, Label
|
||||||
from ...shapes import Polygon, Path, PolyCollection, RectCollection
|
from ...shapes import Polygon, Path, PolyCollection, RectCollection
|
||||||
from ...repetition import Grid
|
from ...repetition import Grid
|
||||||
from ...library import Library
|
from ...library import Library
|
||||||
|
|
||||||
|
try:
|
||||||
|
from pheon import arrow as pheon_arrow
|
||||||
|
except ModuleNotFoundError as error:
|
||||||
|
if error.name == 'pheon':
|
||||||
|
raise ImportError('Install masque[arrow] to use the Pheon GDSII reader') from error
|
||||||
|
raise
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
import mmap
|
import mmap
|
||||||
|
import pathlib
|
||||||
|
import pyarrow
|
||||||
|
|
||||||
from numpy.typing import NDArray
|
from numpy.typing import NDArray
|
||||||
|
|
||||||
|
|
@ -58,22 +53,6 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
ffi.cdef(
|
|
||||||
"""
|
|
||||||
const char* last_error_message(void);
|
|
||||||
int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema);
|
|
||||||
int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema);
|
|
||||||
int read_cells_bytes(
|
|
||||||
uint8_t* data,
|
|
||||||
size_t size,
|
|
||||||
uint64_t* ranges,
|
|
||||||
size_t range_count,
|
|
||||||
struct ArrowArray* array,
|
|
||||||
struct ArrowSchema* schema
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
_PATH_CAP_MAP = {
|
_PATH_CAP_MAP = {
|
||||||
0: Path.Cap.Flush,
|
0: Path.Cap.Flush,
|
||||||
1: Path.Cap.Circle,
|
1: Path.Cap.Circle,
|
||||||
|
|
@ -100,80 +79,11 @@ def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDAr
|
||||||
return numpy.stack((xx, yy), axis=-1)
|
return numpy.stack((xx, yy), axis=-1)
|
||||||
|
|
||||||
|
|
||||||
def _local_library_filename() -> str:
|
def _native_read(reader: Callable[..., pyarrow.StructArray], *args: Any) -> pyarrow.StructArray:
|
||||||
if sys.platform.startswith('linux'):
|
|
||||||
return 'libklamath_rs_ext.so'
|
|
||||||
if sys.platform == 'darwin':
|
|
||||||
return 'libklamath_rs_ext.dylib'
|
|
||||||
if sys.platform == 'win32':
|
|
||||||
return 'klamath_rs_ext.dll'
|
|
||||||
raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}')
|
|
||||||
|
|
||||||
|
|
||||||
def _installed_library_candidates() -> list[pathlib.Path]:
|
|
||||||
candidates: list[pathlib.Path] = []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext')
|
return reader(*args)
|
||||||
except ModuleNotFoundError:
|
except pheon_arrow.PheonError as error:
|
||||||
spec = None
|
raise KlamathError(str(error)) from error
|
||||||
if spec is not None and spec.origin is not None:
|
|
||||||
candidates.append(pathlib.Path(spec.origin))
|
|
||||||
|
|
||||||
try:
|
|
||||||
pkg_spec = importlib.util.find_spec('klamath_rs_ext')
|
|
||||||
except ModuleNotFoundError:
|
|
||||||
pkg_spec = None
|
|
||||||
if pkg_spec is not None and pkg_spec.submodule_search_locations is not None:
|
|
||||||
for location in pkg_spec.submodule_search_locations:
|
|
||||||
pkg_dir = pathlib.Path(location)
|
|
||||||
for suffix in EXTENSION_SUFFIXES:
|
|
||||||
candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}')))
|
|
||||||
|
|
||||||
return candidates
|
|
||||||
|
|
||||||
|
|
||||||
def _repo_library_candidates() -> list[pathlib.Path]:
|
|
||||||
repo_root = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
library_name = _local_library_filename()
|
|
||||||
return [
|
|
||||||
repo_root / 'klamath-rs' / 'target' / 'release' / library_name,
|
|
||||||
repo_root / 'klamath-rs' / 'target' / 'debug' / library_name,
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _find_klamath_rs_library() -> pathlib.Path | None:
|
|
||||||
env_path = os.environ.get('KLAMATH_RS_EXT_LIB')
|
|
||||||
if env_path:
|
|
||||||
candidate = pathlib.Path(env_path).expanduser()
|
|
||||||
if candidate.exists():
|
|
||||||
return candidate.resolve()
|
|
||||||
|
|
||||||
seen: set[pathlib.Path] = set()
|
|
||||||
for candidate in _installed_library_candidates() + _repo_library_candidates():
|
|
||||||
resolved = candidate.expanduser()
|
|
||||||
if resolved in seen:
|
|
||||||
continue
|
|
||||||
seen.add(resolved)
|
|
||||||
if resolved.exists():
|
|
||||||
return resolved.resolve()
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def is_available() -> bool:
|
|
||||||
return _find_klamath_rs_library() is not None
|
|
||||||
|
|
||||||
|
|
||||||
@cache
|
|
||||||
def _get_clib() -> Any:
|
|
||||||
lib_path = _find_klamath_rs_library()
|
|
||||||
if lib_path is None:
|
|
||||||
raise ImportError(
|
|
||||||
'Could not locate klamath_rs_ext shared library. '
|
|
||||||
'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` '
|
|
||||||
'or set KLAMATH_RS_EXT_LIB to the built library path.'
|
|
||||||
)
|
|
||||||
return ffi.dlopen(str(lib_path))
|
|
||||||
|
|
||||||
|
|
||||||
def _read_annotations(
|
def _read_annotations(
|
||||||
|
|
@ -188,69 +98,19 @@ def _read_annotations(
|
||||||
return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)}
|
return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)}
|
||||||
|
|
||||||
|
|
||||||
def _read_to_arrow(
|
def _read_to_arrow(filename: str | pathlib.Path) -> pyarrow.StructArray:
|
||||||
filename: str | pathlib.Path,
|
return _native_read(pheon_arrow.read_path, filename)
|
||||||
) -> pyarrow.Array:
|
|
||||||
path = pathlib.Path(filename).expanduser().resolve()
|
|
||||||
ptr_array = ffi.new('struct ArrowArray[]', 1)
|
|
||||||
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
|
|
||||||
if is_gzipped(path):
|
|
||||||
with gzip.open(path, mode='rb') as src:
|
|
||||||
data = src.read()
|
|
||||||
with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream:
|
|
||||||
tmp_stream.write(data)
|
|
||||||
tmp_name = tmp_stream.name
|
|
||||||
try:
|
|
||||||
_call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path')
|
|
||||||
finally:
|
|
||||||
pathlib.Path(tmp_name).unlink(missing_ok=True)
|
|
||||||
else:
|
|
||||||
_call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path')
|
|
||||||
return _import_arrow_array(ptr_array, ptr_schema)
|
|
||||||
|
|
||||||
|
|
||||||
def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array:
|
def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.StructArray:
|
||||||
iptr_schema = int(ffi.cast('uintptr_t', ptr_schema))
|
return _native_read(pheon_arrow.scan_bytes, buffer)
|
||||||
iptr_array = int(ffi.cast('uintptr_t', ptr_array))
|
|
||||||
return pyarrow.Array._import_from_c(iptr_array, iptr_schema)
|
|
||||||
|
|
||||||
|
|
||||||
def _call_native(status: int, action: str) -> None:
|
|
||||||
if status == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
err_ptr = _get_clib().last_error_message()
|
|
||||||
if err_ptr == ffi.NULL:
|
|
||||||
raise KlamathError(f'{action} failed')
|
|
||||||
|
|
||||||
message = ffi.string(err_ptr).decode(errors='replace')
|
|
||||||
raise KlamathError(message)
|
|
||||||
|
|
||||||
|
|
||||||
def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array:
|
|
||||||
ptr_array = ffi.new('struct ArrowArray[]', 1)
|
|
||||||
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
|
|
||||||
buf_view = memoryview(buffer)
|
|
||||||
cbuf = ffi.from_buffer('uint8_t[]', buf_view)
|
|
||||||
_call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes')
|
|
||||||
return _import_arrow_array(ptr_array, ptr_schema)
|
|
||||||
|
|
||||||
|
|
||||||
def _read_selected_cells_to_arrow(
|
def _read_selected_cells_to_arrow(
|
||||||
buffer: bytes | mmap.mmap | memoryview,
|
buffer: bytes | mmap.mmap | memoryview,
|
||||||
ranges: NDArray[numpy.uint64],
|
ranges: NDArray[numpy.uint64],
|
||||||
) -> pyarrow.Array:
|
) -> pyarrow.StructArray:
|
||||||
ptr_array = ffi.new('struct ArrowArray[]', 1)
|
return _native_read(pheon_arrow.read_cells_bytes, buffer, ranges)
|
||||||
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
|
|
||||||
buf_view = memoryview(buffer)
|
|
||||||
cbuf = ffi.from_buffer('uint8_t[]', buf_view)
|
|
||||||
flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED'))
|
|
||||||
cranges = ffi.from_buffer('uint64_t[]', flat_ranges)
|
|
||||||
_call_native(
|
|
||||||
_get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema),
|
|
||||||
'read_cells_bytes',
|
|
||||||
)
|
|
||||||
return _import_arrow_array(ptr_array, ptr_schema)
|
|
||||||
|
|
||||||
|
|
||||||
def readfile(
|
def readfile(
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
"""
|
"""
|
||||||
Lazy GDSII readers and writers backed by native Arrow scan/materialize paths.
|
Lazy GDSII reader backed by Pheon's Arrow scan and selected-cell reads.
|
||||||
|
|
||||||
This module is intentionally separate from `gdsii.arrow` so the eager read path
|
This module is intentionally separate from `gdsii.arrow` so the eager read path
|
||||||
keeps its current behavior and performance profile.
|
keeps its current behavior and performance profile.
|
||||||
|
|
@ -7,7 +7,7 @@ keeps its current behavior and performance profile.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import IO, TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
import gzip
|
import gzip
|
||||||
import logging
|
import logging
|
||||||
import mmap
|
import mmap
|
||||||
|
|
@ -16,7 +16,6 @@ import pathlib
|
||||||
import numpy
|
import numpy
|
||||||
|
|
||||||
from . import arrow
|
from . import arrow
|
||||||
from ..utils import is_gzipped
|
|
||||||
from ...library import (
|
from ...library import (
|
||||||
ILibraryView,
|
ILibraryView,
|
||||||
IMaterializable,
|
IMaterializable,
|
||||||
|
|
@ -41,16 +40,6 @@ class _StructRange:
|
||||||
end: int
|
end: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class _SourceBuffer:
|
|
||||||
path: pathlib.Path
|
|
||||||
data: bytes | mmap.mmap
|
|
||||||
handle: IO[bytes] | None = None
|
|
||||||
|
|
||||||
def raw_slice(self, start: int, end: int) -> bytes:
|
|
||||||
return self.data[start:end]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _ScanRefs:
|
class _ScanRefs:
|
||||||
offsets: NDArray[numpy.integer[Any]]
|
offsets: NDArray[numpy.integer[Any]]
|
||||||
|
|
@ -82,15 +71,17 @@ class _ScanPayload:
|
||||||
cells: dict[str, _CellScan]
|
cells: dict[str, _CellScan]
|
||||||
refs: _ScanRefs
|
refs: _ScanRefs
|
||||||
|
|
||||||
def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer:
|
|
||||||
if is_gzipped(path):
|
|
||||||
with gzip.open(path, mode='rb') as stream:
|
|
||||||
data = stream.read()
|
|
||||||
return _SourceBuffer(path=path, data=data)
|
|
||||||
|
|
||||||
handle = path.open(mode='rb', buffering=0)
|
def _open_source_buffer(path: pathlib.Path) -> bytes | mmap.mmap:
|
||||||
mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ)
|
with path.open(mode='rb') as stream:
|
||||||
return _SourceBuffer(path=path, data=mapped, handle=handle)
|
magic = stream.read(2)
|
||||||
|
if not magic:
|
||||||
|
return b''
|
||||||
|
if magic == b'\x1f\x8b':
|
||||||
|
stream.seek(0)
|
||||||
|
with gzip.GzipFile(fileobj=stream) as source:
|
||||||
|
return source.read()
|
||||||
|
return mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ)
|
||||||
|
|
||||||
|
|
||||||
def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload:
|
def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload:
|
||||||
|
|
@ -204,7 +195,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
|
||||||
*,
|
*,
|
||||||
path: pathlib.Path,
|
path: pathlib.Path,
|
||||||
payload: _ScanPayload,
|
payload: _ScanPayload,
|
||||||
source: _SourceBuffer,
|
source: bytes | mmap.mmap,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.path = path
|
self.path = path
|
||||||
self.library_info = payload.library_info
|
self.library_info = payload.library_info
|
||||||
|
|
@ -217,10 +208,15 @@ class ArrowLibrary(ILibraryView, IMaterializable):
|
||||||
def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary:
|
def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary:
|
||||||
path = pathlib.Path(filename).expanduser().resolve()
|
path = pathlib.Path(filename).expanduser().resolve()
|
||||||
source = _open_source_buffer(path)
|
source = _open_source_buffer(path)
|
||||||
scan_arr = arrow._scan_buffer_to_arrow(source.data)
|
try:
|
||||||
assert len(scan_arr) == 1
|
scan_arr = arrow._scan_buffer_to_arrow(source)
|
||||||
payload = _extract_scan_payload(scan_arr[0])
|
assert len(scan_arr) == 1
|
||||||
return cls(path=path, payload=payload, source=source)
|
payload = _extract_scan_payload(scan_arr[0])
|
||||||
|
return cls(path=path, payload=payload, source=source)
|
||||||
|
except BaseException:
|
||||||
|
if isinstance(source, mmap.mmap):
|
||||||
|
source.close()
|
||||||
|
raise
|
||||||
|
|
||||||
def __getitem__(self, key: str) -> Pattern:
|
def __getitem__(self, key: str) -> Pattern:
|
||||||
return self.materialize(key, persist=True)
|
return self.materialize(key, persist=True)
|
||||||
|
|
@ -239,7 +235,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
|
||||||
|
|
||||||
def raw_struct_bytes(self, name: str) -> bytes:
|
def raw_struct_bytes(self, name: str) -> bytes:
|
||||||
struct_range = self._payload.cells[name].struct_range
|
struct_range = self._payload.cells[name].struct_range
|
||||||
return self._source.raw_slice(struct_range.start, struct_range.end)
|
return self._source[struct_range.start:struct_range.end]
|
||||||
|
|
||||||
def can_copy_raw_struct(self, name: str) -> bool:
|
def can_copy_raw_struct(self, name: str) -> bool:
|
||||||
return name not in self._cache
|
return name not in self._cache
|
||||||
|
|
@ -285,7 +281,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
|
||||||
],
|
],
|
||||||
dtype=numpy.uint64,
|
dtype=numpy.uint64,
|
||||||
)
|
)
|
||||||
arrow_arr = arrow._read_selected_cells_to_arrow(self._source.data, ranges)
|
arrow_arr = arrow._read_selected_cells_to_arrow(self._source, ranges)
|
||||||
assert len(arrow_arr) == 1
|
assert len(arrow_arr) == 1
|
||||||
selected_lib, _info = arrow.read_arrow(arrow_arr[0])
|
selected_lib, _info = arrow.read_arrow(arrow_arr[0])
|
||||||
for name in uncached:
|
for name in uncached:
|
||||||
|
|
@ -349,12 +345,8 @@ class ArrowLibrary(ILibraryView, IMaterializable):
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
data = self._source.data
|
if isinstance(self._source, mmap.mmap):
|
||||||
if isinstance(data, mmap.mmap):
|
self._source.close()
|
||||||
data.close()
|
|
||||||
if self._source.handle is not None:
|
|
||||||
self._source.handle.close()
|
|
||||||
self._source.handle = None
|
|
||||||
|
|
||||||
def __enter__(self) -> ArrowLibrary:
|
def __enter__(self) -> ArrowLibrary:
|
||||||
return self
|
return self
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,9 @@
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import cast
|
from typing import cast
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
|
|
||||||
import numpy
|
import numpy
|
||||||
import pytest
|
import pytest
|
||||||
from numpy.testing import assert_equal, assert_allclose
|
from numpy.testing import assert_equal, assert_allclose
|
||||||
|
|
@ -11,6 +15,28 @@ from ..file import gdsii
|
||||||
from ..shapes import Path as MPath, Polygon
|
from ..shapes import Path as MPath, Polygon
|
||||||
|
|
||||||
|
|
||||||
|
def test_gdsii_import_requires_pheon_only_for_arrow_readers() -> None:
|
||||||
|
script = textwrap.dedent('''
|
||||||
|
from importlib import import_module
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.modules['pheon'] = None
|
||||||
|
from masque.file import gdsii
|
||||||
|
|
||||||
|
assert 'masque.file.gdsii.arrow' not in sys.modules
|
||||||
|
assert 'masque.file.gdsii.lazy_arrow' not in sys.modules
|
||||||
|
for reader in ('arrow', 'lazy_arrow'):
|
||||||
|
try:
|
||||||
|
import_module(f'{gdsii.__name__}.{reader}')
|
||||||
|
except ImportError as error:
|
||||||
|
assert 'Install masque[arrow]' in str(error), error
|
||||||
|
else:
|
||||||
|
raise AssertionError(f'{reader} imported without Pheon')
|
||||||
|
''')
|
||||||
|
result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
|
||||||
|
|
||||||
def test_gdsii_roundtrip(tmp_path: Path) -> None:
|
def test_gdsii_roundtrip(tmp_path: Path) -> None:
|
||||||
lib = Library()
|
lib = Library()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
|
from importlib.util import find_spec
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import textwrap
|
import textwrap
|
||||||
|
|
@ -7,20 +9,31 @@ import klamath
|
||||||
import numpy
|
import numpy
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
pytest.importorskip('pyarrow')
|
|
||||||
|
|
||||||
from .. import Ref, Label, PatternError
|
from .. import Ref, Label, PatternError
|
||||||
from ..library import Library
|
from ..library import Library
|
||||||
from ..pattern import Pattern
|
from ..pattern import Pattern
|
||||||
from ..repetition import Grid
|
from ..repetition import Grid
|
||||||
from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection
|
from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection
|
||||||
from ..file import gdsii
|
from ..file import gdsii
|
||||||
from ..file.gdsii import arrow as gdsii_arrow
|
|
||||||
from tools.generate_gds_perf import write_fixture
|
from tools.generate_gds_perf import write_fixture
|
||||||
|
|
||||||
|
|
||||||
if not gdsii_arrow.is_available():
|
if find_spec('pheon') is None:
|
||||||
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
|
pytest.skip('pheon package is not available', allow_module_level=True)
|
||||||
|
|
||||||
|
from ..file.gdsii import arrow as gdsii_arrow # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def test_arrow_import_reports_unloadable_native_library(tmp_path: Path) -> None:
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, '-c', 'from masque.file.gdsii import arrow'],
|
||||||
|
env={**os.environ, 'PHEON_LIBRARY': str(tmp_path / 'missing-library')},
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert 'Could not load Pheon library' in result.stderr
|
||||||
|
|
||||||
|
|
||||||
def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None:
|
def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,28 +1,30 @@
|
||||||
|
from importlib.util import find_spec
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import mmap
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import textwrap
|
import textwrap
|
||||||
|
|
||||||
import klamath
|
import klamath
|
||||||
|
from klamath.basic import KlamathError
|
||||||
import numpy
|
import numpy
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
pytest.importorskip('pyarrow')
|
|
||||||
|
|
||||||
from .. import PatternError, LibraryError
|
from .. import PatternError, LibraryError
|
||||||
from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView
|
from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView
|
||||||
from ..pattern import Pattern
|
from ..pattern import Pattern
|
||||||
from ..repetition import Grid
|
from ..repetition import Grid
|
||||||
from ..file import gdsii
|
from ..file import gdsii
|
||||||
from ..file.utils import preflight_source_aware
|
from ..file.utils import preflight_source_aware
|
||||||
from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow
|
|
||||||
from ..file.gdsii import arrow as gdsii_arrow
|
|
||||||
from ..file.gdsii import writer as gdsii_writer
|
from ..file.gdsii import writer as gdsii_writer
|
||||||
from tools.generate_gds_perf import write_fixture
|
from tools.generate_gds_perf import write_fixture
|
||||||
|
|
||||||
|
|
||||||
if not gdsii_arrow.is_available():
|
if find_spec('pheon') is None:
|
||||||
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
|
pytest.skip('pheon package is not available', allow_module_level=True)
|
||||||
|
|
||||||
|
from ..file.gdsii import arrow as gdsii_arrow # noqa: E402
|
||||||
|
from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('cached', [False, True])
|
@pytest.mark.parametrize('cached', [False, True])
|
||||||
|
|
@ -268,9 +270,10 @@ def test_gdsii_lazy_arrow_detached_batch_preserves_native_batching(
|
||||||
assert not mapped._cache
|
assert not mapped._cache
|
||||||
|
|
||||||
|
|
||||||
def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None:
|
@pytest.mark.parametrize('content', [b'not-a-gds', b''])
|
||||||
|
def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path, content: bytes) -> None:
|
||||||
gds_file = tmp_path / 'invalid.gds'
|
gds_file = tmp_path / 'invalid.gds'
|
||||||
gds_file.write_bytes(b'not-a-gds')
|
gds_file.write_bytes(content)
|
||||||
|
|
||||||
script = textwrap.dedent(f"""
|
script = textwrap.dedent(f"""
|
||||||
from masque.file.gdsii import lazy_arrow as gdsii_lazy_arrow
|
from masque.file.gdsii import lazy_arrow as gdsii_lazy_arrow
|
||||||
|
|
@ -290,6 +293,25 @@ def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) ->
|
||||||
assert 'KlamathError' in result.stdout
|
assert 'KlamathError' in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_gdsii_lazy_arrow_closes_mapping_when_scan_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
gds_file = tmp_path / 'invalid.gds'
|
||||||
|
gds_file.write_bytes(b'not-a-gds')
|
||||||
|
scan = gdsii_arrow._scan_buffer_to_arrow
|
||||||
|
mappings: list[mmap.mmap] = []
|
||||||
|
|
||||||
|
def record_mapping(source: bytes | mmap.mmap | memoryview) -> object:
|
||||||
|
assert isinstance(source, mmap.mmap)
|
||||||
|
mappings.append(source)
|
||||||
|
return scan(source)
|
||||||
|
|
||||||
|
monkeypatch.setattr(gdsii_arrow, '_scan_buffer_to_arrow', record_mapping)
|
||||||
|
with pytest.raises(KlamathError):
|
||||||
|
gdsii_lazy_arrow.readfile(gds_file)
|
||||||
|
|
||||||
|
assert len(mappings) == 1
|
||||||
|
assert mappings[0].closed
|
||||||
|
|
||||||
|
|
||||||
def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None:
|
def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None:
|
||||||
gds_file = tmp_path / 'invalid_path_type.gds'
|
gds_file = tmp_path / 'invalid_path_type.gds'
|
||||||
_write_invalid_path_type_fixture(gds_file)
|
_write_invalid_path_type_fixture(gds_file)
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ packages = ["masque"]
|
||||||
path = "masque/__init__.py"
|
path = "masque/__init__.py"
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
arrow = ["pyarrow", "cffi"]
|
arrow = ["pheon>=0.3,<0.4"]
|
||||||
oasis = ["fatamorgana>=0.14"]
|
oasis = ["fatamorgana>=0.14"]
|
||||||
dxf = ["ezdxf~=1.4", "pyclipper"]
|
dxf = ["ezdxf~=1.4", "pyclipper"]
|
||||||
svg = ["svgwrite"]
|
svg = ["svgwrite"]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue