From bd0c8c9d169c9983467ca181a47ebbdbb45bb1a9 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 01/15] [gdsii_arrow] add gdsii_arrow --- masque/file/gdsii_arrow.py | 453 +++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..e56a48e --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,453 @@ +# ruff: noqa: ARG001, F401 +""" +GDSII file format readers and writers using the `TODO` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_equal +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path, PolyCollection +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def _read_to_arrow( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> pyarrow.Array: + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + return arrow_arr + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_tups = [tuple(pair) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + rf = libarr['cells'].values.field('refs') + refs = dict( + offsets = rf.offsets.to_numpy(), + targets = rf.values.field('target').to_numpy(), + xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), + angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + scale = rf.values.field('mag').fill_null(1).to_numpy(), + rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), + rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + prop_off = rf.values.field('properties').offsets.to_numpy(), + prop_key = rf.values.field('properties').values.field('key').to_numpy(), + prop_val = rf.values.field('properties').values.field('value').to_pylist(), + ) + + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), + ) + + elements = dict( + boundaries = get_geom(libarr, 'boundaries'), + paths = get_geom(libarr, 'paths'), + boxes = get_geom(libarr, 'boxes'), + nodes = get_geom(libarr, 'nodes'), + texts = texts, + refs = refs, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').fill_null(0).to_numpy(), + path_type = paths.values.field('path_type').fill_null(0).to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').fill_null(0).to_numpy(), + paths.values.field('extension_end').fill_null(0).to_numpy(), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + raw_mode = raw_mode, + ) + + mlib = Library() + for cc in range(len(libarr['cells'])): + name = cell_names[cell_ids[cc]] + pat = Pattern() + _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def _grefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + targets = elem['targets'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_invert_y = elem['invert_y'][elem_slc][:elem_count] + elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] + elem_scale = elem['scale'][elem_slc][:elem_count] + elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] + elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] + elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] + rep_valid = elem['rep_valid'][elem_slc][:elem_count] + + + for ee in range(elem_count): + target = cell_names[targets[ee]] + offset = xy[ee] + mirr = elem_invert_y[ee] + rot = elem_angle_rad[ee] + mag = elem_scale[ee] + + rep: None | Grid = None + if rep_valid[ee]: + a_vector = elem_rep_xy0[ee] + b_vector = elem_rep_xy1[ee] + a_count, b_count = elem_rep_counts[ee] + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + pat.refs[target].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_strings = elem['string'][elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + offset = xy[ee] + string = elem_strings[ee] + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + mlabel = Label(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_widths = elem['width'][elem_slc][:elem_count] + elem_path_types = elem['path_type'][elem_slc][:elem_count] + elem_extensions = elem['extensions'][elem_slc][:elem_count] + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem_widths[ee] + cap_int = elem_path_types[ee] + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem_extensions[ee] + else: + cap_extensions = None + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + width=width, cap=cap,cap_extensions=cap_extensions) + pat.shapes[layer].append(path) + + +def _boundaries_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + xy_counts = xy_offs[1:] - xy_offs[:-1] + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + prop_counts = prop_offs[1:] - prop_offs[:-1] + elem_layer_inds = layer_inds[elem_slc][:elem_count] + + order = numpy.argsort(elem_layer_inds, stable=True) + unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): + ee_inds = order[ff:ff + nn] + layer = layer_tups[layer_ind] + propless_mask = prop_counts[ee_inds] == 0 + + poly_count_on_layer = propless_mask.sum() + if poly_count_on_layer == 1: + propless_mask[:] = 0 # Never make a 1-element collection + elif poly_count_on_layer > 1: + propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point + vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) + vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) + + for ii, ee in enumerate(ee_inds[propless_mask]): + vo = vertex_offsets[ii] + vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] + + polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) + pat.shapes[layer].append(polys) + + # Handle single polygons + for ee in ee_inds[~propless_mask]: + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} + + poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) + + +#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: +# return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From 18f12792c4ffbf66a2954e8977e3c82f7d8f4183 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:20:50 -0700 Subject: [PATCH 02/15] add some missing deps --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af8802c..ad03fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ [dependency-groups] dev = [ - "pytest", + "masque[arrow]", "masque[oasis]", "masque[dxf]", "masque[svg]", @@ -52,6 +52,7 @@ dev = [ "masque[text]", "masque[manhattanize]", "masque[manhattanize_slow]", + "masque[boolean]", "matplotlib>=3.10.8", "pytest>=9.0.2", "ruff>=0.15.5", @@ -66,6 +67,7 @@ build-backend = "hatchling.build" path = "masque/__init__.py" [project.optional-dependencies] +arrow = ["pyarrow", "cffi"] oasis = ["fatamorgana~=0.11"] dxf = ["ezdxf~=1.4"] svg = ["svgwrite"] @@ -121,4 +123,3 @@ mypy_path = "stubs" python_version = "3.11" strict = false check_untyped_defs = true - From 09fec67a2173e9d777c00672bb26fa4b9898a691 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:21:59 -0700 Subject: [PATCH 03/15] [gdsii_arrow] misc correctness work --- masque/file/gdsii_arrow.py | 173 +++++++++++++++++++++++------- masque/test/test_gdsii_arrow.py | 180 ++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 37 deletions(-) create mode 100644 masque/test/test_gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index e56a48e..546d575 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -25,12 +25,16 @@ Notes: """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable +from importlib.machinery import EXTENSION_SUFFIXES +import importlib.util import io import mmap import logging +import os import pathlib import gzip import string +import sys from pprint import pformat import numpy @@ -49,9 +53,10 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +clib: Any | None = None + path_cap_map = { 0: Path.Cap.Flush, @@ -65,16 +70,123 @@ def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) +def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]: + layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16) + dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16) + return numpy.stack((layer, dtype), axis=-1) + + +def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]: + a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64) + b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64) + return numpy.stack((a_count, b_count), axis=-1) + + +def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]: + xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32) + yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32) + return numpy.stack((xx, yy), axis=-1) + + +def _local_library_filename() -> str: + if sys.platform.startswith('linux'): + return 'libklamath_rs_ext.so' + if sys.platform == 'darwin': + return 'libklamath_rs_ext.dylib' + if sys.platform == 'win32': + return 'klamath_rs_ext.dll' + raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}') + + +def _installed_library_candidates() -> list[pathlib.Path]: + candidates: list[pathlib.Path] = [] + + try: + spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext') + except ModuleNotFoundError: + spec = None + if spec is not None and spec.origin is not None: + candidates.append(pathlib.Path(spec.origin)) + + try: + pkg_spec = importlib.util.find_spec('klamath_rs_ext') + except ModuleNotFoundError: + pkg_spec = None + if pkg_spec is not None and pkg_spec.submodule_search_locations is not None: + for location in pkg_spec.submodule_search_locations: + pkg_dir = pathlib.Path(location) + for suffix in EXTENSION_SUFFIXES: + candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}'))) + + return candidates + + +def _repo_library_candidates() -> list[pathlib.Path]: + repo_root = pathlib.Path(__file__).resolve().parents[2] + library_name = _local_library_filename() + return [ + repo_root / 'klamath-rs' / 'target' / 'release' / library_name, + repo_root / 'klamath-rs' / 'target' / 'debug' / library_name, + ] + + +def find_klamath_rs_library() -> pathlib.Path | None: + env_path = os.environ.get('KLAMATH_RS_EXT_LIB') + if env_path: + candidate = pathlib.Path(env_path).expanduser() + if candidate.exists(): + return candidate.resolve() + + seen: set[pathlib.Path] = set() + for candidate in _installed_library_candidates() + _repo_library_candidates(): + resolved = candidate.expanduser() + if resolved in seen: + continue + seen.add(resolved) + if resolved.exists(): + return resolved.resolve() + return None + + +def is_available() -> bool: + return find_klamath_rs_library() is not None + + +def _get_clib() -> Any: + global clib + if clib is None: + lib_path = find_klamath_rs_library() + if lib_path is None: + raise ImportError( + 'Could not locate klamath_rs_ext shared library. ' + 'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` ' + 'or set KLAMATH_RS_EXT_LIB to the built library path.' + ) + clib = ffi.dlopen(str(lib_path)) + return clib + + +def _read_annotations( + prop_offs: NDArray[numpy.integer[Any]], + prop_key: NDArray[numpy.integer[Any]], + prop_val: list[str], + ee: int, + ) -> annotations_t: + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii >= prop_ff: + return None + return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + def _read_to_arrow( filename: str | pathlib.Path, *args, **kwargs, ) -> pyarrow.Array: - path = pathlib.Path(filename) - path.resolve() + path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - clib.read_path(str(path).encode(), ptr_array, ptr_schema) + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) @@ -132,7 +244,7 @@ def read_arrow( """ library_info = _read_header(libarr) - layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) layer_tups = [tuple(pair) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() @@ -155,14 +267,14 @@ def read_arrow( refs = dict( offsets = rf.offsets.to_numpy(), targets = rf.values.field('target').to_numpy(), - xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), scale = rf.values.field('mag').fill_null(1).to_numpy(), rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), + rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), + rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), prop_off = rf.values.field('properties').offsets.to_numpy(), prop_key = rf.values.field('properties').values.field('key').to_numpy(), prop_val = rf.values.field('properties').values.field('value').to_pylist(), @@ -172,7 +284,7 @@ def read_arrow( texts = dict( offsets = txt.offsets.to_numpy(), layer_inds = txt.values.field('layer').to_numpy(), - xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()), string = txt.values.field('string').to_pylist(), prop_off = txt.values.field('properties').offsets.to_numpy(), prop_key = txt.values.field('properties').values.field('key').to_numpy(), @@ -222,9 +334,9 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: Read the file header and create the library_info dict. """ library_info = dict( - name = libarr['lib_name'], - meters_per_unit = libarr['meters_per_db_unit'], - logical_units_per_unit = libarr['user_units_per_db_unit'], + name = libarr['lib_name'].as_py(), + meters_per_unit = libarr['meters_per_db_unit'].as_py(), + logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(), ) return library_info @@ -245,6 +357,8 @@ def _grefs_to_mrefs( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_targets = targets[elem_slc][:elem_count] + elem_xy = xy[elem_slc][:elem_count] elem_invert_y = elem['invert_y'][elem_slc][:elem_count] elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] elem_scale = elem['scale'][elem_slc][:elem_count] @@ -255,8 +369,8 @@ def _grefs_to_mrefs( for ee in range(elem_count): - target = cell_names[targets[ee]] - offset = xy[ee] + target = cell_names[elem_targets[ee]] + offset = elem_xy[ee] mirr = elem_invert_y[ee] rot = elem_angle_rad[ee] mag = elem_scale[ee] @@ -268,11 +382,7 @@ def _grefs_to_mrefs( a_count, b_count = elem_rep_counts[ee] rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -293,19 +403,16 @@ def _texts_to_labels( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] - offset = xy[ee] + offset = elem_xy[ee] string = elem_strings[ee] - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -345,11 +452,7 @@ def _gpaths_to_mpaths( else: cap_extensions = None - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) @@ -406,11 +509,7 @@ def _boundaries_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py new file mode 100644 index 0000000..26713e9 --- /dev/null +++ b/masque/test/test_gdsii_arrow.py @@ -0,0 +1,180 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..shapes import Path as MPath +from ..file import gdsii, gdsii_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: + if annotations is None: + return None + return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) + + +def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(values, dtype=float)).astype(int) + if arr.ndim == 1: + return tuple(arr.tolist()) + return tuple(tuple(row.tolist()) for row in arr) + + +def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: + if isinstance(shape, MPath): + cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) + return [( + 'path', + layer, + _coord_key(shape.vertices), + _coord_key(shape.offset), + int(round(float(shape.width))), + shape.cap.name, + cap_extensions, + _annotations_key(shape.annotations), + )] + + keys = [] + for poly in shape.to_polygons(): + keys.append(( + 'polygon', + layer, + _coord_key(poly.vertices), + _coord_key(poly.offset), + _annotations_key(poly.annotations), + )) + return keys + + +def _ref_key(target: str, ref: object) -> tuple[object, ...]: + repetition = None + if ref.repetition is not None: + repetition = ( + _coord_key(ref.repetition.a_vector), + int(ref.repetition.a_count), + _coord_key(ref.repetition.b_vector), + int(ref.repetition.b_count), + ) + return ( + target, + _coord_key(ref.offset), + round(float(ref.rotation), 8), + round(float(ref.scale), 8), + bool(ref.mirrored), + repetition, + _annotations_key(ref.annotations), + ) + + +def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: + return ( + layer, + label.string, + _coord_key(label.offset), + _annotations_key(label.annotations), + ) + + +def _pattern_summary(pattern: Pattern) -> dict[str, object]: + shape_keys: list[tuple[object, ...]] = [] + for layer, shapes in pattern.shapes.items(): + for shape in shapes: + shape_keys.extend(_shape_key(shape, layer)) + + ref_keys = [ + _ref_key(target, ref) + for target, refs in pattern.refs.items() + for ref in refs + ] + + label_keys = [ + _label_key(layer, label) + for layer, labels in pattern.labels.items() + for label in labels + ] + + return { + 'shapes': sorted(shape_keys), + 'refs': sorted(ref_keys), + 'labels': sorted(label_keys), + } + + +def _library_summary(lib: Library) -> dict[str, dict[str, object]]: + return {name: _pattern_summary(pattern) for name, pattern in lib.items()} + + +def _make_arrow_test_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) + lib['leaf'] = leaf + + child = Pattern() + child.path( + (2, 0), + vertices=[[0, 0], [15, 5], [30, 5]], + width=6, + cap=MPath.Cap.SquareCustom, + cap_extensions=(2, 4), + annotations={'2': ['child-path']}, + ) + child.label((11, 0), string='CHILD', offset=(7, 8), annotations={'11': ['child-label']}) + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25, annotations={'12': ['child-ref']}) + lib['child'] = child + + sibling = Pattern() + sibling.polygon((3, 0), vertices=[[0, 0], [5, 0], [5, 6], [0, 6]]) + sibling.label((12, 0), string='SIB', offset=(1, 2), annotations={'13': ['sib-label']}) + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + annotations={'14': ['sib-ref']}, + ) + lib['sibling'] = sibling + + top = Pattern() + top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) + lib['top'] = top + + return lib + + +def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + +def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_smoke.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert 'TOP' in lib + assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 From 9df42000b71a15292f7ea4861f0caf55132a7ad2 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:22:27 -0700 Subject: [PATCH 04/15] [gdsii] add some profiling helpers --- examples/generate_gds_perf.py | 5 + examples/profile_gdsii_readers.py | 74 ++++ masque/file/gdsii_perf.py | 633 ++++++++++++++++++++++++++++++ masque/test/test_gdsii_perf.py | 24 ++ 4 files changed, 736 insertions(+) create mode 100644 examples/generate_gds_perf.py create mode 100644 examples/profile_gdsii_readers.py create mode 100644 masque/file/gdsii_perf.py create mode 100644 masque/test/test_gdsii_perf.py diff --git a/examples/generate_gds_perf.py b/examples/generate_gds_perf.py new file mode 100644 index 0000000..6bdd999 --- /dev/null +++ b/examples/generate_gds_perf.py @@ -0,0 +1,5 @@ +from masque.file.gdsii_perf import main + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py new file mode 100644 index 0000000..c77c589 --- /dev/null +++ b/examples/profile_gdsii_readers.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import importlib +import json +import time +from pathlib import Path + +from masque import LibraryError + + +READERS = { + 'gdsii': ('masque.file.gdsii', 'readfile'), + 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), + } + + +def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: + assert hasattr(lib, '__len__') + assert hasattr(lib, 'tops') + tops = lib.tops() # type: ignore[no-any-return, attr-defined] + try: + unique_top = lib.top() # type: ignore[no-any-return, attr-defined] + except LibraryError: + unique_top = None + + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'library_name': info['name'], + 'cell_count': len(lib), # type: ignore[arg-type] + 'topcells': tops, + 'topcell': unique_top, + } + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') + parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--path', type=Path, required=True) + parser.add_argument('--warmup', type=int, default=1) + parser.add_argument('--repeat', type=int, default=1) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + module_name, attr_name = READERS[args.reader] + readfile = getattr(importlib.import_module(module_name), attr_name) + path = args.path.expanduser().resolve() + + for _ in range(args.warmup): + readfile(path) + + runs = [] + for _ in range(args.repeat): + start = time.perf_counter() + lib, info = readfile(path) + elapsed_s = time.perf_counter() - start + runs.append(_summarize(path, elapsed_s, info, lib)) + + print(json.dumps({ + 'reader': args.reader, + 'warmup': args.warmup, + 'repeat': args.repeat, + 'runs': runs, + }, indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/file/gdsii_perf.py b/masque/file/gdsii_perf.py new file mode 100644 index 0000000..38d1a7d --- /dev/null +++ b/masque/file/gdsii_perf.py @@ -0,0 +1,633 @@ +""" +Synthetic GDS fixture generation for reader/writer performance testing. + +The presets here are intentionally hierarchical and deterministic. They aim to +approximate a pair of real-world layout families discussed during GDS reader and +writer work: + +* `many_cells`: tens of thousands of cells, moderate reference count, very heavy + box usage after flattening, and moderate polygon density. +* `many_instances`: a much smaller cell library with very high reference count, + similar box density, and far fewer polygons. + +Fixtures are written by streaming structures through `klamath` directly so large +benchmark files can be produced without first materializing an equally large +`masque.Library` in Python. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +import argparse +import json +import math + +import numpy +import klamath +from klamath import elements + + +EMPTY_PROPERTIES: dict[int, bytes] = {} +METERS_PER_DB_UNIT = 1e-9 +USER_UNITS_PER_DB_UNIT = 1e-3 +TOTAL_LAYERS = 200 + + +@dataclass(frozen=True) +class FixturePreset: + name: str + total_layers: int + box_layers: int + heavy_box_layers: int + polygon_layers: int + box_cells: int + poly_cells: int + box_wrappers: int + poly_wrappers: int + box_clusters: int + poly_clusters: int + box_cluster_refs: int + poly_cluster_refs: int + top_direct_box_refs: int + top_direct_poly_refs: int + heavy_boxes_per_cell: int + regular_boxes_per_cell: int + polygons_per_cell: int + path_stride: int + text_stride: int + box_cluster_array: tuple[int, int] + top_box_array: tuple[int, int] + poly_cluster_array: tuple[int, int] + top_poly_array: tuple[int, int] + rare_annotation_stride: int + + +PRESETS: dict[str, FixturePreset] = { + 'many_cells': FixturePreset( + name='many_cells', + total_layers=TOTAL_LAYERS, + box_layers=20, + heavy_box_layers=3, + polygon_layers=20, + box_cells=17_000, + poly_cells=6_000, + box_wrappers=18_000, + poly_wrappers=6_000, + box_clusters=2_000, + poly_clusters=999, + box_cluster_refs=24, + poly_cluster_refs=16, + top_direct_box_refs=21_000, + top_direct_poly_refs=7_000, + heavy_boxes_per_cell=6, + regular_boxes_per_cell=2, + polygons_per_cell=50, + path_stride=2, + text_stride=3, + box_cluster_array=(24, 16), + top_box_array=(8, 8), + poly_cluster_array=(4, 2), + top_poly_array=(3, 2), + rare_annotation_stride=1_250, + ), + 'many_instances': FixturePreset( + name='many_instances', + total_layers=TOTAL_LAYERS, + box_layers=25, + heavy_box_layers=3, + polygon_layers=10, + box_cells=2_500, + poly_cells=500, + box_wrappers=1_000, + poly_wrappers=500, + box_clusters=1_000, + poly_clusters=499, + box_cluster_refs=1_200, + poly_cluster_refs=400, + top_direct_box_refs=102_001, + top_direct_poly_refs=0, + heavy_boxes_per_cell=40, + regular_boxes_per_cell=16, + polygons_per_cell=60, + path_stride=1, + text_stride=2, + box_cluster_array=(1, 1), + top_box_array=(1, 1), + poly_cluster_array=(1, 1), + top_poly_array=(1, 1), + rare_annotation_stride=250, + ), + } + + +@dataclass(frozen=True) +class FixtureManifest: + preset: str + scale: float + gds_path: str + library_name: str + cells: int + refs: int + layers: int + box_layers: int + heavy_box_layers: list[list[int]] + polygon_layers: list[list[int]] + hierarchical_boxes_per_heavy_layer: int + hierarchical_boxes_per_regular_layer: int + hierarchical_polygons_total: int + hierarchical_paths_total: int + hierarchical_texts_total: int + flattened_box_placements: int + flattened_poly_placements: int + estimated_flat_boxes_per_heavy_layer: int + estimated_flat_polygons_per_active_polygon_layer: int + + +def _scaled_count(value: int, scale: float, minimum: int = 0) -> int: + if value == 0: + return 0 + scaled = int(math.ceil(value * scale)) + return max(minimum, scaled) + + +def _scaled_preset(preset: FixturePreset, scale: float) -> FixturePreset: + if scale <= 0: + raise ValueError(f'scale must be positive, got {scale!r}') + + return FixturePreset( + name=preset.name, + total_layers=preset.total_layers, + box_layers=min(preset.box_layers, preset.total_layers), + heavy_box_layers=min(preset.heavy_box_layers, preset.box_layers), + polygon_layers=min(preset.polygon_layers, preset.total_layers), + box_cells=_scaled_count(preset.box_cells, scale, minimum=1), + poly_cells=_scaled_count(preset.poly_cells, scale, minimum=1), + box_wrappers=_scaled_count(preset.box_wrappers, scale), + poly_wrappers=_scaled_count(preset.poly_wrappers, scale), + box_clusters=_scaled_count(preset.box_clusters, scale, minimum=1), + poly_clusters=_scaled_count(preset.poly_clusters, scale, minimum=1), + box_cluster_refs=_scaled_count(preset.box_cluster_refs, scale, minimum=1), + poly_cluster_refs=_scaled_count(preset.poly_cluster_refs, scale, minimum=1), + top_direct_box_refs=_scaled_count(preset.top_direct_box_refs, scale), + top_direct_poly_refs=_scaled_count(preset.top_direct_poly_refs, scale), + heavy_boxes_per_cell=max(1, preset.heavy_boxes_per_cell), + regular_boxes_per_cell=max(1, preset.regular_boxes_per_cell), + polygons_per_cell=max(1, preset.polygons_per_cell), + path_stride=max(1, preset.path_stride), + text_stride=max(1, preset.text_stride), + box_cluster_array=preset.box_cluster_array, + top_box_array=preset.top_box_array, + poly_cluster_array=preset.poly_cluster_array, + top_poly_array=preset.top_poly_array, + rare_annotation_stride=max(1, _scaled_count(preset.rare_annotation_stride, scale, minimum=1)), + ) + + +def _rect_xy(xmin: int, ymin: int, xmax: int, ymax: int) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + return numpy.array( + [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]], + dtype=numpy.int32, + ) + + +def _poly_xy(points: list[tuple[int, int]]) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + closed = points + [points[0]] + return numpy.array(closed, dtype=numpy.int32) + + +def _sref( + target: str, + xy: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([xy], dtype=numpy.int32), + colrow=None, + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _aref( + target: str, + origin: tuple[int, int], + counts: tuple[int, int], + step: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + cols, rows = counts + dx, dy = step + xy = numpy.array( + [ + origin, + (origin[0] + cols * dx, origin[1]), + (origin[0], origin[1] + rows * dy), + ], + dtype=numpy.int32, + ) + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=xy, + colrow=(cols, rows), + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _annotation(index: int) -> dict[int, bytes]: + return {1: f'perf-{index}'.encode('ASCII')} + + +def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 17) * 600 + ybase = (index // 17) * 180 + + for layer in range(cfg.heavy_box_layers): + for box_idx in range(cfg.heavy_boxes_per_cell): + x0 = xbase + box_idx * 22 + y0 = ybase + layer * 40 + width = 10 + ((index + box_idx + layer) % 7) * 6 + height = 10 + ((index * 3 + box_idx + layer) % 5) * 8 + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 and box_idx == 0 and layer == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=properties, + )) + + for layer in range(cfg.heavy_box_layers, cfg.box_layers): + for box_idx in range(cfg.regular_boxes_per_cell): + x0 = xbase + box_idx * 38 + y0 = ybase + (layer - cfg.heavy_box_layers) * 28 + 400 + width = 18 + ((index + layer + box_idx) % 9) * 4 + height = 12 + ((index + 2 * layer + box_idx) % 6) * 5 + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=EMPTY_PROPERTIES, + )) + + return cell_elements + + +def _make_poly_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 19) * 900 + ybase = (index // 19) * 260 + + for poly_idx in range(cfg.polygons_per_cell): + layer = poly_idx % cfg.polygon_layers + dx = xbase + (poly_idx % 5) * 120 + dy = ybase + (poly_idx // 5) * 80 + size = 18 + ((index + poly_idx + layer) % 11) * 7 + points = [ + (dx, dy), + (dx + size, dy + size // 5), + (dx + size + size // 3, dy + size), + (dx + size // 2, dy + size + size // 2), + (dx - size // 4, dy + size // 2), + ] + properties = _annotation(index) if poly_idx == 0 and index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_poly_xy(points), + properties=properties, + )) + + if index % cfg.path_stride == 0: + layer = index % cfg.polygon_layers + cell_elements.append(elements.Path( + layer=(layer, 1), + path_type=2, + width=12 + (index % 5) * 4, + extension=(0, 0), + xy=numpy.array( + [ + [xbase, ybase + 900], + [xbase + 240, ybase + 930], + [xbase + 420, ybase + 960], + ], + dtype=numpy.int32, + ), + properties=EMPTY_PROPERTIES, + )) + + if index % cfg.text_stride == 0: + layer = index % cfg.polygon_layers + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Text( + layer=(layer, 2), + presentation=0, + path_type=0, + width=0, + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([[xbase + 64, ybase + 1536]], dtype=numpy.int32), + string=f'T{index:05d}'.encode('ASCII'), + properties=properties, + )) + + return cell_elements + + +def _write_struct(stream: Any, name: str, cell_elements: list[elements.Element]) -> None: + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=cell_elements) + + +def _box_name(index: int) -> str: + return f'box_{index:05d}' + + +def _poly_name(index: int) -> str: + return f'poly_{index:05d}' + + +def _box_wrapper_name(index: int) -> str: + return f'box_wrap_{index:05d}' + + +def _poly_wrapper_name(index: int) -> str: + return f'poly_wrap_{index:05d}' + + +def _box_cluster_name(index: int) -> str: + return f'box_cluster_{index:05d}' + + +def _poly_cluster_name(index: int) -> str: + return f'poly_cluster_{index:05d}' + + +def _write_box_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_cells): + _write_struct(stream, _box_name(idx), _make_box_cell(_box_name(idx), idx, cfg)) + + +def _write_poly_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.poly_cells): + _write_struct(stream, _poly_name(idx), _make_poly_cell(_poly_name(idx), idx, cfg)) + + +def _write_wrappers(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_wrappers): + target = _box_name(idx % cfg.box_cells) + origin = ((idx % 97) * 2_000, (idx // 97) * 2_000) + _write_struct(stream, _box_wrapper_name(idx), [_sref(target, origin)]) + + for idx in range(cfg.poly_wrappers): + target = _poly_name(idx % cfg.poly_cells) + origin = ((idx % 61) * 3_200, (idx // 61) * 3_200) + _write_struct(stream, _poly_wrapper_name(idx), [_sref(target, origin)]) + + +def _write_box_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + for idx in range(cfg.box_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.box_cluster_refs): + target = _box_name((idx * cfg.box_cluster_refs + ref_idx) % cfg.box_cells) + origin = ( + (ref_idx % 6) * 48_000, + (ref_idx // 6) * 48_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.box_cluster_array, (720, 900))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _box_cluster_name(idx), cell_elements) + + +def _write_poly_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + for idx in range(cfg.poly_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.poly_cluster_refs): + target = _poly_name((idx * cfg.poly_cluster_refs + ref_idx) % cfg.poly_cells) + origin = ( + (ref_idx % 10) * 96_000, + (ref_idx // 10) * 96_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.poly_cluster_array, (12_000, 8_500))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _poly_cluster_name(idx), cell_elements) + + +def _top_box_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.box_wrappers): + refs.append(_sref( + _box_wrapper_name(idx), + ((idx % 240) * 240_000, (idx // 240) * 240_000), + )) + + for idx in range(cfg.box_clusters): + refs.append(_sref( + _box_cluster_name(idx), + ((idx % 100) * 800_000, (idx // 100) * 800_000 + 14_000_000), + )) + + for idx in range(cfg.top_direct_box_refs): + target = _box_name(idx % cfg.box_cells) + origin = ( + (idx % 150) * 160_000, + (idx // 150) * 160_000 + 26_000_000, + ) + if cfg.top_box_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_box_array, (1_100, 1_350))) + + return refs + + +def _top_poly_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.poly_wrappers): + refs.append(_sref( + _poly_wrapper_name(idx), + ((idx % 180) * 360_000, (idx // 180) * 360_000 + 44_000_000), + )) + + for idx in range(cfg.poly_clusters): + refs.append(_sref( + _poly_cluster_name(idx), + ((idx % 70) * 1_100_000, (idx // 70) * 1_100_000 + 58_000_000), + )) + + for idx in range(cfg.top_direct_poly_refs): + target = _poly_name(idx % cfg.poly_cells) + origin = ( + (idx % 110) * 420_000, + (idx // 110) * 420_000 + 72_000_000, + ) + if cfg.top_poly_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_poly_array, (16_000, 14_000))) + + return refs + + +def _write_top(stream: Any, cfg: FixturePreset) -> None: + cell_elements: list[elements.Element] = [] + cell_elements.extend(_top_box_refs(cfg)) + cell_elements.extend(_top_poly_refs(cfg)) + _write_struct(stream, 'TOP', cell_elements) + + +def _poly_paths_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.path_stride + 1 + + +def _poly_texts_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.text_stride + 1 + + +def _ref_instances_per_box_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] + return array_refs * array_mult + (cfg.box_cluster_refs - array_refs) + + +def _ref_instances_per_poly_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] + return array_refs * array_mult + (cfg.poly_cluster_refs - array_refs) + + +def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> FixtureManifest: + base = PRESETS[preset] + cfg = _scaled_preset(base, scale) + + flattened_box_placements = ( + cfg.box_wrappers + + cfg.box_clusters * _ref_instances_per_box_cluster(cfg) + + cfg.top_direct_box_refs * cfg.top_box_array[0] * cfg.top_box_array[1] + ) + flattened_poly_placements = ( + cfg.poly_wrappers + + cfg.poly_clusters * _ref_instances_per_poly_cluster(cfg) + + cfg.top_direct_poly_refs * cfg.top_poly_array[0] * cfg.top_poly_array[1] + ) + polygon_layers = max(1, cfg.polygon_layers) + polys_per_layer = (cfg.poly_cells * cfg.polygons_per_cell) // polygon_layers + + return FixtureManifest( + preset=cfg.name, + scale=scale, + gds_path=str(Path(path)), + library_name=f'masque-perf-{cfg.name}', + cells=cfg.box_cells + cfg.poly_cells + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + 1, + refs=( + cfg.box_wrappers + + cfg.poly_wrappers + + cfg.box_clusters * cfg.box_cluster_refs + + cfg.poly_clusters * cfg.poly_cluster_refs + + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + + cfg.top_direct_box_refs + cfg.top_direct_poly_refs + ), + layers=cfg.total_layers, + box_layers=cfg.box_layers, + heavy_box_layers=[[layer, 0] for layer in range(cfg.heavy_box_layers)], + polygon_layers=[[layer, 0] for layer in range(cfg.polygon_layers)], + hierarchical_boxes_per_heavy_layer=cfg.box_cells * cfg.heavy_boxes_per_cell, + hierarchical_boxes_per_regular_layer=cfg.box_cells * cfg.regular_boxes_per_cell, + hierarchical_polygons_total=cfg.poly_cells * cfg.polygons_per_cell, + hierarchical_paths_total=_poly_paths_total(cfg), + hierarchical_texts_total=_poly_texts_total(cfg), + flattened_box_placements=flattened_box_placements, + flattened_poly_placements=flattened_poly_placements, + estimated_flat_boxes_per_heavy_layer=flattened_box_placements * cfg.heavy_boxes_per_cell, + estimated_flat_polygons_per_active_polygon_layer=flattened_poly_placements * polys_per_layer // cfg.poly_cells if cfg.poly_cells else 0, + ) + + +def write_fixture( + path: str | Path, + *, + preset: str, + scale: float = 1.0, + write_manifest: bool = True, + ) -> FixtureManifest: + if preset not in PRESETS: + known = ', '.join(sorted(PRESETS)) + raise KeyError(f'unknown preset {preset!r}; expected one of: {known}') + + manifest = fixture_manifest(path, preset, scale) + cfg = _scaled_preset(PRESETS[preset], scale) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + + with output.open('wb') as stream: + header = klamath.library.FileHeader( + name=manifest.library_name.encode('ASCII'), + user_units_per_db_unit=USER_UNITS_PER_DB_UNIT, + meters_per_db_unit=METERS_PER_DB_UNIT, + ) + header.write(stream) + _write_box_cells(stream, cfg) + _write_poly_cells(stream, cfg) + _write_wrappers(stream, cfg) + _write_box_clusters(stream, cfg) + _write_poly_clusters(stream, cfg) + _write_top(stream, cfg) + klamath.records.ENDLIB.write(stream, None) + + if write_manifest: + manifest_path = output.with_suffix(output.suffix + '.json') + manifest_path.write_text(json.dumps(asdict(manifest), indent=2, sort_keys=True) + '\n') + + return manifest + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Generate synthetic GDS fixtures for GDS reader/writer performance work.') + parser.add_argument( + 'preset', + nargs='?', + default='many_cells', + choices=sorted(PRESETS), + help='Fixture family to generate.', + ) + parser.add_argument( + 'output', + nargs='?', + help='Output .gds path. Defaults to build/gds_perf/.gds', + ) + parser.add_argument( + '--scale', + type=float, + default=1.0, + help='Scale the preset counts down or up while keeping the same shape mix. Default: 1.0', + ) + parser.add_argument( + '--no-manifest', + action='store_true', + help='Do not write the sidecar JSON manifest.', + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + output = Path(args.output) if args.output is not None else Path('build/gds_perf') / f'{args.preset}.gds' + manifest = write_fixture(output, preset=args.preset, scale=args.scale, write_manifest=not args.no_manifest) + print(json.dumps(asdict(manifest), indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/test/test_gdsii_perf.py b/masque/test/test_gdsii_perf.py new file mode 100644 index 0000000..a595fe8 --- /dev/null +++ b/masque/test/test_gdsii_perf.py @@ -0,0 +1,24 @@ +from dataclasses import asdict +import json +from pathlib import Path + +from ..file import gdsii +from ..file.gdsii_perf import fixture_manifest, write_fixture + + +def test_gdsii_perf_fixture_smoke(tmp_path: Path) -> None: + output = tmp_path / 'many_cells.gds' + manifest = write_fixture(output, preset='many_cells', scale=0.002) + expected = fixture_manifest(output, preset='many_cells', scale=0.002) + + assert output.exists() + assert manifest == expected + + sidecar = json.loads(output.with_suffix('.gds.json').read_text()) + assert sidecar == asdict(manifest) + + read_lib, info = gdsii.readfile(output) + assert info['name'] == manifest.library_name + assert len(read_lib) == manifest.cells + assert 'TOP' in read_lib + assert len(read_lib['TOP'].refs) > 0 From d3870662286e6feba03da5058722e6ae3d37d3a0 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:46:53 -0700 Subject: [PATCH 05/15] [Label / Ref / Grid] add raw constructors --- masque/label.py | 16 ++++++++++++++++ masque/ref.py | 20 ++++++++++++++++++++ masque/repetition.py | 16 ++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/masque/label.py b/masque/label.py index 6a2a61f..98a65df 100644 --- a/masque/label.py +++ b/masque/label.py @@ -53,6 +53,22 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + string: str, + *, + offset: NDArray[numpy.float64], + repetition: Repetition | None = None, + annotations: annotations_t | None = None, + ) -> Self: + new = cls.__new__(cls) + new._string = string + new._offset = offset + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> Self: return type(self)( string=self.string, diff --git a/masque/ref.py b/masque/ref.py index 268d8d4..1d71432 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -86,6 +86,26 @@ class Ref( self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + *, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + scale: float, + repetition: Repetition | None, + annotations: annotations_t | None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._rotation = rotation % (2 * pi) + new._scale = scale + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> 'Ref': new = Ref( offset=self.offset.copy(), diff --git a/masque/repetition.py b/masque/repetition.py index 1e12fcd..9e8af26 100644 --- a/masque/repetition.py +++ b/masque/repetition.py @@ -113,6 +113,22 @@ class Grid(Repetition): self.a_count = a_count self.b_count = b_count + @classmethod + def _from_raw( + cls: type[GG], + *, + a_vector: NDArray[numpy.float64], + a_count: int, + b_vector: NDArray[numpy.float64], + b_count: int, + ) -> GG: + new = cls.__new__(cls) + new._a_vector = a_vector + new._b_vector = b_vector + new._a_count = int(a_count) + new._b_count = int(b_count) + return new + @classmethod def aligned( cls: type[GG], From 28562f73f634a62f75a1a7a9043024835308f7f4 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:47:41 -0700 Subject: [PATCH 06/15] [gdsii_arrow] further improvements to speed --- examples/profile_gdsii_readers.py | 71 ++++++++++++--- masque/file/gdsii_arrow.py | 144 ++++++++++++++++++++---------- masque/test/test_gdsii_arrow.py | 81 +++++++++++++++++ 3 files changed, 235 insertions(+), 61 deletions(-) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index c77c589..129a391 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -5,17 +5,18 @@ import importlib import json import time from pathlib import Path +from typing import Any from masque import LibraryError -READERS = { - 'gdsii': ('masque.file.gdsii', 'readfile'), - 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), +READERS: dict[str, tuple[str, tuple[str, ...]]] = { + 'gdsii': ('masque.file.gdsii', ('readfile',)), + 'gdsii_arrow': ('masque.file.gdsii_arrow', ('readfile', 'arrow_import', 'arrow_convert')), } -def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: +def _summarize_library(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: assert hasattr(lib, '__len__') assert hasattr(lib, 'tops') tops = lib.tops() # type: ignore[no-any-return, attr-defined] @@ -34,12 +35,50 @@ def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: objec } +def _summarize_arrow_import(path: Path, elapsed_s: float, arrow_arr: Any) -> dict[str, object]: + libarr = arrow_arr[0] + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': len(arrow_arr), + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + + +def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: + start = time.perf_counter() + + if stage == 'readfile': + lib, info = module.readfile(path) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + if stage == 'arrow_import': + arrow_arr = module._read_to_arrow(path) + elapsed_s = time.perf_counter() - start + return _summarize_arrow_import(path, elapsed_s, arrow_arr) + + if stage == 'arrow_convert': + arrow_arr = module._read_to_arrow(path) + libarr = arrow_arr[0] + start = time.perf_counter() + lib, info = module.read_arrow(libarr) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + raise ValueError(f'Unsupported stage {stage!r}') + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--stage', default='readfile') parser.add_argument('--path', type=Path, required=True) parser.add_argument('--warmup', type=int, default=1) parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--output-json', type=Path) return parser @@ -47,26 +86,32 @@ def main(argv: list[str] | None = None) -> int: parser = build_arg_parser() args = parser.parse_args(argv) - module_name, attr_name = READERS[args.reader] - readfile = getattr(importlib.import_module(module_name), attr_name) + module_name, stages = READERS[args.reader] + if args.stage not in stages: + parser.error(f'reader {args.reader!r} only supports stages: {", ".join(stages)}') + + module = importlib.import_module(module_name) path = args.path.expanduser().resolve() for _ in range(args.warmup): - readfile(path) + _profile_stage(module, args.stage, path) runs = [] for _ in range(args.repeat): - start = time.perf_counter() - lib, info = readfile(path) - elapsed_s = time.perf_counter() - start - runs.append(_summarize(path, elapsed_s, info, lib)) + runs.append(_profile_stage(module, args.stage, path)) - print(json.dumps({ + payload = { 'reader': args.reader, + 'stage': args.stage, 'warmup': args.warmup, 'repeat': args.repeat, 'runs': runs, - }, indent=2, sort_keys=True)) + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(rendered + '\n') + print(rendered) return 0 diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 546d575..f6a7739 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -57,6 +57,8 @@ ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchem clib: Any | None = None +ZERO_OFFSET = numpy.zeros(2) + path_cap_map = { 0: Path.Cap.Flush, @@ -263,6 +265,29 @@ def read_arrow( ) return elem + def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('boundary_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = batches.values.field('vertices').offsets.to_numpy() // 2, + poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(), + poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), + ) + + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: + boundaries = libarr['cells'].values.field('boundary_props') + return dict( + offsets = boundaries.offsets.to_numpy(), + layer_inds = boundaries.values.field('layer').to_numpy(), + vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2, + prop_off = boundaries.values.field('properties').offsets.to_numpy(), + prop_key = boundaries.values.field('properties').values.field('key').to_numpy(), + prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), + ) + rf = libarr['cells'].values.field('refs') refs = dict( offsets = rf.offsets.to_numpy(), @@ -292,10 +317,9 @@ def read_arrow( ) elements = dict( - boundaries = get_geom(libarr, 'boundaries'), + boundary_batches = get_boundary_batches(libarr), + boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), - boxes = get_geom(libarr, 'boxes'), - nodes = get_geom(libarr, 'nodes'), texts = texts, refs = refs, ) @@ -320,7 +344,8 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() - _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) + _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) _grefs_to_mrefs(pat, global_args, elements['refs'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) @@ -366,6 +391,7 @@ def _grefs_to_mrefs( elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] rep_valid = elem['rep_valid'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): @@ -380,10 +406,16 @@ def _grefs_to_mrefs( a_vector = elem_rep_xy0[ee] b_vector = elem_rep_xy1[ee] a_count, b_count = elem_rep_counts[ee] - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + if raw_mode: + rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + else: + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + if raw_mode: + ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + else: + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -406,6 +438,7 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -413,7 +446,10 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - mlabel = Label(string=string, offset=offset, annotations=annotations) + if raw_mode: + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) + else: + mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -439,7 +475,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -453,65 +488,78 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) -def _boundaries_to_polygons( +def _boundary_batches_to_polygons( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: elem_off = elem['offsets'] # which elements belong to each cell - xy_val = elem['xy_arr'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + poly_off = elem['poly_off'] + poly_offsets = elem['poly_offsets'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) # +1 to capture ending location for last elem + elem_vert_off = vert_off[elem_slc] + elem_poly_off = poly_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] + vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + + if vertex_offsets.size == 1: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(poly) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(polys) + + +def _boundary_props_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] layer_inds = elem['layer_inds'] layer_tups = global_args['layer_tups'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element - xy_counts = xy_offs[1:] - xy_offs[:-1] - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - prop_counts = prop_offs[1:] - prop_offs[:-1] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + elem_vert_off = vert_off[elem_slc] + prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - order = numpy.argsort(elem_layer_inds, stable=True) - unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) - - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] - for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): - ee_inds = order[ff:ff + nn] - layer = layer_tups[layer_ind] - propless_mask = prop_counts[ee_inds] == 0 - - poly_count_on_layer = propless_mask.sum() - if poly_count_on_layer == 1: - propless_mask[:] = 0 # Never make a 1-element collection - elif poly_count_on_layer > 1: - propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point - vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) - vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) - - for ii, ee in enumerate(ee_inds[propless_mask]): - vo = vertex_offsets[ii] - vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] - - polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) - pat.shapes[layer].append(polys) - - # Handle single polygons - for ee in ee_inds[~propless_mask]: - layer = layer_tups[elem_layer_inds[ee]] - vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) - pat.shapes[layer].append(poly) + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) #def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 26713e9..b005cd7 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -5,6 +5,7 @@ import pytest pytest.importorskip('pyarrow') +from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -119,7 +120,10 @@ def _make_arrow_test_library() -> Library: leaf = Pattern() leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((2, 0), vertices=[[40, 0], [50, 0], [50, 10], [40, 10]]) leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.polygon((1, 0), vertices=[[80, 0], [90, 0], [90, 10], [80, 10]]) + leaf.polygon((2, 0), vertices=[[60, 0], [70, 0], [70, 10], [60, 10]], annotations={'18': ['leaf-poly-2']}) leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) lib['leaf'] = leaf @@ -178,3 +182,80 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert len(lib) == manifest.cells assert 'TOP' in lib assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 + + +def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + + leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + + boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() + boundary_props = cells.field('boundary_props')[leaf_index].as_py() + + assert len(boundary_batches) == 2 + assert len(boundary_props) == 2 + + batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} + assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] + assert len(batch_by_layer[(1, 0)]['vertices']) == 16 + assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] + assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + + props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} + assert sorted(props_by_layer) == [(1, 0), (2, 0)] + assert props_by_layer[(1, 0)]['properties'][0]['value'] == 'leaf-poly' + assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' + + +def test_raw_ref_grid_label_constructors_match_public() -> None: + raw_grid = Grid._from_raw( + a_vector=numpy.array([20, 0]), + a_count=3, + b_vector=numpy.array([0, 30]), + b_count=2, + ) + public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) + assert raw_grid == public_grid + + raw_ref = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=raw_grid, + annotations={'12': ['child-ref']}, + ) + public_ref = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=public_grid, + annotations={'12': ['child-ref']}, + ) + assert raw_ref == public_ref + assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + + raw_label = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations={'10': ['leaf-label']}, + ) + public_label = Label( + 'LEAF', + offset=(3, 4), + annotations={'10': ['leaf-label']}, + ) + assert raw_label == public_label + assert numpy.array_equal(raw_label.get_bounds_single(), public_label.get_bounds_single()) From 7130d26112f9bb1cfe5b8ceb83e73d68d18d5f58 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:56:42 -0700 Subject: [PATCH 07/15] enable annotations=None by default --- masque/label.py | 2 +- masque/ref.py | 2 +- masque/test/test_gdsii_arrow.py | 34 ++++++++++++++++++++++++++++++++- masque/utils/comparisons.py | 10 ++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/masque/label.py b/masque/label.py index 98a65df..d220fee 100644 --- a/masque/label.py +++ b/masque/label.py @@ -66,7 +66,7 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl new._string = string new._offset = offset new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> Self: diff --git a/masque/ref.py b/masque/ref.py index 1d71432..0cc911f 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -103,7 +103,7 @@ class Ref( new._scale = scale new._mirrored = mirrored new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> 'Ref': diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index b005cd7..cec48c8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -19,7 +19,7 @@ if not gdsii_arrow.is_available(): def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: - if annotations is None: + if not annotations: return None return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) @@ -228,6 +228,25 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_ref_empty = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + public_ref_empty = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + assert raw_ref_empty.annotations is None + assert raw_ref_empty == public_ref_empty + raw_ref = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, @@ -247,6 +266,19 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: assert raw_ref == public_ref assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + raw_label_empty = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations=None, + ) + public_label_empty = Label( + 'LEAF', + offset=(3, 4), + annotations=None, + ) + assert raw_label_empty.annotations is None + assert raw_label_empty == public_label_empty + raw_label = Label._from_raw( 'LEAF', offset=numpy.array([3, 4]), diff --git a/masque/utils/comparisons.py b/masque/utils/comparisons.py index ffb7206..bb2dfee 100644 --- a/masque/utils/comparisons.py +++ b/masque/utils/comparisons.py @@ -9,7 +9,15 @@ def annotation2key(aaa: int | float | str) -> tuple[bool, Any]: return (isinstance(aaa, str), aaa) +def _normalized_annotations(annotations: annotations_t) -> annotations_t: + if not annotations: + return None + return annotations + + def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is not None elif bb is None: # noqa: RET505 @@ -36,6 +44,8 @@ def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: def annotations_eq(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is None elif bb is None: # noqa: RET505 From ec7803156508b4ff1f3e9a869fa139f83b4d82e3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:17 -0700 Subject: [PATCH 08/15] [RectCollection] add a RectCollection shape --- masque/__init__.py | 1 + masque/file/gdsii.py | 16 +- masque/shapes/__init__.py | 1 + masque/shapes/rect_collection.py | 256 ++++++++++++++++++++++++++++ masque/test/test_rect_collection.py | 70 ++++++++ 5 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 masque/shapes/rect_collection.py create mode 100644 masque/test/test_rect_collection.py diff --git a/masque/__init__.py b/masque/__init__.py index 8e13bb9..b6cc5e9 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -42,6 +42,7 @@ from .error import ( from .shapes import ( Shape as Shape, Polygon as Polygon, + RectCollection as RectCollection, Path as Path, Circle as Circle, Arc as Arc, diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 116fa07..323dabd 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -37,7 +37,7 @@ from klamath import records from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path +from ..shapes import Polygon, Path, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -466,6 +466,20 @@ def _shapes_to_elements( properties=properties, ) elements.append(path) + elif isinstance(shape, RectCollection): + for rect in shape.rects: + xy_closed = numpy.empty((5, 2), dtype=numpy.int32) + xy_closed[0] = rint_cast((rect[0], rect[1])) + xy_closed[1] = rint_cast((rect[0], rect[3])) + xy_closed[2] = rint_cast((rect[2], rect[3])) + xy_closed[3] = rint_cast((rect[2], rect[1])) + xy_closed[4] = xy_closed[0] + boundary = klamath.elements.Boundary( + layer=(layer, data_type), + xy=xy_closed, + properties=properties, + ) + elements.append(boundary) elif isinstance(shape, Polygon): polygon = shape xy_closed = numpy.empty((polygon.vertices.shape[0] + 1, 2), dtype=numpy.int32) diff --git a/masque/shapes/__init__.py b/masque/shapes/__init__.py index fd66c59..ac3a14b 100644 --- a/masque/shapes/__init__.py +++ b/masque/shapes/__init__.py @@ -11,6 +11,7 @@ from .shape import ( from .polygon import Polygon as Polygon from .poly_collection import PolyCollection as PolyCollection +from .rect_collection import RectCollection as RectCollection from .circle import Circle as Circle from .ellipse import Ellipse as Ellipse from .arc import Arc as Arc diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py new file mode 100644 index 0000000..96becfd --- /dev/null +++ b/masque/shapes/rect_collection.py @@ -0,0 +1,256 @@ +from typing import Any, cast, Self +from collections.abc import Iterator +import copy +import functools + +import numpy +from numpy import pi +from numpy.typing import NDArray, ArrayLike + +from . import Shape, normalized_shape_tuple +from .polygon import Polygon +from ..error import PatternError +from ..repetition import Repetition +from ..utils import annotations_lt, annotations_eq, rep2key, annotations_t + + +def _normalize_rects(rects: ArrayLike) -> NDArray[numpy.float64]: + arr = numpy.asarray(rects, dtype=float) + if arr.ndim != 2 or arr.shape[1] != 4: + raise PatternError('Rectangles must be an Nx4 array of [xmin, ymin, xmax, ymax]') + if numpy.any(arr[:, 0] > arr[:, 2]) or numpy.any(arr[:, 1] > arr[:, 3]): + raise PatternError('Rectangles must satisfy xmin <= xmax and ymin <= ymax') + if arr.shape[0] <= 1: + return arr + order = numpy.lexsort((arr[:, 3], arr[:, 2], arr[:, 1], arr[:, 0])) + return arr[order] + + +def _renormalize_rects_in_place(rects: NDArray[numpy.float64]) -> None: + x0 = numpy.minimum(rects[:, 0], rects[:, 2]) + x1 = numpy.maximum(rects[:, 0], rects[:, 2]) + y0 = numpy.minimum(rects[:, 1], rects[:, 3]) + y1 = numpy.maximum(rects[:, 1], rects[:, 3]) + rects[:, 0] = x0 + rects[:, 1] = y0 + rects[:, 2] = x1 + rects[:, 3] = y1 + + +@functools.total_ordering +class RectCollection(Shape): + """ + A collection of axis-aligned rectangles, stored as an Nx4 array of + `[xmin, ymin, xmax, ymax]` rows. + """ + __slots__ = ( + '_rects', + '_repetition', '_annotations', + ) + + _rects: NDArray[numpy.float64] + + @property + def rects(self) -> NDArray[numpy.float64]: + return self._rects + + @rects.setter + def rects(self, val: ArrayLike) -> None: + self._rects = _normalize_rects(val) + + @property + def offset(self) -> NDArray[numpy.float64]: + return numpy.zeros(2) + + @offset.setter + def offset(self, val: ArrayLike) -> None: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + + def set_offset(self, val: ArrayLike) -> Self: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + return self + + def translate(self, offset: ArrayLike) -> Self: + delta = numpy.asarray(offset, dtype=float).reshape(2) + self._rects[:, [0, 2]] += delta[0] + self._rects[:, [1, 3]] += delta[1] + return self + + def __init__( + self, + rects: ArrayLike, + *, + offset: ArrayLike = (0.0, 0.0), + rotation: float = 0.0, + repetition: Repetition | None = None, + annotations: annotations_t = None, + raw: bool = False, + ) -> None: + if raw: + assert isinstance(rects, numpy.ndarray) + self._rects = rects + self._repetition = repetition + self._annotations = annotations + else: + self.rects = rects + self.repetition = repetition + self.annotations = annotations + if rotation: + self.rotate(rotation) + if numpy.any(offset): + self.translate(offset) + + @classmethod + def _from_raw( + cls, + *, + rects: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._rects = rects + new._repetition = repetition + new._annotations = annotations + return new + + @property + def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: + for rect in self._rects: + xmin, ymin, xmax, ymax = rect + yield numpy.array([ + [xmin, ymin], + [xmin, ymax], + [xmax, ymax], + [xmax, ymin], + ], dtype=float) + + def __deepcopy__(self, memo: dict | None = None) -> Self: + memo = {} if memo is None else memo + new = copy.copy(self) + new._rects = self._rects.copy() + new._repetition = copy.deepcopy(self._repetition, memo) + new._annotations = copy.deepcopy(self._annotations) + return new + + def _sorted_rects(self) -> NDArray[numpy.float64]: + if self._rects.shape[0] <= 1: + return self._rects + order = numpy.lexsort((self._rects[:, 3], self._rects[:, 2], self._rects[:, 1], self._rects[:, 0])) + return self._rects[order] + + def __eq__(self, other: Any) -> bool: + return ( + type(self) is type(other) + and numpy.array_equal(self._sorted_rects(), other._sorted_rects()) + and self.repetition == other.repetition + and annotations_eq(self.annotations, other.annotations) + ) + + def __lt__(self, other: Shape) -> bool: + if type(self) is not type(other): + if repr(type(self)) != repr(type(other)): + return repr(type(self)) < repr(type(other)) + return id(type(self)) < id(type(other)) + + other = cast('RectCollection', other) + self_rects = self._sorted_rects() + other_rects = other._sorted_rects() + if not numpy.array_equal(self_rects, other_rects): + min_len = min(self_rects.shape[0], other_rects.shape[0]) + eq_mask = self_rects[:min_len] != other_rects[:min_len] + eq_lt = self_rects[:min_len] < other_rects[:min_len] + eq_lt_masked = eq_lt[eq_mask] + if eq_lt_masked.size > 0: + return bool(eq_lt_masked.flat[0]) + return self_rects.shape[0] < other_rects.shape[0] + if self.repetition != other.repetition: + return rep2key(self.repetition) < rep2key(other.repetition) + return annotations_lt(self.annotations, other.annotations) + + def to_polygons( + self, + num_vertices: int | None = None, # unused # noqa: ARG002 + max_arclen: float | None = None, # unused # noqa: ARG002 + ) -> list[Polygon]: + return [ + Polygon( + vertices=vertices, + repetition=copy.deepcopy(self.repetition), + annotations=copy.deepcopy(self.annotations), + ) + for vertices in self.polygon_vertices + ] + + def get_bounds_single(self) -> NDArray[numpy.float64] | None: + if self._rects.size == 0: + return None + mins = self._rects[:, :2].min(axis=0) + maxs = self._rects[:, 2:].max(axis=0) + return numpy.vstack((mins, maxs)) + + def rotate(self, theta: float) -> Self: + quarter_turns = int(numpy.rint(theta / (pi / 2))) + if not numpy.isclose(theta, quarter_turns * (pi / 2)): + raise PatternError('RectCollection only supports Manhattan rotations') + turns = quarter_turns % 4 + if turns == 0 or self._rects.size == 0: + return self + + corners = numpy.stack(( + self._rects[:, [0, 1]], + self._rects[:, [0, 3]], + self._rects[:, [2, 3]], + self._rects[:, [2, 1]], + ), axis=1) + flat = corners.reshape(-1, 2) + if turns == 1: + rotated = numpy.column_stack((-flat[:, 1], flat[:, 0])) + elif turns == 2: + rotated = -flat + else: + rotated = numpy.column_stack((flat[:, 1], -flat[:, 0])) + corners = rotated.reshape(corners.shape) + self._rects[:, 0] = corners[:, :, 0].min(axis=1) + self._rects[:, 1] = corners[:, :, 1].min(axis=1) + self._rects[:, 2] = corners[:, :, 0].max(axis=1) + self._rects[:, 3] = corners[:, :, 1].max(axis=1) + return self + + def mirror(self, axis: int = 0) -> Self: + if axis not in (0, 1): + raise PatternError('Axis must be 0 or 1') + if axis == 0: + self._rects[:, [1, 3]] *= -1 + else: + self._rects[:, [0, 2]] *= -1 + _renormalize_rects_in_place(self._rects) + return self + + def scale_by(self, c: float) -> Self: + self._rects *= c + _renormalize_rects_in_place(self._rects) + return self + + def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + rects = self._sorted_rects() + centers = 0.5 * (rects[:, :2] + rects[:, 2:]) + offset = centers.mean(axis=0) + zeroed = rects.copy() + zeroed[:, [0, 2]] -= offset[0] + zeroed[:, [1, 3]] -= offset[1] + normed = zeroed / norm_value + return ( + (type(self), normed.data.tobytes()), + (offset, 1.0, 0.0, False), + lambda: RectCollection(rects=normed * norm_value), + ) + + def __repr__(self) -> str: + if self._rects.size == 0: + return '' + centers = 0.5 * (self._rects[:, :2] + self._rects[:, 2:]) + centroid = centers.mean(axis=0) + return f'' diff --git a/masque/test/test_rect_collection.py b/masque/test/test_rect_collection.py new file mode 100644 index 0000000..449f4fa --- /dev/null +++ b/masque/test/test_rect_collection.py @@ -0,0 +1,70 @@ +import copy + +import numpy +import pytest +from numpy.testing import assert_allclose, assert_equal + +from ..error import PatternError +from ..shapes import Polygon, RectCollection + + +def test_rect_collection_init_and_to_polygons() -> None: + rects = RectCollection([[10, 10, 12, 12], [0, 0, 5, 5]]) + assert_equal(rects.rects, [[0, 0, 5, 5], [10, 10, 12, 12]]) + + polys = rects.to_polygons() + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_equal(polys[0].vertices, [[0, 0], [0, 5], [5, 5], [5, 0]]) + + +def test_rect_collection_rejects_invalid_rects() -> None: + with pytest.raises(PatternError): + RectCollection([[0, 0, 1]]) + with pytest.raises(PatternError): + RectCollection([[5, 0, 1, 2]]) + with pytest.raises(PatternError): + RectCollection([[0, 5, 1, 2]]) + + +def test_rect_collection_raw_constructor_matches_public() -> None: + raw = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'1': ['rects']}, + ) + public = RectCollection( + [[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'1': ['rects']}, + ) + assert raw == public + assert_equal(raw.get_bounds_single(), [[0, 0], [12, 12]]) + + +def test_rect_collection_manhattan_transforms() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + + mirrored = copy.deepcopy(rects).mirror(1) + assert_equal(mirrored.rects, [[-2, 0, 0, 4], [-12, 20, -10, 22]]) + + scaled = copy.deepcopy(rects).scale_by(-2) + assert_equal(scaled.rects, [[-4, -8, 0, 0], [-24, -44, -20, -40]]) + + rotated = copy.deepcopy(rects).rotate(numpy.pi / 2) + assert_equal(rotated.rects, [[-4, 0, 0, 2], [-22, 10, -20, 12]]) + + +def test_rect_collection_non_manhattan_rotation_raises() -> None: + rects = RectCollection([[0, 0, 2, 4]]) + with pytest.raises(PatternError, match='Manhattan rotations'): + rects.rotate(numpy.pi / 4) + + +def test_rect_collection_normalized_form_rebuild_is_independent() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + _intrinsic, extrinsic, rebuild = rects.normalized_form(2) + + clone = rebuild() + clone.rects[:] = [[1, 1, 2, 2], [3, 3, 4, 4]] + + assert_allclose(extrinsic[0], [6, 11.5]) + assert_equal(rects.rects, [[0, 0, 2, 4], [10, 20, 12, 22]]) From 27f4f0e86ec0421998266e6bd093169b6807ac4e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:39 -0700 Subject: [PATCH 09/15] [Polygon / PolyCollection] add raw constructors --- masque/shapes/poly_collection.py | 16 ++++++++++++++++ masque/shapes/polygon.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 6c23da7..0b5543f 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -119,6 +119,22 @@ class PolyCollection(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertex_lists: NDArray[numpy.float64], + vertex_offsets: NDArray[numpy.integer[Any]], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertex_lists = vertex_lists + new._vertex_offsets = vertex_offsets + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index a743de2..42c2f78 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -131,6 +131,20 @@ class Polygon(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Polygon': memo = {} if memo is None else memo new = copy.copy(self) From 3f63599abe174b21ba053a9065445a26621ef6d5 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:52 -0700 Subject: [PATCH 10/15] [gdsii_arrow] more performance work --- masque/file/gdsii_arrow.py | 356 ++++++++++++++++++++++++----- masque/shapes/poly_collection.py | 13 +- masque/test/test_file_roundtrip.py | 29 ++- masque/test/test_gdsii_arrow.py | 264 ++++++++++++++++++--- 4 files changed, 562 insertions(+), 100 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index f6a7739..c5e7681 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -45,7 +45,7 @@ from pyarrow.cffi import ffi from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, PolyCollection +from ..shapes import Polygon, Path, PolyCollection, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -276,6 +276,15 @@ def read_arrow( poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), ) + def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('rect_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_off = batches.values.field('rects').offsets.to_numpy() // 4, + ) + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: boundaries = libarr['cells'].values.field('boundary_props') return dict( @@ -288,22 +297,46 @@ def read_arrow( prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), ) - rf = libarr['cells'].values.field('refs') - refs = dict( - offsets = rf.offsets.to_numpy(), - targets = rf.values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), - invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), - scale = rf.values.field('mag').fill_null(1).to_numpy(), - rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), - rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), - rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), - prop_off = rf.values.field('properties').offsets.to_numpy(), - prop_key = rf.values.field('properties').values.field('key').to_numpy(), - prop_val = rf.values.field('properties').values.field('value').to_pylist(), - ) + def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + prop_off = values.field('properties').offsets.to_numpy(), + prop_key = values.field('properties').values.field('key').to_numpy(), + prop_val = values.field('properties').values.field('value').to_pylist(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem txt = libarr['cells'].values.field('texts') texts = dict( @@ -317,11 +350,15 @@ def read_arrow( ) elements = dict( + srefs = get_refs(libarr, 'srefs', has_repetition=False), + arefs = get_refs(libarr, 'arefs', has_repetition=True), + sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False), + aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True), + rect_batches = get_rect_batches(libarr), boundary_batches = get_boundary_batches(libarr), boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), texts = texts, - refs = refs, ) paths = libarr['cells'].values.field('paths') @@ -344,10 +381,14 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() + _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) - _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _srefs_to_mrefs(pat, global_args, elements['srefs'], cc) + _arefs_to_mrefs(pat, global_args, elements['arefs'], cc) + _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc) + _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) mlib[name] = pat @@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: return library_info -def _grefs_to_mrefs( +def _srefs_to_mrefs( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: cell_names = global_args['cell_names'] - elem_off = elem['offsets'] # which elements belong to each cell - xy = elem['xy'] - prop_key = elem['prop_key'] - prop_val = elem['prop_val'] - targets = elem['targets'] - + elem_off = elem['offsets'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - elem_targets = targets[elem_slc][:elem_count] - elem_xy = xy[elem_slc][:elem_count] - elem_invert_y = elem['invert_y'][elem_slc][:elem_count] - elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] - elem_scale = elem['scale'][elem_slc][:elem_count] - elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] - elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] - elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] - rep_valid = elem['rep_valid'][elem_slc][:elem_count] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] raw_mode = global_args['raw_mode'] + _append_plain_refs_sorted( + pat=pat, + cell_names=cell_names, + elem_targets=elem_targets, + elem_xy=elem_xy, + elem_invert_y=elem_invert_y, + elem_angle_rad=elem_angle_rad, + elem_scale=elem_scale, + raw_mode=raw_mode, + ) + + +def _append_plain_refs_sorted( + *, + pat: Pattern, + cell_names: list[str], + elem_targets: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.integer[Any]], + elem_invert_y: NDArray[numpy.bool_ | numpy.bool], + elem_angle_rad: NDArray[numpy.floating[Any]], + elem_scale: NDArray[numpy.floating[Any]], + raw_mode: bool, + ) -> None: + elem_count = len(elem_targets) + if elem_count == 0: + return + + make_ref = Ref._from_raw if raw_mode else Ref + + target_start = 0 + while target_start < elem_count: + target_id = elem_targets[target_start] + target_stop = target_start + 1 + while target_stop < elem_count and elem_targets[target_stop] == target_id: + target_stop += 1 + + append_refs = pat.refs[cell_names[target_id]].extend + append_refs( + make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=None, + ) + for ee in range(target_start, target_stop) + ) + + target_start = target_stop + + +def _arefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + elem_xy0 = elem['xy0'][start:stop] + elem_xy1 = elem['xy1'][start:stop] + elem_counts = elem['counts'][start:stop] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + if len(elem_targets) == 0: + return + + target = None + append_ref: Callable[[Ref], Any] | None = None + for ee in range(len(elem_targets)): + if target != elem_targets[ee]: + target = elem_targets[ee] + append_ref = pat.refs[cell_names[target]].append + assert append_ref is not None + a_count, b_count = elem_counts[ee] + append_ref(make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=None, + )) + + +def _sref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): - target = cell_names[elem_targets[ee]] - offset = elem_xy[ee] - mirr = elem_invert_y[ee] - rot = elem_angle_rad[ee] - mag = elem_scale[ee] - - rep: None | Grid = None - if rep_valid[ee]: - a_vector = elem_rep_xy0[ee] - b_vector = elem_rep_xy1[ee] - a_count, b_count = elem_rep_counts[ee] - if raw_mode: - rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - else: - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - else: - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - pat.refs[target].append(ref) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) + + +def _aref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] + elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] + elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + for ee in range(elem_count): + a_count, b_count = elem_counts[ee] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) def _texts_to_labels( @@ -520,16 +712,53 @@ def _boundary_batches_to_polygons( for bb in range(batch_count): layer = layer_tups[elem_layer_inds[bb]] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] - vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=None) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(poly) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(polys) +def _rect_batches_to_rectcollections( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + rect_arr = elem['rect_arr'] + rect_off = elem['rect_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) + elem_rect_off = rect_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] + if raw_mode: + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) + else: + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + pat.shapes[layer].append(rect_collection) + + def _boundary_props_to_polygons( pat: Pattern, global_args: dict[str, Any], @@ -558,7 +787,10 @@ def _boundary_props_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) pat.shapes[layer].append(poly) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 0b5543f..f90c29a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -34,7 +34,7 @@ class PolyCollection(Shape): _vertex_lists: NDArray[numpy.float64] """ 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """ - _vertex_offsets: NDArray[numpy.intp] + _vertex_offsets: NDArray[numpy.integer[Any]] """ 1D NDArray specifying the starting offset for each polygon """ @property @@ -45,7 +45,7 @@ class PolyCollection(Shape): return self._vertex_lists @property - def vertex_offsets(self) -> NDArray[numpy.intp]: + def vertex_offsets(self) -> NDArray[numpy.integer[Any]]: """ Starting offset (in `vertex_lists`) for each polygon """ @@ -63,7 +63,7 @@ class PolyCollection(Shape): chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]), strict=True, ): - yield slice(ii, ff) + yield slice(int(ii), int(ff)) @property def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: @@ -105,6 +105,7 @@ class PolyCollection(Shape): if raw: assert isinstance(vertex_lists, numpy.ndarray) assert isinstance(vertex_offsets, numpy.ndarray) + assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) self._vertex_lists = vertex_lists self._vertex_offsets = vertex_offsets self._repetition = repetition @@ -148,7 +149,7 @@ class PolyCollection(Shape): return ( type(self) is type(other) and numpy.array_equal(self._vertex_lists, other._vertex_lists) - and numpy.array_equal(self._vertex_offsets, other._vertex_offsets) + and numpy.array_equal(self.vertex_offsets, other.vertex_offsets) and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -231,11 +232,11 @@ class PolyCollection(Shape): # TODO: normalize mirroring? - return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()), + return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()), (offset, scale / norm_value, rotation, False), lambda: PolyCollection( vertex_lists=rotated_vertices * norm_value, - vertex_offsets=self._vertex_offsets.copy(), + vertex_offsets=self.vertex_offsets.copy(), ), ) diff --git a/masque/test/test_file_roundtrip.py b/masque/test/test_file_roundtrip.py index 2cfb0d1..283a863 100644 --- a/masque/test/test_file_roundtrip.py +++ b/masque/test/test_file_roundtrip.py @@ -5,7 +5,7 @@ from numpy.testing import assert_allclose from ..pattern import Pattern from ..library import Library -from ..shapes import Path as MPath, Circle, Polygon +from ..shapes import Path as MPath, Circle, Polygon, RectCollection from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: @@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: assert poly.repetition is not None assert isinstance(poly.repetition, Grid) assert poly.repetition.a_count == 5 + + +def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None: + from ..file import gdsii + + lib = Library() + pat = Pattern() + pat.shapes[(5, 0)].append( + RectCollection( + rects=[[0, 0, 10, 5], [20, -5, 30, 10]], + annotations={'1': ['rects']}, + ) + ) + lib['rects'] = pat + + gds_file = tmp_path / 'rect_collection.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + read_lib, _ = gdsii.readfile(gds_file) + polys = read_lib['rects'].shapes[(5, 0)] + + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]]) + assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]]) + assert polys[0].annotations == {'1': ['rects']} + assert polys[1].annotations == {'1': ['rects']} diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index cec48c8..2ef4d83 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -9,7 +9,7 @@ from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid -from ..shapes import Path as MPath +from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection from ..file import gdsii, gdsii_arrow from ..file.gdsii_perf import write_fixture @@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: return tuple(tuple(row.tolist()) for row in arr) +def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int) + rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])] + rev = arr[::-1] + rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0])) + return min(rows) + + def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: if isinstance(shape, MPath): cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) @@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...] keys.append(( 'polygon', layer, - _coord_key(poly.vertices), + _canonical_polygon_key(poly.vertices), _coord_key(poly.offset), _annotations_key(poly.annotations), )) return keys -def _ref_key(target: str, ref: object) -> tuple[object, ...]: - repetition = None - if ref.repetition is not None: - repetition = ( - _coord_key(ref.repetition.a_vector), - int(ref.repetition.a_count), - _coord_key(ref.repetition.b_vector), - int(ref.repetition.b_count), - ) - return ( - target, - _coord_key(ref.offset), - round(float(ref.rotation), 8), - round(float(ref.scale), 8), - bool(ref.mirrored), - repetition, - _annotations_key(ref.annotations), - ) +def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]: + keys = [] + for transform in ref.as_transforms(): + keys.append(( + target, + _coord_key(transform[:2]), + round(float(transform[2]), 8), + round(float(transform[4]), 8), + bool(int(round(float(transform[3])))), + _annotations_key(ref.annotations), + )) + return keys def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: @@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]: for shape in shapes: shape_keys.extend(_shape_key(shape, layer)) - ref_keys = [ - _ref_key(target, ref) - for target, refs in pattern.refs.items() - for ref in refs - ] + ref_keys: list[tuple[object, ...]] = [] + for target, refs in pattern.refs.items(): + for ref in refs: + ref_keys.extend(_ref_keys(target, ref)) label_keys = [ _label_key(layer, label) @@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library: ) lib['sibling'] = sibling + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(20, 0)) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2)) + fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2)) + fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']}) + fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2), + annotations={'20': ['fanout-aref']}) + lib['fanout'] = fanout + top = Pattern() top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.ref('fanout', offset=(250, -75)) top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) lib['top'] = top @@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 +def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, _ = gdsii.readfile(gds_file) + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + assert _library_summary(arrow_lib) == _library_summary(canonical_lib) + + decoded_ref = arrow_lib['top'].refs['leaf'][0] + assert decoded_ref.repetition is None + + +def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'plain_srefs.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + fanout = arrow_lib['fanout'] + + plain_leaf_refs = [ + ref + for ref in fanout.refs['leaf'] + if ref.annotations is None and ref.repetition is None + ] + assert len(plain_leaf_refs) == 2 + assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs) + + +def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref_schema.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + srefs = cells.field('srefs')[top_index].as_py() + arefs = cells.field('arefs')[top_index].as_py() + + assert len(srefs) == 1 + assert len(arefs) == 0 + assert cell_names[srefs[0]['target']] == 'leaf' + + def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_batches.gds' @@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + rect_batches = cells.field('rect_batches')[leaf_index].as_py() boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() boundary_props = cells.field('boundary_props')[leaf_index].as_py() - assert len(boundary_batches) == 2 + assert len(rect_batches) == 2 + assert len(boundary_batches) == 0 assert len(boundary_props) == 2 - batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} - assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] - assert len(batch_by_layer[(1, 0)]['vertices']) == 16 - assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] - assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches} + assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10] + assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10] props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} assert sorted(props_by_layer) == [(1, 0), (2, 0)] @@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' +def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None: + lib = Library() + top = Pattern() + top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]])) + top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]]) + top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]]) + lib['top'] = top + + gds_file = tmp_path / 'arrow_rect_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + rect_batches = cells.field('rect_batches')[top_index].as_py() + boundary_batches = cells.field('boundary_batches')[top_index].as_py() + + assert len(rect_batches) == 1 + assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0) + assert rect_batches[0]['rects'] == [ + 0, 0, 10, 10, + 20, 0, 30, 10, + 40, 0, 50, 10, + 60, 0, 70, 10, + ] + + assert len(boundary_batches) == 1 + assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0) + assert boundary_batches[0]['vertex_offsets'] == [0, 3] + + +def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_ref_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout') + + srefs = cells.field('srefs')[fanout_index].as_py() + arefs = cells.field('arefs')[fanout_index].as_py() + sref_props = cells.field('sref_props')[fanout_index].as_py() + aref_props = cells.field('aref_props')[fanout_index].as_py() + + sref_target_ids = [entry['target'] for entry in srefs] + sref_targets = [cell_names[target] for target in sref_target_ids] + assert sorted(sref_targets) == ['child', 'leaf', 'leaf'] + assert sref_target_ids == sorted(sref_target_ids) + sref_by_target = {} + for entry in srefs: + sref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in sref_by_target['child']] == [True] + assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1]) + assert len(sref_by_target['leaf']) == 2 + + aref_target_ids = [entry['target'] for entry in arefs] + aref_targets = [cell_names[target] for target in aref_target_ids] + assert sorted(aref_targets) == ['child', 'leaf', 'leaf'] + assert aref_target_ids == sorted(aref_target_ids) + aref_by_target = {} + for entry in arefs: + aref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in aref_by_target['child']] == [True] + assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2]) + assert len(aref_by_target['leaf']) == 2 + + assert len(sref_props) == 1 + assert cell_names[sref_props[0]['target']] == 'leaf' + assert sref_props[0]['properties'][0]['value'] == 'fanout-sref' + + assert len(aref_props) == 1 + assert cell_names[aref_props[0]['target']] == 'child' + assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), @@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_poly = Polygon._from_raw( + vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]), + annotations={'1': ['poly']}, + ) + public_poly = Polygon( + vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], + annotations={'1': ['poly']}, + ) + assert raw_poly == public_poly + + raw_poly_collection = PolyCollection._from_raw( + vertex_lists=numpy.array([ + [0.0, 0.0], [2.0, 0.0], [2.0, 2.0], + [10.0, 10.0], [12.0, 10.0], [12.0, 12.0], + ]), + vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32), + annotations={'2': ['pc']}, + ) + public_poly_collection = PolyCollection( + vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]], + vertex_offsets=[0, 3], + annotations={'2': ['pc']}, + ) + assert raw_poly_collection == public_poly_collection + assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)] + + raw_rect_collection = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'3': ['rects']}, + ) + public_rect_collection = RectCollection( + rects=[[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'3': ['rects']}, + ) + assert raw_rect_collection == public_rect_collection + raw_ref_empty = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, From 37d462525cea9272de2e8e3bfe602c49611e9358 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:55:30 -0700 Subject: [PATCH 11/15] [shapes] move to per-shape purpose-built _from_raw constructors --- masque/file/gdsii.py | 44 ++++++++----- masque/file/gdsii_arrow.py | 27 ++++++-- masque/shapes/arc.py | 49 ++++++++------ masque/shapes/circle.py | 32 +++++---- masque/shapes/ellipse.py | 38 ++++++----- masque/shapes/path.py | 47 ++++++++------ masque/shapes/poly_collection.py | 18 ++---- masque/shapes/polygon.py | 13 +--- masque/shapes/rect_collection.py | 13 +--- masque/shapes/text.py | 49 ++++++++------ masque/test/test_raw_constructors.py | 97 ++++++++++++++++++++++++++++ 11 files changed, 288 insertions(+), 139 deletions(-) create mode 100644 masque/test/test_raw_constructors.py diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 323dabd..1d8c3d1 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -323,26 +323,40 @@ def _gpath_to_mpath(gpath: klamath.library.Path, raw_mode: bool) -> tuple[layer_ else: raise PatternError(f'Unrecognized path type: {gpath.path_type}') - mpath = Path( - vertices=gpath.xy.astype(float), - width=gpath.width, - cap=cap, - offset=numpy.zeros(2), - annotations=_properties_to_annotations(gpath.properties), - raw=raw_mode, - ) + vertices = gpath.xy.astype(float) + annotations = _properties_to_annotations(gpath.properties) + cap_extensions = None if cap == Path.Cap.SquareCustom: - mpath.cap_extensions = gpath.extension + cap_extensions = numpy.asarray(gpath.extension, dtype=float) + + if raw_mode: + mpath = Path._from_raw( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + mpath = Path( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + offset=numpy.zeros(2), + annotations=annotations, + ) return gpath.layer, mpath def _boundary_to_polygon(boundary: klamath.library.Boundary, raw_mode: bool) -> tuple[layer_t, Polygon]: - return boundary.layer, Polygon( - vertices=boundary.xy[:-1].astype(float), - offset=numpy.zeros(2), - annotations=_properties_to_annotations(boundary.properties), - raw=raw_mode, - ) + vertices = boundary.xy[:-1].astype(float) + annotations = _properties_to_annotations(boundary.properties) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=numpy.zeros(2), annotations=annotations) + return boundary.layer, poly def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.Reference]: diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index c5e7681..58493c1 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -680,8 +680,23 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, - width=width, cap=cap,cap_extensions=cap_extensions) + if raw_mode: + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + path = Path( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + offset=ZERO_OFFSET, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -718,13 +733,13 @@ def _boundary_batches_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=None) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(poly) else: if raw_mode: polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(polys) @@ -755,7 +770,7 @@ def _rect_batches_to_rectcollections( if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(rect_collection) @@ -790,7 +805,7 @@ def _boundary_props_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=annotations) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index 00c5714..9d5f65d 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -159,27 +159,36 @@ class Arc(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(angles, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._angles = angles - self._width = width - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.angles = angles - self.width = width - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.angles = angles + self.width = width + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + angles: NDArray[numpy.float64], + width: float, + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Arc': + new = cls.__new__(cls) + new._radii = radii + new._angles = angles + new._width = width + new._offset = offset + new._rotation = rotation % (2 * pi) + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Arc': memo = {} if memo is None else memo diff --git a/masque/shapes/circle.py b/masque/shapes/circle.py index 19f3f2b..d7591db 100644 --- a/masque/shapes/circle.py +++ b/masque/shapes/circle.py @@ -50,19 +50,27 @@ class Circle(PositionableImpl, Shape): offset: ArrayLike = (0.0, 0.0), repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._radius = radius - self._offset = offset - self._repetition = repetition - self._annotations = annotations - else: - self.radius = radius - self.offset = offset - self.repetition = repetition - self.annotations = annotations + self.radius = radius + self.offset = offset + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radius: float, + offset: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Circle': + new = cls.__new__(cls) + new._radius = radius + new._offset = offset + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Circle': memo = {} if memo is None else memo diff --git a/masque/shapes/ellipse.py b/masque/shapes/ellipse.py index 38799da..52a3297 100644 --- a/masque/shapes/ellipse.py +++ b/masque/shapes/ellipse.py @@ -95,22 +95,30 @@ class Ellipse(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._radii = radii + new._offset = offset + new._rotation = rotation % pi + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo diff --git a/masque/shapes/path.py b/masque/shapes/path.py index 7038309..a1e04af 100644 --- a/masque/shapes/path.py +++ b/masque/shapes/path.py @@ -201,34 +201,43 @@ class Path(Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: self._cap_extensions = None # Since .cap setter might access it - if raw: - assert isinstance(vertices, numpy.ndarray) - assert isinstance(cap_extensions, numpy.ndarray) or cap_extensions is None - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - self._width = width - self._cap = cap - self._cap_extensions = cap_extensions + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations + self._cap = cap + if cap == PathCap.SquareCustom and cap_extensions is None: + self._cap_extensions = numpy.zeros(2) else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations - self._cap = cap - if cap == PathCap.SquareCustom and cap_extensions is None: - self._cap_extensions = numpy.zeros(2) - else: - self.cap_extensions = cap_extensions - self.width = width + self.cap_extensions = cap_extensions + self.width = width if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + width: float, + cap: PathCap, + cap_extensions: NDArray[numpy.float64] | None = None, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._width = width + new._cap = cap + new._cap_extensions = cap_extensions + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Path': memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index f90c29a..f1c840a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -100,21 +100,11 @@ class PolyCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertex_lists, numpy.ndarray) - assert isinstance(vertex_offsets, numpy.ndarray) - assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) - self._vertex_lists = vertex_lists - self._vertex_offsets = vertex_offsets - self._repetition = repetition - self._annotations = annotations - else: - self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) - self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) - self.repetition = repetition - self.annotations = annotations + self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) + self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index 42c2f78..06e5c2b 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -115,17 +115,10 @@ class Polygon(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertices, numpy.ndarray) - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py index 96becfd..eaf028f 100644 --- a/masque/shapes/rect_collection.py +++ b/masque/shapes/rect_collection.py @@ -86,17 +86,10 @@ class RectCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(rects, numpy.ndarray) - self._rects = rects - self._repetition = repetition - self._annotations = annotations - else: - self.rects = rects - self.repetition = repetition - self.annotations = annotations + self.rects = rects + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/text.py b/masque/shapes/text.py index 5047dc4..c078879 100644 --- a/masque/shapes/text.py +++ b/masque/shapes/text.py @@ -73,27 +73,40 @@ class Text(PositionableImpl, RotatableImpl, Shape): mirrored: bool = False, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._offset = offset - self._string = string - self._height = height - self._rotation = rotation - self._mirrored = mirrored - self._repetition = repetition - self._annotations = annotations - else: - self.offset = offset - self.string = string - self.height = height - self.rotation = rotation - self.mirrored = mirrored - self.repetition = repetition - self.annotations = annotations + self.offset = offset + self.string = string + self.height = height + self.rotation = rotation + self.mirrored = mirrored + self.repetition = repetition + self.annotations = annotations self.font_path = font_path + @classmethod + def _from_raw( + cls, + *, + string: str, + height: float, + font_path: str, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._string = string + new._height = height + new._rotation = rotation % (2 * pi) + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations + new.font_path = font_path + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/test/test_raw_constructors.py b/masque/test/test_raw_constructors.py new file mode 100644 index 0000000..2f86ba0 --- /dev/null +++ b/masque/test/test_raw_constructors.py @@ -0,0 +1,97 @@ +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from ..shapes import Arc, Circle, Ellipse, Path, Text + + +def test_circle_raw_constructor_matches_public() -> None: + raw = Circle._from_raw( + radius=5.0, + offset=numpy.array([1.0, 2.0]), + annotations={'1': ['circle']}, + ) + public = Circle( + radius=5.0, + offset=(1.0, 2.0), + annotations={'1': ['circle']}, + ) + assert raw == public + + +def test_ellipse_raw_constructor_matches_public() -> None: + raw = Ellipse._from_raw( + radii=numpy.array([3.0, 5.0]), + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + public = Ellipse( + radii=(3.0, 5.0), + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + assert raw == public + + +def test_arc_raw_constructor_matches_public() -> None: + raw = Arc._from_raw( + radii=numpy.array([10.0, 6.0]), + angles=numpy.array([0.0, pi / 2]), + width=2.0, + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + public = Arc( + radii=(10.0, 6.0), + angles=(0.0, pi / 2), + width=2.0, + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + assert raw == public + + +def test_path_raw_constructor_matches_public() -> None: + raw = Path._from_raw( + vertices=numpy.array([[0.0, 0.0], [10.0, 0.0], [10.0, 5.0]]), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=numpy.array([1.0, 3.0]), + annotations={'4': ['path']}, + ) + public = Path( + vertices=((0.0, 0.0), (10.0, 0.0), (10.0, 5.0)), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=(1.0, 3.0), + annotations={'4': ['path']}, + ) + assert raw == public + assert raw.cap_extensions is not None + assert_allclose(raw.cap_extensions, [1.0, 3.0]) + + +def test_text_raw_constructor_matches_public() -> None: + raw = Text._from_raw( + string='RAW', + height=12.0, + font_path='font.otf', + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + public = Text( + string='RAW', + height=12.0, + font_path='font.otf', + offset=(1.0, 2.0), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + assert raw == public From 5c1050b0fff906ec8e32c820239dadf8b74d1905 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:31 -0700 Subject: [PATCH 12/15] [arrow] add lazy arrow reader --- examples/profile_gdsii_readers.py | 12 + masque/file/gdsii_arrow.py | 95 ++- masque/file/gdsii_lazy_arrow.py | 960 +++++++++++++++++++++++++++ masque/test/test_gdsii_arrow.py | 12 + masque/test/test_gdsii_lazy_arrow.py | 174 +++++ 5 files changed, 1237 insertions(+), 16 deletions(-) create mode 100644 masque/file/gdsii_lazy_arrow.py create mode 100644 masque/test/test_gdsii_lazy_arrow.py diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index 129a391..fb7c99e 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -56,6 +56,18 @@ def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: return _summarize_library(path, elapsed_s, info, lib) if stage == 'arrow_import': + if hasattr(module, 'readfile_arrow'): + libarr, _info = module.readfile_arrow(path) + elapsed_s = time.perf_counter() - start + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': 1, + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + arrow_arr = module._read_to_arrow(path) elapsed_s = time.perf_counter() - start return _summarize_arrow_import(path, elapsed_s, arrow_arr) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 58493c1..b97005b 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -53,7 +53,20 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +ffi.cdef( + """ + void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); + void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + void read_cells_bytes( + uint8_t* data, + size_t size, + uint64_t* ranges, + size_t range_count, + struct ArrowArray* array, + struct ArrowSchema* schema + ); + """ +) clib: Any | None = None @@ -189,12 +202,36 @@ def _read_to_arrow( ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + +def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + return pyarrow.Array._import_from_c(iptr_array, iptr_schema) - return arrow_arr + +def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + + +def _read_selected_cells_to_arrow( + buffer: bytes | mmap.mmap | memoryview, + ranges: NDArray[numpy.uint64], + ) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) + cranges = ffi.from_buffer('uint64_t[]', flat_ranges) + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) def readfile( @@ -211,6 +248,9 @@ def readfile( filename: Filename to save to. *args: passed to `read()` **kwargs: passed to `read()` + + For callers that can consume Arrow directly, prefer `readfile_arrow()` + to skip Python `Pattern` construction entirely. """ arrow_arr = _read_to_arrow(filename) assert len(arrow_arr) == 1 @@ -220,6 +260,28 @@ def readfile( return results +def readfile_arrow( + filename: str | pathlib.Path, + ) -> tuple[pyarrow.StructScalar, dict[str, Any]]: + """ + Read a GDSII file into the native Arrow representation without converting + it into `masque.Library` / `Pattern` objects. + + This is the lowest-overhead public read path exposed by this module. + + Args: + filename: Filename to read. + + Returns: + - Arrow struct scalar for the library payload + - dict of GDSII library info + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + libarr = arrow_arr[0] + return libarr, _read_header(libarr) + + def read_arrow( libarr: pyarrow.Array, raw_mode: bool = True, @@ -247,7 +309,7 @@ def read_arrow( library_info = _read_header(libarr) layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) - layer_tups = [tuple(pair) for pair in layer_names_np] + layer_tups = [(int(pair[0]), int(pair[1])) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() @@ -379,7 +441,7 @@ def read_arrow( mlib = Library() for cc in range(len(libarr['cells'])): - name = cell_names[cell_ids[cc]] + name = cell_names[int(cell_ids[cc])] pat = Pattern() _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) @@ -459,7 +521,7 @@ def _append_plain_refs_sorted( target_start = 0 while target_start < elem_count: - target_id = elem_targets[target_start] + target_id = int(elem_targets[target_start]) target_stop = target_start + 1 while target_stop < elem_count and elem_targets[target_stop] == target_id: target_stop += 1 @@ -513,9 +575,10 @@ def _arefs_to_mrefs( target = None append_ref: Callable[[Ref], Any] | None = None for ee in range(len(elem_targets)): - if target != elem_targets[ee]: - target = elem_targets[ee] - append_ref = pat.refs[cell_names[target]].append + target_id = int(elem_targets[ee]) + if target != target_id: + target = target_id + append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] append_ref(make_ref( @@ -564,7 +627,7 @@ def _sref_props_to_mrefs( repetition=None, annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _aref_props_to_mrefs( @@ -608,7 +671,7 @@ def _aref_props_to_mrefs( repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _texts_to_labels( @@ -633,7 +696,7 @@ def _texts_to_labels( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] offset = elem_xy[ee] string = elem_strings[ee] @@ -669,7 +732,7 @@ def _gpaths_to_mpaths( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] cap_int = elem_path_types[ee] @@ -725,7 +788,7 @@ def _boundary_batches_to_polygons( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] @@ -765,7 +828,7 @@ def _rect_batches_to_rectcollections( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) @@ -799,7 +862,7 @@ def _boundary_props_to_polygons( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) if raw_mode: diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py new file mode 100644 index 0000000..9a03960 --- /dev/null +++ b/masque/file/gdsii_lazy_arrow.py @@ -0,0 +1,960 @@ +""" +Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. + +This module is intentionally separate from `gdsii_arrow` so the eager read path +keeps its current behavior and performance profile. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping, Sequence +import copy +import gzip +import logging +import mmap +import pathlib + +import numpy +from numpy.typing import NDArray +import pyarrow +import klamath + +from . import gdsii, gdsii_arrow +from .utils import is_gzipped, tmpfile +from ..error import LibraryError +from ..library import ILibrary, ILibraryView, Library, LibraryView, dangling_mode_t +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _StructRange: + start: int + end: int + + +@dataclass +class _SourceBuffer: + path: pathlib.Path + data: bytes | mmap.mmap + handle: IO[bytes] | None = None + + def raw_slice(self, start: int, end: int) -> bytes: + return self.data[start:end] + + +@dataclass +class _ScanRefs: + offsets: NDArray[numpy.integer[Any]] + targets: NDArray[numpy.integer[Any]] + xy: NDArray[numpy.int32] + xy0: NDArray[numpy.int32] + xy1: NDArray[numpy.int32] + counts: NDArray[numpy.int64] + invert_y: NDArray[numpy.bool_ | numpy.bool] + angle_rad: NDArray[numpy.floating[Any]] + scale: NDArray[numpy.floating[Any]] + + +@dataclass(frozen=True) +class _CellScan: + cell_id: int + struct_range: _StructRange + ref_start: int + ref_stop: int + children: set[str] + + +@dataclass +class _ScanPayload: + libarr: pyarrow.StructScalar + library_info: dict[str, Any] + cell_names: list[str] + cell_order: list[str] + cells: dict[str, _CellScan] + refs: _ScanRefs + + +@dataclass +class _SourceLayer: + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + layer_index: int + source_name: str + + +def is_available() -> bool: + return gdsii_arrow.is_available() + + +def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: + return gdsii_arrow._read_header(libarr) + + +def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: + if is_gzipped(path): + with gzip.open(path, mode='rb') as stream: + data = stream.read() + return _SourceBuffer(path=path, data=data) + + handle = path.open(mode='rb', buffering=0) + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + return _SourceBuffer(path=path, data=mapped, handle=handle) + + +def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: + library_info = _read_header(libarr) + cell_names = libarr['cell_names'].as_py() + + cells = libarr['cells'] + cell_values = cells.values + cell_ids = cell_values.field('id').to_numpy() + struct_starts = cell_values.field('struct_start_offset').to_numpy() + struct_ends = cell_values.field('struct_end_offset').to_numpy() + + refs = cell_values.field('refs') + ref_values = refs.values + ref_offsets = refs.offsets.to_numpy() + targets = ref_values.field('target').to_numpy() + xy = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) + xy0 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) + xy1 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) + counts = gdsii_arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) + invert_y = ref_values.field('invert_y').to_numpy(zero_copy_only=False) + angle_rad = ref_values.field('angle_rad').to_numpy() + scale = ref_values.field('scale').to_numpy() + + ref_payload = _ScanRefs( + offsets=ref_offsets, + targets=targets, + xy=xy, + xy0=xy0, + xy1=xy1, + counts=counts, + invert_y=invert_y, + angle_rad=angle_rad, + scale=scale, + ) + + cell_order = [cell_names[int(cell_id)] for cell_id in cell_ids] + cell_scan: dict[str, _CellScan] = {} + for cc, name in enumerate(cell_order): + ref_start = int(ref_offsets[cc]) + ref_stop = int(ref_offsets[cc + 1]) + children = { + cell_names[int(target)] + for target in targets[ref_start:ref_stop] + } + cell_scan[name] = _CellScan( + cell_id=int(cell_ids[cc]), + struct_range=_StructRange(int(struct_starts[cc]), int(struct_ends[cc])), + ref_start=ref_start, + ref_stop=ref_stop, + children=children, + ) + + return _ScanPayload( + libarr=libarr, + library_info=library_info, + cell_names=cell_names, + cell_order=cell_order, + cells=cell_scan, + refs=ref_payload, + ) + + +def _pattern_children(pat: Pattern) -> set[str]: + return {child for child, refs in pat.refs.items() if child is not None and refs} + + +def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: + if not pat.refs: + return pat + pat.refs = map_targets(pat.refs, remap) + return pat + + +def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: + if isinstance(source, ILibraryView): + return source + return LibraryView(source) + + +def _source_order(source: ILibraryView) -> list[str]: + if isinstance(source, ArrowLibrary): + return list(source.source_order()) + return list(source.keys()) + + +def _make_ref_rows( + xy: NDArray[numpy.integer[Any]], + angle_rad: NDArray[numpy.floating[Any]], + invert_y: NDArray[numpy.bool_ | numpy.bool], + scale: NDArray[numpy.floating[Any]], + ) -> NDArray[numpy.float64]: + rows = numpy.empty((len(xy), 5), dtype=float) + rows[:, :2] = xy + rows[:, 2] = angle_rad + rows[:, 3] = invert_y.astype(float) + rows[:, 4] = scale + return rows + + +def _expand_aref_row( + xy: NDArray[numpy.integer[Any]], + xy0: NDArray[numpy.integer[Any]], + xy1: NDArray[numpy.integer[Any]], + counts: NDArray[numpy.integer[Any]], + angle_rad: float, + invert_y: bool, + scale: float, + ) -> NDArray[numpy.float64]: + a_count = int(counts[0]) + b_count = int(counts[1]) + aa, bb = numpy.meshgrid(numpy.arange(a_count), numpy.arange(b_count), indexing='ij') + displacements = aa.reshape(-1, 1) * xy0[None, :] + bb.reshape(-1, 1) * xy1[None, :] + rows = numpy.empty((displacements.shape[0], 5), dtype=float) + rows[:, :2] = xy + displacements + rows[:, 2] = angle_rad + rows[:, 3] = float(invert_y) + rows[:, 4] = scale + return rows + + +class ArrowLibrary(ILibraryView): + """ + Read-only library backed by the native lazy Arrow scan schema. + + Materializing a cell via `__getitem__` caches a real `Pattern` for that cell. + Cached cells are treated as edited for future writes from this module. + """ + + path: pathlib.Path + library_info: dict[str, Any] + + def __init__( + self, + *, + path: pathlib.Path, + payload: _ScanPayload, + source: _SourceBuffer, + ) -> None: + self.path = path + self.library_info = payload.library_info + self._payload = payload + self._source = source + self._cache: dict[str, Pattern] = {} + + @classmethod + def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: + path = pathlib.Path(filename).expanduser().resolve() + source = _open_source_buffer(path) + scan_arr = gdsii_arrow._scan_buffer_to_arrow(source.data) + assert len(scan_arr) == 1 + payload = _extract_scan_payload(scan_arr[0]) + return cls(path=path, payload=payload, source=source) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._payload.cell_order) + + def __len__(self) -> int: + return len(self._payload.cell_order) + + def __contains__(self, key: object) -> bool: + return key in self._payload.cells + + def source_order(self) -> tuple[str, ...]: + return tuple(self._payload.cell_order) + + def raw_struct_bytes(self, name: str) -> bytes: + struct_range = self._payload.cells[name].struct_range + return self._source.raw_slice(struct_range.start, struct_range.end) + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = self._materialize_patterns(names, persist=persist) + return LibraryView(mats) + + def _materialize_patterns( + self, + names: Sequence[str], + *, + persist: bool, + ) -> dict[str, Pattern]: + ordered_names = list(dict.fromkeys(names)) + missing = [name for name in ordered_names if name not in self._payload.cells] + if missing: + raise KeyError(missing[0]) + + materialized: dict[str, Pattern] = {} + uncached = [name for name in ordered_names if name not in self._cache] + if uncached: + ranges = numpy.asarray( + [ + [ + self._payload.cells[name].struct_range.start, + self._payload.cells[name].struct_range.end, + ] + for name in uncached + ], + dtype=numpy.uint64, + ) + arrow_arr = gdsii_arrow._read_selected_cells_to_arrow(self._source.data, ranges) + assert len(arrow_arr) == 1 + selected_lib, _info = gdsii_arrow.read_arrow(arrow_arr[0]) + for name in uncached: + pat = selected_lib[name] + materialized[name] = pat + if persist: + self._cache[name] = pat + + for name in ordered_names: + if name in self._cache: + materialized[name] = self._cache[name] + return materialized + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + return self._materialize_patterns((name,), persist=persist)[name] + + def _raw_children(self, name: str) -> set[str]: + return set(self._payload.cells[name].children) + + def _collect_raw_transforms(self, cell: _CellScan, target_id: int) -> list[NDArray[numpy.float64]]: + refs = self._payload.refs + start = cell.ref_start + stop = cell.ref_stop + if stop <= start: + return [] + + targets = refs.targets[start:stop] + mask = targets == target_id + if not mask.any(): + return [] + + rows: list[NDArray[numpy.float64]] = [] + counts = refs.counts[start:stop] + unit_mask = mask & (counts[:, 0] == 1) & (counts[:, 1] == 1) + if unit_mask.any(): + rows.append(_make_ref_rows( + refs.xy[start:stop][unit_mask], + refs.angle_rad[start:stop][unit_mask], + refs.invert_y[start:stop][unit_mask], + refs.scale[start:stop][unit_mask], + )) + + aref_indices = numpy.nonzero(mask & ~unit_mask)[0] + for idx in aref_indices: + abs_idx = start + int(idx) + rows.append(_expand_aref_row( + xy=refs.xy[abs_idx], + xy0=refs.xy0[abs_idx], + xy1=refs.xy1[abs_idx], + counts=refs.counts[abs_idx], + angle_rad=float(refs.angle_rad[abs_idx]), + invert_y=bool(refs.invert_y[abs_idx]), + scale=float(refs.scale[abs_idx]), + )) + return rows + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._payload.cell_order: + if name in self._cache: + graph[name] = _pattern_children(self._cache[name]) + else: + graph[name] = self._raw_children(name) + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + graph = self.child_graph(dangling='ignore') + names = set(graph) + not_toplevel: set[str] = set() + for children in graph.values(): + not_toplevel |= children + return list(names - not_toplevel) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + target_id = self._payload.cells.get(name) + for parent in parent_graph.get(name, set()): + if parent in self._cache: + for ref in self._cache[parent].refs.get(name, []): + instances[parent].append(ref.as_transforms()) + continue + + if target_id is None or parent not in self._payload.cells: + continue + rows = self._collect_raw_transforms(self._payload.cells[parent], target_id.cell_id) + if rows: + instances[parent].extend(rows) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + full_path = (parent,) + path + result[full_path] = instances + return result + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + view = _coerce_library_view(source) + source_order = _source_order(view) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + + for name in source_order: + visible = name + if visible in self._entries or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._entries or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return rename_map + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + remap = lambda target: None if target is None else self._effective_target(layer, target) + pat = _remap_pattern_targets(source_pat, remap) + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = _pattern_children(entry) + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + +def readfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + lib = ArrowLibrary.from_file(filename) + return lib, lib.library_info + + +def load_libraryfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + return readfile(filename) + + +def _get_write_info( + library: Mapping[str, Pattern] | ILibraryView, + *, + meters_per_unit: float | None, + logical_units_per_unit: float | None, + library_name: str | None, + ) -> tuple[float, float, str]: + if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: + return meters_per_unit, logical_units_per_unit, library_name + + infos: list[dict[str, Any]] = [] + if isinstance(library, ArrowLibrary): + infos.append(library.library_info) + elif isinstance(library, OverlayLibrary): + for layer in library._layers: + if isinstance(layer.library, ArrowLibrary): + infos.append(layer.library.library_info) + + if infos: + unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} + if len(unit_pairs) > 1: + raise LibraryError('Merged lazy GDS sources must have identical units before writing') + info = infos[0] + meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit + logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit + name = info['name'] if library_name is None else library_name + return meters, logical, name + + if meters_per_unit is None or logical_units_per_unit is None or library_name is None: + raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') + return meters_per_unit, logical_units_per_unit, library_name + + +def _can_copy_arrow_cell(library: ArrowLibrary, name: str) -> bool: + return name not in library._cache + + +def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: + layer = library._layers[entry.layer_index] + if not isinstance(layer.library, ArrowLibrary): + return False + if name != entry.source_name: + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(library._effective_target(layer, child) == child for child in children) + + +def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: + elements: list[klamath.elements.Element] = [] + elements += gdsii._shapes_to_elements(pat.shapes) + elements += gdsii._labels_to_texts(pat.labels) + elements += gdsii._mrefs_to_grefs(pat.refs) + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) + + +def write( + library: Mapping[str, Pattern] | ILibraryView, + stream: IO[bytes], + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + meters_per_unit, logical_units_per_unit, library_name = _get_write_info( + library, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + + header = klamath.library.FileHeader( + name=library_name.encode('ASCII'), + user_units_per_db_unit=logical_units_per_unit, + meters_per_db_unit=meters_per_unit, + ) + header.write(stream) + + if isinstance(library, ArrowLibrary): + for name in library.source_order(): + if _can_copy_arrow_cell(library, name): + stream.write(library.raw_struct_bytes(name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + if isinstance(library, OverlayLibrary): + for name in library.source_order(): + entry = library._entries[name] + if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + layer = library._layers[entry.layer_index] + assert isinstance(layer.library, ArrowLibrary) + stream.write(layer.library.raw_struct_bytes(entry.source_name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) + + +def writefile( + library: Mapping[str, Pattern] | ILibraryView, + filename: str | pathlib.Path, + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + path = pathlib.Path(filename) + + with tmpfile(path) as base_stream: + streams: tuple[Any, ...] = (base_stream,) + if path.suffix == '.gz': + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + streams = (stream,) + streams + else: + stream = base_stream + + try: + write( + library, + stream, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + finally: + for ss in streams: + ss.close() diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 2ef4d83..f3f4f6a 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_native.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == manifest.library_name + assert libarr['lib_name'].as_py() == manifest.library_name + assert len(libarr['cells']) == manifest.cells + assert 0 < len(libarr['layers']) <= manifest.layers + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py new file mode 100644 index 0000000..61a99af --- /dev/null +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -0,0 +1,174 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..file import gdsii, gdsii_lazy_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_lazy_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _make_small_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 5], [0, 5]]) + lib['leaf'] = leaf + + mid = Pattern() + mid.ref('leaf', offset=(10, 20)) + mid.ref('leaf', offset=(40, 0), repetition=Grid(a_vector=(12, 0), a_count=2, b_vector=(0, 9), b_count=2)) + lib['mid'] = mid + + top = Pattern() + top.ref('mid', offset=(100, 200)) + lib['top'] = top + return lib + + +def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_lazy.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert lib.top() == 'TOP' + assert 'TOP' in lib.child_graph(dangling='ignore') + + +def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: + gds_file = tmp_path / 'refs.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-refs') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + local = lib.find_refs_local('leaf') + assert set(local) == {'mid'} + assert sum(arr.shape[0] for arr in local['mid']) == 5 + + global_refs = lib.find_refs_global('leaf') + assert {path for path in global_refs} == {('top', 'mid', 'leaf')} + assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 + + +def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: + base_a = Library() + leaf_a = Pattern() + leaf_a.polygon((1, 0), vertices=[[0, 0], [8, 0], [8, 8], [0, 8]]) + base_a['leaf'] = leaf_a + top_a = Pattern() + top_a.ref('leaf', offset=(0, 0)) + base_a['top_a'] = top_a + + base_b = Library() + leaf_b = Pattern() + leaf_b.polygon((2, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + base_b['leaf'] = leaf_b + top_b = Pattern() + top_b.ref('leaf', offset=(20, 30)) + base_b['top_b'] = top_b + + gds_a = tmp_path / 'a.gds' + gds_b = tmp_path / 'b.gds' + gdsii.writefile(base_a, gds_a, meters_per_unit=1e-9, library_name='overlay') + gdsii.writefile(base_b, gds_b, meters_per_unit=1e-9, library_name='overlay') + + lib_a, _ = gdsii_lazy_arrow.readfile(gds_a) + lib_b, _ = gdsii_lazy_arrow.readfile(gds_b) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib_a) + rename_map = overlay.add_source(lib_b, rename_theirs=lambda lib, name: lib.get_name(name)) + renamed_leaf = rename_map['leaf'] + + assert rename_map == {'leaf': renamed_leaf} + assert renamed_leaf != 'leaf' + assert len(lib_a._cache) == 0 + assert len(lib_b._cache) == 0 + + overlay.move_references('leaf', renamed_leaf) + + out_file = tmp_path / 'overlay_out.gds' + gdsii_lazy_arrow.writefile(overlay, out_file) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf', renamed_leaf, 'top_a', 'top_b'} + assert 'top_b' in roundtrip + assert list(roundtrip['top_b'].refs.keys()) == [renamed_leaf] + + +def test_gdsii_writer_accepts_overlay_library(tmp_path: Path) -> None: + gds_file = tmp_path / 'overlay_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-src') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib) + overlay.rename('leaf', 'leaf_copy', move_references=True) + + out_file = tmp_path / 'overlay_via_eager_writer.gds' + gdsii.writefile( + overlay, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf_copy', 'mid', 'top'} + assert list(roundtrip['mid'].refs.keys()) == ['leaf_copy'] + + +def test_svg_writer_uses_detached_materialized_copy(tmp_path: Path) -> None: + pytest.importorskip('svgwrite') + from ..file import svg + from ..shapes import Path as MPath + + gds_file = tmp_path / 'svg_source.gds' + src = _make_small_library() + src['top'].path((3, 0), vertices=[[0, 0], [0, 20]], width=4) + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='svg-src') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + top_pat = lib['top'] + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) + + svg_path = tmp_path / 'lazy.svg' + svg.writefile(lib, 'top', str(svg_path)) + + assert svg_path.exists() + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) From 27e1c23c334c87b2e89d01c133c48d0f54b2ac25 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:45 -0700 Subject: [PATCH 13/15] [svg] avoid mutating the original library --- masque/file/svg.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/masque/file/svg.py b/masque/file/svg.py index 621bcdb..772aa39 100644 --- a/masque/file/svg.py +++ b/masque/file/svg.py @@ -45,6 +45,10 @@ def _make_svg_ids(names: Mapping[str, Pattern]) -> dict[str, str]: return svg_ids +def _detached_library(library: Mapping[str, Pattern]) -> dict[str, Pattern]: + return {name: pat.deepcopy() for name, pat in library.items()} + + def writefile( library: Mapping[str, Pattern], top: str, @@ -53,13 +57,12 @@ def writefile( annotate_ports: bool = False, ) -> None: """ - Write a Pattern to an SVG file, by first calling .polygonize() on it + Write a Pattern to an SVG file, by first calling .polygonize() on a detached + materialized copy to change the shapes into polygons, and then writing patterns as SVG groups (, inside ), polygons as paths (), and refs as elements. - Note that this function modifies the Pattern. - If `custom_attributes` is `True`, a non-standard `pattern_layer` attribute is written to the relevant elements. @@ -71,19 +74,21 @@ def writefile( prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. custom_attributes: Whether to write non-standard `pattern_layer` attribute to the SVG elements. annotate_ports: If True, draw an arrow for each port (similar to `Pattern.visualize(..., ports=True)`). """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize pattern pattern.polygonize() - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) @@ -96,10 +101,10 @@ def writefile( # Create file svg = svgwrite.Drawing(filename, profile='full', viewBox=viewbox_string, debug=(not custom_attributes)) - svg_ids = _make_svg_ids(library) + svg_ids = _make_svg_ids(detached) # Now create a group for each pattern and add in any Boundary and Use elements - for name, pat in library.items(): + for name, pat in detached.items(): svg_group = svg.g(id=svg_ids[name], fill='blue', stroke='red') for layer, shapes in pat.shapes.items(): @@ -158,21 +163,21 @@ def writefile_inverted( box and drawing the polygons with reverse vertex order inside it, all within one `` element. - Note that this function modifies the Pattern. - If you want pattern polygonized with non-default arguments, just call `pattern.polygonize()` prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize and flatten pattern - pattern.polygonize().flatten(library) + pattern.polygonize().flatten(detached) - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) From 9e8a8d6a2216efcdf5b05c1fc5d8482a9297877b Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 21:34:10 -0700 Subject: [PATCH 14/15] [gdsii_arrow] remove non-raw-mode arrow option; fix gzip wrapper --- masque/file/gdsii_arrow.py | 115 ++++++++------------------- masque/test/test_gdsii_arrow.py | 39 +++++++++ masque/test/test_gdsii_lazy_arrow.py | 85 ++++++++++++++++++++ 3 files changed, 158 insertions(+), 81 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index b97005b..a38dee5 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -27,7 +27,6 @@ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable from importlib.machinery import EXTENSION_SUFFIXES import importlib.util -import io import mmap import logging import os @@ -35,11 +34,11 @@ import pathlib import gzip import string import sys +import tempfile from pprint import pformat import numpy from numpy.typing import ArrayLike, NDArray -from numpy.testing import assert_equal import pyarrow from pyarrow.cffi import ffi @@ -70,8 +69,6 @@ ffi.cdef( clib: Any | None = None -ZERO_OFFSET = numpy.zeros(2) - path_cap_map = { 0: Path.Cap.Flush, @@ -195,13 +192,22 @@ def _read_annotations( def _read_to_arrow( filename: str | pathlib.Path, - *args, - **kwargs, ) -> pyarrow.Array: path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + if is_gzipped(path): + with gzip.open(path, mode='rb') as src: + data = src.read() + with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream: + tmp_stream.write(data) + tmp_name = tmp_stream.name + try: + _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + finally: + pathlib.Path(tmp_name).unlink(missing_ok=True) + else: + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) return _import_arrow_array(ptr_array, ptr_schema) @@ -236,18 +242,14 @@ def _read_selected_cells_to_arrow( def readfile( filename: str | pathlib.Path, - *args, - **kwargs, ) -> tuple[Library, dict[str, Any]]: """ - Wrapper for `read()` that takes a filename or path instead of a stream. + Read a GDSII file from a path into `masque.Library` / `Pattern` objects. Will automatically decompress gzipped files. Args: - filename: Filename to save to. - *args: passed to `read()` - **kwargs: passed to `read()` + filename: Filename to read. For callers that can consume Arrow directly, prefer `readfile_arrow()` to skip Python `Pattern` construction entirely. @@ -284,7 +286,6 @@ def readfile_arrow( def read_arrow( libarr: pyarrow.Array, - raw_mode: bool = True, ) -> tuple[Library, dict[str, Any]]: """ # TODO check GDSII file for cycles! @@ -299,8 +300,7 @@ def read_arrow( per database unit Args: - stream: Stream to read from. - raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + libarr: Arrow library payload as returned by `readfile_arrow()`. Returns: - dict of pattern_name:Patterns generated from GDSII structures @@ -436,7 +436,6 @@ def read_arrow( global_args = dict( cell_names = cell_names, layer_tups = layer_tups, - raw_mode = raw_mode, ) mlib = Library() @@ -488,7 +487,6 @@ def _srefs_to_mrefs( elem_invert_y = elem['invert_y'][start:stop] elem_angle_rad = elem['angle_rad'][start:stop] elem_scale = elem['scale'][start:stop] - raw_mode = global_args['raw_mode'] _append_plain_refs_sorted( pat=pat, @@ -498,7 +496,6 @@ def _srefs_to_mrefs( elem_invert_y=elem_invert_y, elem_angle_rad=elem_angle_rad, elem_scale=elem_scale, - raw_mode=raw_mode, ) @@ -511,14 +508,11 @@ def _append_plain_refs_sorted( elem_invert_y: NDArray[numpy.bool_ | numpy.bool], elem_angle_rad: NDArray[numpy.floating[Any]], elem_scale: NDArray[numpy.floating[Any]], - raw_mode: bool, ) -> None: elem_count = len(elem_targets) if elem_count == 0: return - make_ref = Ref._from_raw if raw_mode else Ref - target_start = 0 while target_start < elem_count: target_id = int(elem_targets[target_start]) @@ -528,7 +522,7 @@ def _append_plain_refs_sorted( append_refs = pat.refs[cell_names[target_id]].extend append_refs( - make_ref( + Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -564,10 +558,6 @@ def _arefs_to_mrefs( elem_xy0 = elem['xy0'][start:stop] elem_xy1 = elem['xy1'][start:stop] elem_counts = elem['counts'][start:stop] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid if len(elem_targets) == 0: return @@ -581,12 +571,12 @@ def _arefs_to_mrefs( append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] - append_ref(make_ref( + append_ref(Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=None, )) @@ -613,13 +603,10 @@ def _sref_props_to_mrefs( elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -655,20 +642,16 @@ def _aref_props_to_mrefs( elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid for ee in range(elem_count): a_count, b_count = elem_counts[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) pat.refs[cell_names[int(elem_targets[ee])]].append(ref) @@ -693,7 +676,6 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] @@ -701,10 +683,7 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) - else: - mlabel = Label(string=string, offset=offset, annotations=annotations) + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -730,7 +709,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] @@ -743,23 +721,13 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - path = Path._from_raw( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - annotations=annotations, - ) - else: - path = Path( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - offset=ZERO_OFFSET, - annotations=annotations, - ) + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -786,23 +754,16 @@ def _boundary_batches_to_polygons( elem_poly_off = poly_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=None) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) + poly = Polygon._from_raw(vertices=vertices, annotations=None) pat.shapes[layer].append(poly) else: - if raw_mode: - polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) - else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) pat.shapes[layer].append(polys) @@ -826,14 +787,10 @@ def _rect_batches_to_rectcollections( elem_rect_off = rect_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] - if raw_mode: - rect_collection = RectCollection._from_raw(rects=rects, annotations=None) - else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) pat.shapes[layer].append(rect_collection) @@ -860,15 +817,11 @@ def _boundary_props_to_polygons( prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=annotations) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index f3f4f6a..660818c 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_matches_gdsii_readfile_for_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_native.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -199,6 +211,33 @@ def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> No assert 0 < len(libarr['layers']) <= manifest.layers +def test_gdsii_arrow_readfile_arrow_reads_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'native_payload.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == 'masque-klamath' + assert libarr['lib_name'].as_py() == 'masque-klamath' + assert len(libarr['cells']) == len(lib) + assert len(libarr['layers']) > 0 + + +def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'removed_raw_mode.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, _ = gdsii_arrow.readfile_arrow(gds_file) + + with pytest.raises(TypeError): + gdsii_arrow.readfile(gds_file, raw_mode=False) + + with pytest.raises(TypeError): + gdsii_arrow.read_arrow(libarr, raw_mode=False) + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 61a99af..b30b058 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -34,6 +34,78 @@ def _make_small_library() -> Library: return lib +def _make_complex_ref_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]]) + lib['leaf'] = leaf + + child = Pattern() + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25) + lib['child'] = child + + sibling = Pattern() + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + ) + lib['sibling'] = sibling + + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref( + 'child', + offset=(40, 0), + mirrored=True, + rotation=numpy.pi / 4, + scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2), + ) + lib['fanout'] = fanout + + top = Pattern() + top.ref('child', offset=(500, 600)) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi) + top.ref('fanout', offset=(250, -75)) + lib['top'] = top + + return lib + + +def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: + arr = numpy.asarray(values, dtype=float) + arr = numpy.atleast_2d(arr) + rows = [ + ( + round(float(row[0]), 8), + round(float(row[1]), 8), + round(float(row[2]), 8), + bool(int(round(float(row[3])))), + round(float(row[4]), 8), + ) + for row in arr + ] + return tuple(sorted(rows)) + + +def _local_refs_key(refs: dict[str, list[numpy.ndarray]]) -> dict[str, tuple[tuple[object, ...], ...]]: + return { + parent: _transform_rows_key(numpy.concatenate(transforms)) + for parent, transforms in refs.items() + } + + +def _global_refs_key(refs: dict[tuple[str, ...], numpy.ndarray]) -> dict[tuple[str, ...], tuple[tuple[object, ...], ...]]: + return { + path: _transform_rows_key(transforms) + for path, transforms in refs.items() + } + + def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_lazy.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -62,6 +134,19 @@ def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 +def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None: + gds_file = tmp_path / 'complex_refs.gds' + src = _make_complex_ref_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-complex-refs') + + eager, _ = gdsii.readfile(gds_file) + lazy, _ = gdsii_lazy_arrow.readfile(gds_file) + + for name in ('leaf', 'child'): + assert _local_refs_key(lazy.find_refs_local(name)) == _local_refs_key(eager.find_refs_local(name)) + assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() From 7439e4cae9df160eaabc473b75512f135d863a32 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 21 Apr 2026 21:50:18 -0700 Subject: [PATCH 15/15] [arrow] improve test coverage and error handling --- masque/file/gdsii_arrow.py | 35 ++++++++++--- masque/test/test_gdsii_arrow.py | 59 +++++++++++++++++++++- masque/test/test_gdsii_lazy_arrow.py | 75 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index a38dee5..95229ea 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -37,6 +37,7 @@ import sys import tempfile from pprint import pformat +from klamath.basic import KlamathError import numpy from numpy.typing import ArrayLike, NDArray import pyarrow @@ -54,9 +55,10 @@ logger = logging.getLogger(__name__) ffi.cdef( """ - void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); - void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); - void read_cells_bytes( + const char* last_error_message(void); + int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); + int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + int read_cells_bytes( uint8_t* data, size_t size, uint64_t* ranges, @@ -203,11 +205,11 @@ def _read_to_arrow( tmp_stream.write(data) tmp_name = tmp_stream.name try: - _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path') finally: pathlib.Path(tmp_name).unlink(missing_ok=True) else: - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path') return _import_arrow_array(ptr_array, ptr_schema) @@ -217,12 +219,24 @@ def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: return pyarrow.Array._import_from_c(iptr_array, iptr_schema) +def _call_native(status: int, action: str) -> None: + if status == 0: + return + + err_ptr = _get_clib().last_error_message() + if err_ptr == ffi.NULL: + raise KlamathError(f'{action} failed') + + message = ffi.string(err_ptr).decode(errors='replace') + raise KlamathError(message) + + def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) buf_view = memoryview(buffer) cbuf = ffi.from_buffer('uint8_t[]', buf_view) - _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes') return _import_arrow_array(ptr_array, ptr_schema) @@ -236,7 +250,10 @@ def _read_selected_cells_to_arrow( cbuf = ffi.from_buffer('uint8_t[]', buf_view) flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) cranges = ffi.from_buffer('uint64_t[]', flat_ranges) - _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + _call_native( + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema), + 'read_cells_bytes', + ) return _import_arrow_array(ptr_array, ptr_schema) @@ -713,7 +730,9 @@ def _gpaths_to_mpaths( layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] - cap_int = elem_path_types[ee] + cap_int = int(elem_path_types[ee]) + if cap_int not in path_cap_map: + raise PatternError(f'Unrecognized path type: {cap_int}') cap = path_cap_map[cap_int] if cap_int == 4: cap_extensions = elem_extensions[ee] diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 660818c..b2962f8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -1,11 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') -from .. import Ref, Label +from .. import Ref, Label, PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -175,6 +179,26 @@ def _make_arrow_test_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_roundtrip.gds' @@ -238,6 +262,28 @@ def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: gdsii_arrow.read_arrow(libarr, raw_mode=False) +def test_gdsii_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_arrow + try: + gdsii_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -435,6 +481,17 @@ def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' +def test_gdsii_arrow_invalid_path_type_matches_gdsii(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii_arrow.readfile(gds_file) + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index b30b058..eb62721 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -1,10 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') +from .. import PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -76,6 +81,26 @@ def _make_complex_ref_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: arr = numpy.asarray(values, dtype=float) arr = numpy.atleast_2d(arr) @@ -147,6 +172,38 @@ def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) +def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_lazy_arrow + try: + gdsii_lazy_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_lazy_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + +def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + lib['top'] + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() @@ -165,6 +222,24 @@ def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> Non assert out_file.read_bytes() == gds_file.read_bytes() +def test_gdsii_lazy_arrow_gzipped_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds.gz' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through-gz') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds.gz' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: base_a = Library() leaf_a = Pattern()