From bd0c8c9d169c9983467ca181a47ebbdbb45bb1a9 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 01/36] [gdsii_arrow] add gdsii_arrow --- masque/file/gdsii_arrow.py | 453 +++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..e56a48e --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,453 @@ +# ruff: noqa: ARG001, F401 +""" +GDSII file format readers and writers using the `TODO` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_equal +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path, PolyCollection +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def _read_to_arrow( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> pyarrow.Array: + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + return arrow_arr + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_tups = [tuple(pair) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + rf = libarr['cells'].values.field('refs') + refs = dict( + offsets = rf.offsets.to_numpy(), + targets = rf.values.field('target').to_numpy(), + xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), + angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + scale = rf.values.field('mag').fill_null(1).to_numpy(), + rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), + rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + prop_off = rf.values.field('properties').offsets.to_numpy(), + prop_key = rf.values.field('properties').values.field('key').to_numpy(), + prop_val = rf.values.field('properties').values.field('value').to_pylist(), + ) + + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), + ) + + elements = dict( + boundaries = get_geom(libarr, 'boundaries'), + paths = get_geom(libarr, 'paths'), + boxes = get_geom(libarr, 'boxes'), + nodes = get_geom(libarr, 'nodes'), + texts = texts, + refs = refs, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').fill_null(0).to_numpy(), + path_type = paths.values.field('path_type').fill_null(0).to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').fill_null(0).to_numpy(), + paths.values.field('extension_end').fill_null(0).to_numpy(), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + raw_mode = raw_mode, + ) + + mlib = Library() + for cc in range(len(libarr['cells'])): + name = cell_names[cell_ids[cc]] + pat = Pattern() + _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def _grefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + targets = elem['targets'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_invert_y = elem['invert_y'][elem_slc][:elem_count] + elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] + elem_scale = elem['scale'][elem_slc][:elem_count] + elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] + elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] + elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] + rep_valid = elem['rep_valid'][elem_slc][:elem_count] + + + for ee in range(elem_count): + target = cell_names[targets[ee]] + offset = xy[ee] + mirr = elem_invert_y[ee] + rot = elem_angle_rad[ee] + mag = elem_scale[ee] + + rep: None | Grid = None + if rep_valid[ee]: + a_vector = elem_rep_xy0[ee] + b_vector = elem_rep_xy1[ee] + a_count, b_count = elem_rep_counts[ee] + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + pat.refs[target].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_strings = elem['string'][elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + offset = xy[ee] + string = elem_strings[ee] + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + mlabel = Label(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_widths = elem['width'][elem_slc][:elem_count] + elem_path_types = elem['path_type'][elem_slc][:elem_count] + elem_extensions = elem['extensions'][elem_slc][:elem_count] + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem_widths[ee] + cap_int = elem_path_types[ee] + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem_extensions[ee] + else: + cap_extensions = None + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + width=width, cap=cap,cap_extensions=cap_extensions) + pat.shapes[layer].append(path) + + +def _boundaries_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + xy_counts = xy_offs[1:] - xy_offs[:-1] + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + prop_counts = prop_offs[1:] - prop_offs[:-1] + elem_layer_inds = layer_inds[elem_slc][:elem_count] + + order = numpy.argsort(elem_layer_inds, stable=True) + unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): + ee_inds = order[ff:ff + nn] + layer = layer_tups[layer_ind] + propless_mask = prop_counts[ee_inds] == 0 + + poly_count_on_layer = propless_mask.sum() + if poly_count_on_layer == 1: + propless_mask[:] = 0 # Never make a 1-element collection + elif poly_count_on_layer > 1: + propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point + vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) + vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) + + for ii, ee in enumerate(ee_inds[propless_mask]): + vo = vertex_offsets[ii] + vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] + + polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) + pat.shapes[layer].append(polys) + + # Handle single polygons + for ee in ee_inds[~propless_mask]: + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} + + poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) + + +#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: +# return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From 18f12792c4ffbf66a2954e8977e3c82f7d8f4183 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:20:50 -0700 Subject: [PATCH 02/36] add some missing deps --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af8802c..ad03fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ [dependency-groups] dev = [ - "pytest", + "masque[arrow]", "masque[oasis]", "masque[dxf]", "masque[svg]", @@ -52,6 +52,7 @@ dev = [ "masque[text]", "masque[manhattanize]", "masque[manhattanize_slow]", + "masque[boolean]", "matplotlib>=3.10.8", "pytest>=9.0.2", "ruff>=0.15.5", @@ -66,6 +67,7 @@ build-backend = "hatchling.build" path = "masque/__init__.py" [project.optional-dependencies] +arrow = ["pyarrow", "cffi"] oasis = ["fatamorgana~=0.11"] dxf = ["ezdxf~=1.4"] svg = ["svgwrite"] @@ -121,4 +123,3 @@ mypy_path = "stubs" python_version = "3.11" strict = false check_untyped_defs = true - From 09fec67a2173e9d777c00672bb26fa4b9898a691 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:21:59 -0700 Subject: [PATCH 03/36] [gdsii_arrow] misc correctness work --- masque/file/gdsii_arrow.py | 173 +++++++++++++++++++++++------- masque/test/test_gdsii_arrow.py | 180 ++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 37 deletions(-) create mode 100644 masque/test/test_gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index e56a48e..546d575 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -25,12 +25,16 @@ Notes: """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable +from importlib.machinery import EXTENSION_SUFFIXES +import importlib.util import io import mmap import logging +import os import pathlib import gzip import string +import sys from pprint import pformat import numpy @@ -49,9 +53,10 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +clib: Any | None = None + path_cap_map = { 0: Path.Cap.Flush, @@ -65,16 +70,123 @@ def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) +def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]: + layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16) + dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16) + return numpy.stack((layer, dtype), axis=-1) + + +def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]: + a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64) + b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64) + return numpy.stack((a_count, b_count), axis=-1) + + +def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]: + xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32) + yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32) + return numpy.stack((xx, yy), axis=-1) + + +def _local_library_filename() -> str: + if sys.platform.startswith('linux'): + return 'libklamath_rs_ext.so' + if sys.platform == 'darwin': + return 'libklamath_rs_ext.dylib' + if sys.platform == 'win32': + return 'klamath_rs_ext.dll' + raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}') + + +def _installed_library_candidates() -> list[pathlib.Path]: + candidates: list[pathlib.Path] = [] + + try: + spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext') + except ModuleNotFoundError: + spec = None + if spec is not None and spec.origin is not None: + candidates.append(pathlib.Path(spec.origin)) + + try: + pkg_spec = importlib.util.find_spec('klamath_rs_ext') + except ModuleNotFoundError: + pkg_spec = None + if pkg_spec is not None and pkg_spec.submodule_search_locations is not None: + for location in pkg_spec.submodule_search_locations: + pkg_dir = pathlib.Path(location) + for suffix in EXTENSION_SUFFIXES: + candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}'))) + + return candidates + + +def _repo_library_candidates() -> list[pathlib.Path]: + repo_root = pathlib.Path(__file__).resolve().parents[2] + library_name = _local_library_filename() + return [ + repo_root / 'klamath-rs' / 'target' / 'release' / library_name, + repo_root / 'klamath-rs' / 'target' / 'debug' / library_name, + ] + + +def find_klamath_rs_library() -> pathlib.Path | None: + env_path = os.environ.get('KLAMATH_RS_EXT_LIB') + if env_path: + candidate = pathlib.Path(env_path).expanduser() + if candidate.exists(): + return candidate.resolve() + + seen: set[pathlib.Path] = set() + for candidate in _installed_library_candidates() + _repo_library_candidates(): + resolved = candidate.expanduser() + if resolved in seen: + continue + seen.add(resolved) + if resolved.exists(): + return resolved.resolve() + return None + + +def is_available() -> bool: + return find_klamath_rs_library() is not None + + +def _get_clib() -> Any: + global clib + if clib is None: + lib_path = find_klamath_rs_library() + if lib_path is None: + raise ImportError( + 'Could not locate klamath_rs_ext shared library. ' + 'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` ' + 'or set KLAMATH_RS_EXT_LIB to the built library path.' + ) + clib = ffi.dlopen(str(lib_path)) + return clib + + +def _read_annotations( + prop_offs: NDArray[numpy.integer[Any]], + prop_key: NDArray[numpy.integer[Any]], + prop_val: list[str], + ee: int, + ) -> annotations_t: + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii >= prop_ff: + return None + return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + def _read_to_arrow( filename: str | pathlib.Path, *args, **kwargs, ) -> pyarrow.Array: - path = pathlib.Path(filename) - path.resolve() + path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - clib.read_path(str(path).encode(), ptr_array, ptr_schema) + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) @@ -132,7 +244,7 @@ def read_arrow( """ library_info = _read_header(libarr) - layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) layer_tups = [tuple(pair) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() @@ -155,14 +267,14 @@ def read_arrow( refs = dict( offsets = rf.offsets.to_numpy(), targets = rf.values.field('target').to_numpy(), - xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), scale = rf.values.field('mag').fill_null(1).to_numpy(), rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), + rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), + rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), prop_off = rf.values.field('properties').offsets.to_numpy(), prop_key = rf.values.field('properties').values.field('key').to_numpy(), prop_val = rf.values.field('properties').values.field('value').to_pylist(), @@ -172,7 +284,7 @@ def read_arrow( texts = dict( offsets = txt.offsets.to_numpy(), layer_inds = txt.values.field('layer').to_numpy(), - xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()), string = txt.values.field('string').to_pylist(), prop_off = txt.values.field('properties').offsets.to_numpy(), prop_key = txt.values.field('properties').values.field('key').to_numpy(), @@ -222,9 +334,9 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: Read the file header and create the library_info dict. """ library_info = dict( - name = libarr['lib_name'], - meters_per_unit = libarr['meters_per_db_unit'], - logical_units_per_unit = libarr['user_units_per_db_unit'], + name = libarr['lib_name'].as_py(), + meters_per_unit = libarr['meters_per_db_unit'].as_py(), + logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(), ) return library_info @@ -245,6 +357,8 @@ def _grefs_to_mrefs( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_targets = targets[elem_slc][:elem_count] + elem_xy = xy[elem_slc][:elem_count] elem_invert_y = elem['invert_y'][elem_slc][:elem_count] elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] elem_scale = elem['scale'][elem_slc][:elem_count] @@ -255,8 +369,8 @@ def _grefs_to_mrefs( for ee in range(elem_count): - target = cell_names[targets[ee]] - offset = xy[ee] + target = cell_names[elem_targets[ee]] + offset = elem_xy[ee] mirr = elem_invert_y[ee] rot = elem_angle_rad[ee] mag = elem_scale[ee] @@ -268,11 +382,7 @@ def _grefs_to_mrefs( a_count, b_count = elem_rep_counts[ee] rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -293,19 +403,16 @@ def _texts_to_labels( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] - offset = xy[ee] + offset = elem_xy[ee] string = elem_strings[ee] - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -345,11 +452,7 @@ def _gpaths_to_mpaths( else: cap_extensions = None - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) @@ -406,11 +509,7 @@ def _boundaries_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py new file mode 100644 index 0000000..26713e9 --- /dev/null +++ b/masque/test/test_gdsii_arrow.py @@ -0,0 +1,180 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..shapes import Path as MPath +from ..file import gdsii, gdsii_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: + if annotations is None: + return None + return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) + + +def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(values, dtype=float)).astype(int) + if arr.ndim == 1: + return tuple(arr.tolist()) + return tuple(tuple(row.tolist()) for row in arr) + + +def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: + if isinstance(shape, MPath): + cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) + return [( + 'path', + layer, + _coord_key(shape.vertices), + _coord_key(shape.offset), + int(round(float(shape.width))), + shape.cap.name, + cap_extensions, + _annotations_key(shape.annotations), + )] + + keys = [] + for poly in shape.to_polygons(): + keys.append(( + 'polygon', + layer, + _coord_key(poly.vertices), + _coord_key(poly.offset), + _annotations_key(poly.annotations), + )) + return keys + + +def _ref_key(target: str, ref: object) -> tuple[object, ...]: + repetition = None + if ref.repetition is not None: + repetition = ( + _coord_key(ref.repetition.a_vector), + int(ref.repetition.a_count), + _coord_key(ref.repetition.b_vector), + int(ref.repetition.b_count), + ) + return ( + target, + _coord_key(ref.offset), + round(float(ref.rotation), 8), + round(float(ref.scale), 8), + bool(ref.mirrored), + repetition, + _annotations_key(ref.annotations), + ) + + +def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: + return ( + layer, + label.string, + _coord_key(label.offset), + _annotations_key(label.annotations), + ) + + +def _pattern_summary(pattern: Pattern) -> dict[str, object]: + shape_keys: list[tuple[object, ...]] = [] + for layer, shapes in pattern.shapes.items(): + for shape in shapes: + shape_keys.extend(_shape_key(shape, layer)) + + ref_keys = [ + _ref_key(target, ref) + for target, refs in pattern.refs.items() + for ref in refs + ] + + label_keys = [ + _label_key(layer, label) + for layer, labels in pattern.labels.items() + for label in labels + ] + + return { + 'shapes': sorted(shape_keys), + 'refs': sorted(ref_keys), + 'labels': sorted(label_keys), + } + + +def _library_summary(lib: Library) -> dict[str, dict[str, object]]: + return {name: _pattern_summary(pattern) for name, pattern in lib.items()} + + +def _make_arrow_test_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) + lib['leaf'] = leaf + + child = Pattern() + child.path( + (2, 0), + vertices=[[0, 0], [15, 5], [30, 5]], + width=6, + cap=MPath.Cap.SquareCustom, + cap_extensions=(2, 4), + annotations={'2': ['child-path']}, + ) + child.label((11, 0), string='CHILD', offset=(7, 8), annotations={'11': ['child-label']}) + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25, annotations={'12': ['child-ref']}) + lib['child'] = child + + sibling = Pattern() + sibling.polygon((3, 0), vertices=[[0, 0], [5, 0], [5, 6], [0, 6]]) + sibling.label((12, 0), string='SIB', offset=(1, 2), annotations={'13': ['sib-label']}) + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + annotations={'14': ['sib-ref']}, + ) + lib['sibling'] = sibling + + top = Pattern() + top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) + lib['top'] = top + + return lib + + +def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + +def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_smoke.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert 'TOP' in lib + assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 From 9df42000b71a15292f7ea4861f0caf55132a7ad2 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:22:27 -0700 Subject: [PATCH 04/36] [gdsii] add some profiling helpers --- examples/generate_gds_perf.py | 5 + examples/profile_gdsii_readers.py | 74 ++++ masque/file/gdsii_perf.py | 633 ++++++++++++++++++++++++++++++ masque/test/test_gdsii_perf.py | 24 ++ 4 files changed, 736 insertions(+) create mode 100644 examples/generate_gds_perf.py create mode 100644 examples/profile_gdsii_readers.py create mode 100644 masque/file/gdsii_perf.py create mode 100644 masque/test/test_gdsii_perf.py diff --git a/examples/generate_gds_perf.py b/examples/generate_gds_perf.py new file mode 100644 index 0000000..6bdd999 --- /dev/null +++ b/examples/generate_gds_perf.py @@ -0,0 +1,5 @@ +from masque.file.gdsii_perf import main + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py new file mode 100644 index 0000000..c77c589 --- /dev/null +++ b/examples/profile_gdsii_readers.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import importlib +import json +import time +from pathlib import Path + +from masque import LibraryError + + +READERS = { + 'gdsii': ('masque.file.gdsii', 'readfile'), + 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), + } + + +def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: + assert hasattr(lib, '__len__') + assert hasattr(lib, 'tops') + tops = lib.tops() # type: ignore[no-any-return, attr-defined] + try: + unique_top = lib.top() # type: ignore[no-any-return, attr-defined] + except LibraryError: + unique_top = None + + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'library_name': info['name'], + 'cell_count': len(lib), # type: ignore[arg-type] + 'topcells': tops, + 'topcell': unique_top, + } + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') + parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--path', type=Path, required=True) + parser.add_argument('--warmup', type=int, default=1) + parser.add_argument('--repeat', type=int, default=1) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + module_name, attr_name = READERS[args.reader] + readfile = getattr(importlib.import_module(module_name), attr_name) + path = args.path.expanduser().resolve() + + for _ in range(args.warmup): + readfile(path) + + runs = [] + for _ in range(args.repeat): + start = time.perf_counter() + lib, info = readfile(path) + elapsed_s = time.perf_counter() - start + runs.append(_summarize(path, elapsed_s, info, lib)) + + print(json.dumps({ + 'reader': args.reader, + 'warmup': args.warmup, + 'repeat': args.repeat, + 'runs': runs, + }, indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/file/gdsii_perf.py b/masque/file/gdsii_perf.py new file mode 100644 index 0000000..38d1a7d --- /dev/null +++ b/masque/file/gdsii_perf.py @@ -0,0 +1,633 @@ +""" +Synthetic GDS fixture generation for reader/writer performance testing. + +The presets here are intentionally hierarchical and deterministic. They aim to +approximate a pair of real-world layout families discussed during GDS reader and +writer work: + +* `many_cells`: tens of thousands of cells, moderate reference count, very heavy + box usage after flattening, and moderate polygon density. +* `many_instances`: a much smaller cell library with very high reference count, + similar box density, and far fewer polygons. + +Fixtures are written by streaming structures through `klamath` directly so large +benchmark files can be produced without first materializing an equally large +`masque.Library` in Python. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +import argparse +import json +import math + +import numpy +import klamath +from klamath import elements + + +EMPTY_PROPERTIES: dict[int, bytes] = {} +METERS_PER_DB_UNIT = 1e-9 +USER_UNITS_PER_DB_UNIT = 1e-3 +TOTAL_LAYERS = 200 + + +@dataclass(frozen=True) +class FixturePreset: + name: str + total_layers: int + box_layers: int + heavy_box_layers: int + polygon_layers: int + box_cells: int + poly_cells: int + box_wrappers: int + poly_wrappers: int + box_clusters: int + poly_clusters: int + box_cluster_refs: int + poly_cluster_refs: int + top_direct_box_refs: int + top_direct_poly_refs: int + heavy_boxes_per_cell: int + regular_boxes_per_cell: int + polygons_per_cell: int + path_stride: int + text_stride: int + box_cluster_array: tuple[int, int] + top_box_array: tuple[int, int] + poly_cluster_array: tuple[int, int] + top_poly_array: tuple[int, int] + rare_annotation_stride: int + + +PRESETS: dict[str, FixturePreset] = { + 'many_cells': FixturePreset( + name='many_cells', + total_layers=TOTAL_LAYERS, + box_layers=20, + heavy_box_layers=3, + polygon_layers=20, + box_cells=17_000, + poly_cells=6_000, + box_wrappers=18_000, + poly_wrappers=6_000, + box_clusters=2_000, + poly_clusters=999, + box_cluster_refs=24, + poly_cluster_refs=16, + top_direct_box_refs=21_000, + top_direct_poly_refs=7_000, + heavy_boxes_per_cell=6, + regular_boxes_per_cell=2, + polygons_per_cell=50, + path_stride=2, + text_stride=3, + box_cluster_array=(24, 16), + top_box_array=(8, 8), + poly_cluster_array=(4, 2), + top_poly_array=(3, 2), + rare_annotation_stride=1_250, + ), + 'many_instances': FixturePreset( + name='many_instances', + total_layers=TOTAL_LAYERS, + box_layers=25, + heavy_box_layers=3, + polygon_layers=10, + box_cells=2_500, + poly_cells=500, + box_wrappers=1_000, + poly_wrappers=500, + box_clusters=1_000, + poly_clusters=499, + box_cluster_refs=1_200, + poly_cluster_refs=400, + top_direct_box_refs=102_001, + top_direct_poly_refs=0, + heavy_boxes_per_cell=40, + regular_boxes_per_cell=16, + polygons_per_cell=60, + path_stride=1, + text_stride=2, + box_cluster_array=(1, 1), + top_box_array=(1, 1), + poly_cluster_array=(1, 1), + top_poly_array=(1, 1), + rare_annotation_stride=250, + ), + } + + +@dataclass(frozen=True) +class FixtureManifest: + preset: str + scale: float + gds_path: str + library_name: str + cells: int + refs: int + layers: int + box_layers: int + heavy_box_layers: list[list[int]] + polygon_layers: list[list[int]] + hierarchical_boxes_per_heavy_layer: int + hierarchical_boxes_per_regular_layer: int + hierarchical_polygons_total: int + hierarchical_paths_total: int + hierarchical_texts_total: int + flattened_box_placements: int + flattened_poly_placements: int + estimated_flat_boxes_per_heavy_layer: int + estimated_flat_polygons_per_active_polygon_layer: int + + +def _scaled_count(value: int, scale: float, minimum: int = 0) -> int: + if value == 0: + return 0 + scaled = int(math.ceil(value * scale)) + return max(minimum, scaled) + + +def _scaled_preset(preset: FixturePreset, scale: float) -> FixturePreset: + if scale <= 0: + raise ValueError(f'scale must be positive, got {scale!r}') + + return FixturePreset( + name=preset.name, + total_layers=preset.total_layers, + box_layers=min(preset.box_layers, preset.total_layers), + heavy_box_layers=min(preset.heavy_box_layers, preset.box_layers), + polygon_layers=min(preset.polygon_layers, preset.total_layers), + box_cells=_scaled_count(preset.box_cells, scale, minimum=1), + poly_cells=_scaled_count(preset.poly_cells, scale, minimum=1), + box_wrappers=_scaled_count(preset.box_wrappers, scale), + poly_wrappers=_scaled_count(preset.poly_wrappers, scale), + box_clusters=_scaled_count(preset.box_clusters, scale, minimum=1), + poly_clusters=_scaled_count(preset.poly_clusters, scale, minimum=1), + box_cluster_refs=_scaled_count(preset.box_cluster_refs, scale, minimum=1), + poly_cluster_refs=_scaled_count(preset.poly_cluster_refs, scale, minimum=1), + top_direct_box_refs=_scaled_count(preset.top_direct_box_refs, scale), + top_direct_poly_refs=_scaled_count(preset.top_direct_poly_refs, scale), + heavy_boxes_per_cell=max(1, preset.heavy_boxes_per_cell), + regular_boxes_per_cell=max(1, preset.regular_boxes_per_cell), + polygons_per_cell=max(1, preset.polygons_per_cell), + path_stride=max(1, preset.path_stride), + text_stride=max(1, preset.text_stride), + box_cluster_array=preset.box_cluster_array, + top_box_array=preset.top_box_array, + poly_cluster_array=preset.poly_cluster_array, + top_poly_array=preset.top_poly_array, + rare_annotation_stride=max(1, _scaled_count(preset.rare_annotation_stride, scale, minimum=1)), + ) + + +def _rect_xy(xmin: int, ymin: int, xmax: int, ymax: int) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + return numpy.array( + [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]], + dtype=numpy.int32, + ) + + +def _poly_xy(points: list[tuple[int, int]]) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + closed = points + [points[0]] + return numpy.array(closed, dtype=numpy.int32) + + +def _sref( + target: str, + xy: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([xy], dtype=numpy.int32), + colrow=None, + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _aref( + target: str, + origin: tuple[int, int], + counts: tuple[int, int], + step: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + cols, rows = counts + dx, dy = step + xy = numpy.array( + [ + origin, + (origin[0] + cols * dx, origin[1]), + (origin[0], origin[1] + rows * dy), + ], + dtype=numpy.int32, + ) + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=xy, + colrow=(cols, rows), + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _annotation(index: int) -> dict[int, bytes]: + return {1: f'perf-{index}'.encode('ASCII')} + + +def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 17) * 600 + ybase = (index // 17) * 180 + + for layer in range(cfg.heavy_box_layers): + for box_idx in range(cfg.heavy_boxes_per_cell): + x0 = xbase + box_idx * 22 + y0 = ybase + layer * 40 + width = 10 + ((index + box_idx + layer) % 7) * 6 + height = 10 + ((index * 3 + box_idx + layer) % 5) * 8 + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 and box_idx == 0 and layer == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=properties, + )) + + for layer in range(cfg.heavy_box_layers, cfg.box_layers): + for box_idx in range(cfg.regular_boxes_per_cell): + x0 = xbase + box_idx * 38 + y0 = ybase + (layer - cfg.heavy_box_layers) * 28 + 400 + width = 18 + ((index + layer + box_idx) % 9) * 4 + height = 12 + ((index + 2 * layer + box_idx) % 6) * 5 + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=EMPTY_PROPERTIES, + )) + + return cell_elements + + +def _make_poly_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 19) * 900 + ybase = (index // 19) * 260 + + for poly_idx in range(cfg.polygons_per_cell): + layer = poly_idx % cfg.polygon_layers + dx = xbase + (poly_idx % 5) * 120 + dy = ybase + (poly_idx // 5) * 80 + size = 18 + ((index + poly_idx + layer) % 11) * 7 + points = [ + (dx, dy), + (dx + size, dy + size // 5), + (dx + size + size // 3, dy + size), + (dx + size // 2, dy + size + size // 2), + (dx - size // 4, dy + size // 2), + ] + properties = _annotation(index) if poly_idx == 0 and index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_poly_xy(points), + properties=properties, + )) + + if index % cfg.path_stride == 0: + layer = index % cfg.polygon_layers + cell_elements.append(elements.Path( + layer=(layer, 1), + path_type=2, + width=12 + (index % 5) * 4, + extension=(0, 0), + xy=numpy.array( + [ + [xbase, ybase + 900], + [xbase + 240, ybase + 930], + [xbase + 420, ybase + 960], + ], + dtype=numpy.int32, + ), + properties=EMPTY_PROPERTIES, + )) + + if index % cfg.text_stride == 0: + layer = index % cfg.polygon_layers + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Text( + layer=(layer, 2), + presentation=0, + path_type=0, + width=0, + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([[xbase + 64, ybase + 1536]], dtype=numpy.int32), + string=f'T{index:05d}'.encode('ASCII'), + properties=properties, + )) + + return cell_elements + + +def _write_struct(stream: Any, name: str, cell_elements: list[elements.Element]) -> None: + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=cell_elements) + + +def _box_name(index: int) -> str: + return f'box_{index:05d}' + + +def _poly_name(index: int) -> str: + return f'poly_{index:05d}' + + +def _box_wrapper_name(index: int) -> str: + return f'box_wrap_{index:05d}' + + +def _poly_wrapper_name(index: int) -> str: + return f'poly_wrap_{index:05d}' + + +def _box_cluster_name(index: int) -> str: + return f'box_cluster_{index:05d}' + + +def _poly_cluster_name(index: int) -> str: + return f'poly_cluster_{index:05d}' + + +def _write_box_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_cells): + _write_struct(stream, _box_name(idx), _make_box_cell(_box_name(idx), idx, cfg)) + + +def _write_poly_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.poly_cells): + _write_struct(stream, _poly_name(idx), _make_poly_cell(_poly_name(idx), idx, cfg)) + + +def _write_wrappers(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_wrappers): + target = _box_name(idx % cfg.box_cells) + origin = ((idx % 97) * 2_000, (idx // 97) * 2_000) + _write_struct(stream, _box_wrapper_name(idx), [_sref(target, origin)]) + + for idx in range(cfg.poly_wrappers): + target = _poly_name(idx % cfg.poly_cells) + origin = ((idx % 61) * 3_200, (idx // 61) * 3_200) + _write_struct(stream, _poly_wrapper_name(idx), [_sref(target, origin)]) + + +def _write_box_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + for idx in range(cfg.box_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.box_cluster_refs): + target = _box_name((idx * cfg.box_cluster_refs + ref_idx) % cfg.box_cells) + origin = ( + (ref_idx % 6) * 48_000, + (ref_idx // 6) * 48_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.box_cluster_array, (720, 900))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _box_cluster_name(idx), cell_elements) + + +def _write_poly_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + for idx in range(cfg.poly_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.poly_cluster_refs): + target = _poly_name((idx * cfg.poly_cluster_refs + ref_idx) % cfg.poly_cells) + origin = ( + (ref_idx % 10) * 96_000, + (ref_idx // 10) * 96_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.poly_cluster_array, (12_000, 8_500))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _poly_cluster_name(idx), cell_elements) + + +def _top_box_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.box_wrappers): + refs.append(_sref( + _box_wrapper_name(idx), + ((idx % 240) * 240_000, (idx // 240) * 240_000), + )) + + for idx in range(cfg.box_clusters): + refs.append(_sref( + _box_cluster_name(idx), + ((idx % 100) * 800_000, (idx // 100) * 800_000 + 14_000_000), + )) + + for idx in range(cfg.top_direct_box_refs): + target = _box_name(idx % cfg.box_cells) + origin = ( + (idx % 150) * 160_000, + (idx // 150) * 160_000 + 26_000_000, + ) + if cfg.top_box_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_box_array, (1_100, 1_350))) + + return refs + + +def _top_poly_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.poly_wrappers): + refs.append(_sref( + _poly_wrapper_name(idx), + ((idx % 180) * 360_000, (idx // 180) * 360_000 + 44_000_000), + )) + + for idx in range(cfg.poly_clusters): + refs.append(_sref( + _poly_cluster_name(idx), + ((idx % 70) * 1_100_000, (idx // 70) * 1_100_000 + 58_000_000), + )) + + for idx in range(cfg.top_direct_poly_refs): + target = _poly_name(idx % cfg.poly_cells) + origin = ( + (idx % 110) * 420_000, + (idx // 110) * 420_000 + 72_000_000, + ) + if cfg.top_poly_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_poly_array, (16_000, 14_000))) + + return refs + + +def _write_top(stream: Any, cfg: FixturePreset) -> None: + cell_elements: list[elements.Element] = [] + cell_elements.extend(_top_box_refs(cfg)) + cell_elements.extend(_top_poly_refs(cfg)) + _write_struct(stream, 'TOP', cell_elements) + + +def _poly_paths_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.path_stride + 1 + + +def _poly_texts_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.text_stride + 1 + + +def _ref_instances_per_box_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] + return array_refs * array_mult + (cfg.box_cluster_refs - array_refs) + + +def _ref_instances_per_poly_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] + return array_refs * array_mult + (cfg.poly_cluster_refs - array_refs) + + +def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> FixtureManifest: + base = PRESETS[preset] + cfg = _scaled_preset(base, scale) + + flattened_box_placements = ( + cfg.box_wrappers + + cfg.box_clusters * _ref_instances_per_box_cluster(cfg) + + cfg.top_direct_box_refs * cfg.top_box_array[0] * cfg.top_box_array[1] + ) + flattened_poly_placements = ( + cfg.poly_wrappers + + cfg.poly_clusters * _ref_instances_per_poly_cluster(cfg) + + cfg.top_direct_poly_refs * cfg.top_poly_array[0] * cfg.top_poly_array[1] + ) + polygon_layers = max(1, cfg.polygon_layers) + polys_per_layer = (cfg.poly_cells * cfg.polygons_per_cell) // polygon_layers + + return FixtureManifest( + preset=cfg.name, + scale=scale, + gds_path=str(Path(path)), + library_name=f'masque-perf-{cfg.name}', + cells=cfg.box_cells + cfg.poly_cells + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + 1, + refs=( + cfg.box_wrappers + + cfg.poly_wrappers + + cfg.box_clusters * cfg.box_cluster_refs + + cfg.poly_clusters * cfg.poly_cluster_refs + + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + + cfg.top_direct_box_refs + cfg.top_direct_poly_refs + ), + layers=cfg.total_layers, + box_layers=cfg.box_layers, + heavy_box_layers=[[layer, 0] for layer in range(cfg.heavy_box_layers)], + polygon_layers=[[layer, 0] for layer in range(cfg.polygon_layers)], + hierarchical_boxes_per_heavy_layer=cfg.box_cells * cfg.heavy_boxes_per_cell, + hierarchical_boxes_per_regular_layer=cfg.box_cells * cfg.regular_boxes_per_cell, + hierarchical_polygons_total=cfg.poly_cells * cfg.polygons_per_cell, + hierarchical_paths_total=_poly_paths_total(cfg), + hierarchical_texts_total=_poly_texts_total(cfg), + flattened_box_placements=flattened_box_placements, + flattened_poly_placements=flattened_poly_placements, + estimated_flat_boxes_per_heavy_layer=flattened_box_placements * cfg.heavy_boxes_per_cell, + estimated_flat_polygons_per_active_polygon_layer=flattened_poly_placements * polys_per_layer // cfg.poly_cells if cfg.poly_cells else 0, + ) + + +def write_fixture( + path: str | Path, + *, + preset: str, + scale: float = 1.0, + write_manifest: bool = True, + ) -> FixtureManifest: + if preset not in PRESETS: + known = ', '.join(sorted(PRESETS)) + raise KeyError(f'unknown preset {preset!r}; expected one of: {known}') + + manifest = fixture_manifest(path, preset, scale) + cfg = _scaled_preset(PRESETS[preset], scale) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + + with output.open('wb') as stream: + header = klamath.library.FileHeader( + name=manifest.library_name.encode('ASCII'), + user_units_per_db_unit=USER_UNITS_PER_DB_UNIT, + meters_per_db_unit=METERS_PER_DB_UNIT, + ) + header.write(stream) + _write_box_cells(stream, cfg) + _write_poly_cells(stream, cfg) + _write_wrappers(stream, cfg) + _write_box_clusters(stream, cfg) + _write_poly_clusters(stream, cfg) + _write_top(stream, cfg) + klamath.records.ENDLIB.write(stream, None) + + if write_manifest: + manifest_path = output.with_suffix(output.suffix + '.json') + manifest_path.write_text(json.dumps(asdict(manifest), indent=2, sort_keys=True) + '\n') + + return manifest + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Generate synthetic GDS fixtures for GDS reader/writer performance work.') + parser.add_argument( + 'preset', + nargs='?', + default='many_cells', + choices=sorted(PRESETS), + help='Fixture family to generate.', + ) + parser.add_argument( + 'output', + nargs='?', + help='Output .gds path. Defaults to build/gds_perf/.gds', + ) + parser.add_argument( + '--scale', + type=float, + default=1.0, + help='Scale the preset counts down or up while keeping the same shape mix. Default: 1.0', + ) + parser.add_argument( + '--no-manifest', + action='store_true', + help='Do not write the sidecar JSON manifest.', + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + output = Path(args.output) if args.output is not None else Path('build/gds_perf') / f'{args.preset}.gds' + manifest = write_fixture(output, preset=args.preset, scale=args.scale, write_manifest=not args.no_manifest) + print(json.dumps(asdict(manifest), indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/test/test_gdsii_perf.py b/masque/test/test_gdsii_perf.py new file mode 100644 index 0000000..a595fe8 --- /dev/null +++ b/masque/test/test_gdsii_perf.py @@ -0,0 +1,24 @@ +from dataclasses import asdict +import json +from pathlib import Path + +from ..file import gdsii +from ..file.gdsii_perf import fixture_manifest, write_fixture + + +def test_gdsii_perf_fixture_smoke(tmp_path: Path) -> None: + output = tmp_path / 'many_cells.gds' + manifest = write_fixture(output, preset='many_cells', scale=0.002) + expected = fixture_manifest(output, preset='many_cells', scale=0.002) + + assert output.exists() + assert manifest == expected + + sidecar = json.loads(output.with_suffix('.gds.json').read_text()) + assert sidecar == asdict(manifest) + + read_lib, info = gdsii.readfile(output) + assert info['name'] == manifest.library_name + assert len(read_lib) == manifest.cells + assert 'TOP' in read_lib + assert len(read_lib['TOP'].refs) > 0 From d3870662286e6feba03da5058722e6ae3d37d3a0 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:46:53 -0700 Subject: [PATCH 05/36] [Label / Ref / Grid] add raw constructors --- masque/label.py | 16 ++++++++++++++++ masque/ref.py | 20 ++++++++++++++++++++ masque/repetition.py | 16 ++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/masque/label.py b/masque/label.py index 6a2a61f..98a65df 100644 --- a/masque/label.py +++ b/masque/label.py @@ -53,6 +53,22 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + string: str, + *, + offset: NDArray[numpy.float64], + repetition: Repetition | None = None, + annotations: annotations_t | None = None, + ) -> Self: + new = cls.__new__(cls) + new._string = string + new._offset = offset + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> Self: return type(self)( string=self.string, diff --git a/masque/ref.py b/masque/ref.py index 268d8d4..1d71432 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -86,6 +86,26 @@ class Ref( self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + *, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + scale: float, + repetition: Repetition | None, + annotations: annotations_t | None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._rotation = rotation % (2 * pi) + new._scale = scale + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> 'Ref': new = Ref( offset=self.offset.copy(), diff --git a/masque/repetition.py b/masque/repetition.py index 1e12fcd..9e8af26 100644 --- a/masque/repetition.py +++ b/masque/repetition.py @@ -113,6 +113,22 @@ class Grid(Repetition): self.a_count = a_count self.b_count = b_count + @classmethod + def _from_raw( + cls: type[GG], + *, + a_vector: NDArray[numpy.float64], + a_count: int, + b_vector: NDArray[numpy.float64], + b_count: int, + ) -> GG: + new = cls.__new__(cls) + new._a_vector = a_vector + new._b_vector = b_vector + new._a_count = int(a_count) + new._b_count = int(b_count) + return new + @classmethod def aligned( cls: type[GG], From 28562f73f634a62f75a1a7a9043024835308f7f4 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:47:41 -0700 Subject: [PATCH 06/36] [gdsii_arrow] further improvements to speed --- examples/profile_gdsii_readers.py | 71 ++++++++++++--- masque/file/gdsii_arrow.py | 144 ++++++++++++++++++++---------- masque/test/test_gdsii_arrow.py | 81 +++++++++++++++++ 3 files changed, 235 insertions(+), 61 deletions(-) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index c77c589..129a391 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -5,17 +5,18 @@ import importlib import json import time from pathlib import Path +from typing import Any from masque import LibraryError -READERS = { - 'gdsii': ('masque.file.gdsii', 'readfile'), - 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), +READERS: dict[str, tuple[str, tuple[str, ...]]] = { + 'gdsii': ('masque.file.gdsii', ('readfile',)), + 'gdsii_arrow': ('masque.file.gdsii_arrow', ('readfile', 'arrow_import', 'arrow_convert')), } -def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: +def _summarize_library(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: assert hasattr(lib, '__len__') assert hasattr(lib, 'tops') tops = lib.tops() # type: ignore[no-any-return, attr-defined] @@ -34,12 +35,50 @@ def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: objec } +def _summarize_arrow_import(path: Path, elapsed_s: float, arrow_arr: Any) -> dict[str, object]: + libarr = arrow_arr[0] + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': len(arrow_arr), + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + + +def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: + start = time.perf_counter() + + if stage == 'readfile': + lib, info = module.readfile(path) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + if stage == 'arrow_import': + arrow_arr = module._read_to_arrow(path) + elapsed_s = time.perf_counter() - start + return _summarize_arrow_import(path, elapsed_s, arrow_arr) + + if stage == 'arrow_convert': + arrow_arr = module._read_to_arrow(path) + libarr = arrow_arr[0] + start = time.perf_counter() + lib, info = module.read_arrow(libarr) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + raise ValueError(f'Unsupported stage {stage!r}') + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--stage', default='readfile') parser.add_argument('--path', type=Path, required=True) parser.add_argument('--warmup', type=int, default=1) parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--output-json', type=Path) return parser @@ -47,26 +86,32 @@ def main(argv: list[str] | None = None) -> int: parser = build_arg_parser() args = parser.parse_args(argv) - module_name, attr_name = READERS[args.reader] - readfile = getattr(importlib.import_module(module_name), attr_name) + module_name, stages = READERS[args.reader] + if args.stage not in stages: + parser.error(f'reader {args.reader!r} only supports stages: {", ".join(stages)}') + + module = importlib.import_module(module_name) path = args.path.expanduser().resolve() for _ in range(args.warmup): - readfile(path) + _profile_stage(module, args.stage, path) runs = [] for _ in range(args.repeat): - start = time.perf_counter() - lib, info = readfile(path) - elapsed_s = time.perf_counter() - start - runs.append(_summarize(path, elapsed_s, info, lib)) + runs.append(_profile_stage(module, args.stage, path)) - print(json.dumps({ + payload = { 'reader': args.reader, + 'stage': args.stage, 'warmup': args.warmup, 'repeat': args.repeat, 'runs': runs, - }, indent=2, sort_keys=True)) + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(rendered + '\n') + print(rendered) return 0 diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 546d575..f6a7739 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -57,6 +57,8 @@ ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchem clib: Any | None = None +ZERO_OFFSET = numpy.zeros(2) + path_cap_map = { 0: Path.Cap.Flush, @@ -263,6 +265,29 @@ def read_arrow( ) return elem + def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('boundary_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = batches.values.field('vertices').offsets.to_numpy() // 2, + poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(), + poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), + ) + + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: + boundaries = libarr['cells'].values.field('boundary_props') + return dict( + offsets = boundaries.offsets.to_numpy(), + layer_inds = boundaries.values.field('layer').to_numpy(), + vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2, + prop_off = boundaries.values.field('properties').offsets.to_numpy(), + prop_key = boundaries.values.field('properties').values.field('key').to_numpy(), + prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), + ) + rf = libarr['cells'].values.field('refs') refs = dict( offsets = rf.offsets.to_numpy(), @@ -292,10 +317,9 @@ def read_arrow( ) elements = dict( - boundaries = get_geom(libarr, 'boundaries'), + boundary_batches = get_boundary_batches(libarr), + boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), - boxes = get_geom(libarr, 'boxes'), - nodes = get_geom(libarr, 'nodes'), texts = texts, refs = refs, ) @@ -320,7 +344,8 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() - _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) + _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) _grefs_to_mrefs(pat, global_args, elements['refs'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) @@ -366,6 +391,7 @@ def _grefs_to_mrefs( elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] rep_valid = elem['rep_valid'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): @@ -380,10 +406,16 @@ def _grefs_to_mrefs( a_vector = elem_rep_xy0[ee] b_vector = elem_rep_xy1[ee] a_count, b_count = elem_rep_counts[ee] - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + if raw_mode: + rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + else: + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + if raw_mode: + ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + else: + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -406,6 +438,7 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -413,7 +446,10 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - mlabel = Label(string=string, offset=offset, annotations=annotations) + if raw_mode: + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) + else: + mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -439,7 +475,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -453,65 +488,78 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) -def _boundaries_to_polygons( +def _boundary_batches_to_polygons( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: elem_off = elem['offsets'] # which elements belong to each cell - xy_val = elem['xy_arr'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + poly_off = elem['poly_off'] + poly_offsets = elem['poly_offsets'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) # +1 to capture ending location for last elem + elem_vert_off = vert_off[elem_slc] + elem_poly_off = poly_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] + vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + + if vertex_offsets.size == 1: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(poly) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(polys) + + +def _boundary_props_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] layer_inds = elem['layer_inds'] layer_tups = global_args['layer_tups'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element - xy_counts = xy_offs[1:] - xy_offs[:-1] - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - prop_counts = prop_offs[1:] - prop_offs[:-1] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + elem_vert_off = vert_off[elem_slc] + prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - order = numpy.argsort(elem_layer_inds, stable=True) - unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) - - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] - for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): - ee_inds = order[ff:ff + nn] - layer = layer_tups[layer_ind] - propless_mask = prop_counts[ee_inds] == 0 - - poly_count_on_layer = propless_mask.sum() - if poly_count_on_layer == 1: - propless_mask[:] = 0 # Never make a 1-element collection - elif poly_count_on_layer > 1: - propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point - vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) - vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) - - for ii, ee in enumerate(ee_inds[propless_mask]): - vo = vertex_offsets[ii] - vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] - - polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) - pat.shapes[layer].append(polys) - - # Handle single polygons - for ee in ee_inds[~propless_mask]: - layer = layer_tups[elem_layer_inds[ee]] - vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) - pat.shapes[layer].append(poly) + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) #def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 26713e9..b005cd7 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -5,6 +5,7 @@ import pytest pytest.importorskip('pyarrow') +from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -119,7 +120,10 @@ def _make_arrow_test_library() -> Library: leaf = Pattern() leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((2, 0), vertices=[[40, 0], [50, 0], [50, 10], [40, 10]]) leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.polygon((1, 0), vertices=[[80, 0], [90, 0], [90, 10], [80, 10]]) + leaf.polygon((2, 0), vertices=[[60, 0], [70, 0], [70, 10], [60, 10]], annotations={'18': ['leaf-poly-2']}) leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) lib['leaf'] = leaf @@ -178,3 +182,80 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert len(lib) == manifest.cells assert 'TOP' in lib assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 + + +def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + + leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + + boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() + boundary_props = cells.field('boundary_props')[leaf_index].as_py() + + assert len(boundary_batches) == 2 + assert len(boundary_props) == 2 + + batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} + assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] + assert len(batch_by_layer[(1, 0)]['vertices']) == 16 + assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] + assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + + props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} + assert sorted(props_by_layer) == [(1, 0), (2, 0)] + assert props_by_layer[(1, 0)]['properties'][0]['value'] == 'leaf-poly' + assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' + + +def test_raw_ref_grid_label_constructors_match_public() -> None: + raw_grid = Grid._from_raw( + a_vector=numpy.array([20, 0]), + a_count=3, + b_vector=numpy.array([0, 30]), + b_count=2, + ) + public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) + assert raw_grid == public_grid + + raw_ref = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=raw_grid, + annotations={'12': ['child-ref']}, + ) + public_ref = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=public_grid, + annotations={'12': ['child-ref']}, + ) + assert raw_ref == public_ref + assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + + raw_label = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations={'10': ['leaf-label']}, + ) + public_label = Label( + 'LEAF', + offset=(3, 4), + annotations={'10': ['leaf-label']}, + ) + assert raw_label == public_label + assert numpy.array_equal(raw_label.get_bounds_single(), public_label.get_bounds_single()) From 7130d26112f9bb1cfe5b8ceb83e73d68d18d5f58 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:56:42 -0700 Subject: [PATCH 07/36] enable annotations=None by default --- masque/label.py | 2 +- masque/ref.py | 2 +- masque/test/test_gdsii_arrow.py | 34 ++++++++++++++++++++++++++++++++- masque/utils/comparisons.py | 10 ++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/masque/label.py b/masque/label.py index 98a65df..d220fee 100644 --- a/masque/label.py +++ b/masque/label.py @@ -66,7 +66,7 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl new._string = string new._offset = offset new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> Self: diff --git a/masque/ref.py b/masque/ref.py index 1d71432..0cc911f 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -103,7 +103,7 @@ class Ref( new._scale = scale new._mirrored = mirrored new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> 'Ref': diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index b005cd7..cec48c8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -19,7 +19,7 @@ if not gdsii_arrow.is_available(): def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: - if annotations is None: + if not annotations: return None return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) @@ -228,6 +228,25 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_ref_empty = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + public_ref_empty = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + assert raw_ref_empty.annotations is None + assert raw_ref_empty == public_ref_empty + raw_ref = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, @@ -247,6 +266,19 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: assert raw_ref == public_ref assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + raw_label_empty = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations=None, + ) + public_label_empty = Label( + 'LEAF', + offset=(3, 4), + annotations=None, + ) + assert raw_label_empty.annotations is None + assert raw_label_empty == public_label_empty + raw_label = Label._from_raw( 'LEAF', offset=numpy.array([3, 4]), diff --git a/masque/utils/comparisons.py b/masque/utils/comparisons.py index ffb7206..bb2dfee 100644 --- a/masque/utils/comparisons.py +++ b/masque/utils/comparisons.py @@ -9,7 +9,15 @@ def annotation2key(aaa: int | float | str) -> tuple[bool, Any]: return (isinstance(aaa, str), aaa) +def _normalized_annotations(annotations: annotations_t) -> annotations_t: + if not annotations: + return None + return annotations + + def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is not None elif bb is None: # noqa: RET505 @@ -36,6 +44,8 @@ def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: def annotations_eq(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is None elif bb is None: # noqa: RET505 From ec7803156508b4ff1f3e9a869fa139f83b4d82e3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:17 -0700 Subject: [PATCH 08/36] [RectCollection] add a RectCollection shape --- masque/__init__.py | 1 + masque/file/gdsii.py | 16 +- masque/shapes/__init__.py | 1 + masque/shapes/rect_collection.py | 256 ++++++++++++++++++++++++++++ masque/test/test_rect_collection.py | 70 ++++++++ 5 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 masque/shapes/rect_collection.py create mode 100644 masque/test/test_rect_collection.py diff --git a/masque/__init__.py b/masque/__init__.py index 8e13bb9..b6cc5e9 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -42,6 +42,7 @@ from .error import ( from .shapes import ( Shape as Shape, Polygon as Polygon, + RectCollection as RectCollection, Path as Path, Circle as Circle, Arc as Arc, diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 116fa07..323dabd 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -37,7 +37,7 @@ from klamath import records from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path +from ..shapes import Polygon, Path, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -466,6 +466,20 @@ def _shapes_to_elements( properties=properties, ) elements.append(path) + elif isinstance(shape, RectCollection): + for rect in shape.rects: + xy_closed = numpy.empty((5, 2), dtype=numpy.int32) + xy_closed[0] = rint_cast((rect[0], rect[1])) + xy_closed[1] = rint_cast((rect[0], rect[3])) + xy_closed[2] = rint_cast((rect[2], rect[3])) + xy_closed[3] = rint_cast((rect[2], rect[1])) + xy_closed[4] = xy_closed[0] + boundary = klamath.elements.Boundary( + layer=(layer, data_type), + xy=xy_closed, + properties=properties, + ) + elements.append(boundary) elif isinstance(shape, Polygon): polygon = shape xy_closed = numpy.empty((polygon.vertices.shape[0] + 1, 2), dtype=numpy.int32) diff --git a/masque/shapes/__init__.py b/masque/shapes/__init__.py index fd66c59..ac3a14b 100644 --- a/masque/shapes/__init__.py +++ b/masque/shapes/__init__.py @@ -11,6 +11,7 @@ from .shape import ( from .polygon import Polygon as Polygon from .poly_collection import PolyCollection as PolyCollection +from .rect_collection import RectCollection as RectCollection from .circle import Circle as Circle from .ellipse import Ellipse as Ellipse from .arc import Arc as Arc diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py new file mode 100644 index 0000000..96becfd --- /dev/null +++ b/masque/shapes/rect_collection.py @@ -0,0 +1,256 @@ +from typing import Any, cast, Self +from collections.abc import Iterator +import copy +import functools + +import numpy +from numpy import pi +from numpy.typing import NDArray, ArrayLike + +from . import Shape, normalized_shape_tuple +from .polygon import Polygon +from ..error import PatternError +from ..repetition import Repetition +from ..utils import annotations_lt, annotations_eq, rep2key, annotations_t + + +def _normalize_rects(rects: ArrayLike) -> NDArray[numpy.float64]: + arr = numpy.asarray(rects, dtype=float) + if arr.ndim != 2 or arr.shape[1] != 4: + raise PatternError('Rectangles must be an Nx4 array of [xmin, ymin, xmax, ymax]') + if numpy.any(arr[:, 0] > arr[:, 2]) or numpy.any(arr[:, 1] > arr[:, 3]): + raise PatternError('Rectangles must satisfy xmin <= xmax and ymin <= ymax') + if arr.shape[0] <= 1: + return arr + order = numpy.lexsort((arr[:, 3], arr[:, 2], arr[:, 1], arr[:, 0])) + return arr[order] + + +def _renormalize_rects_in_place(rects: NDArray[numpy.float64]) -> None: + x0 = numpy.minimum(rects[:, 0], rects[:, 2]) + x1 = numpy.maximum(rects[:, 0], rects[:, 2]) + y0 = numpy.minimum(rects[:, 1], rects[:, 3]) + y1 = numpy.maximum(rects[:, 1], rects[:, 3]) + rects[:, 0] = x0 + rects[:, 1] = y0 + rects[:, 2] = x1 + rects[:, 3] = y1 + + +@functools.total_ordering +class RectCollection(Shape): + """ + A collection of axis-aligned rectangles, stored as an Nx4 array of + `[xmin, ymin, xmax, ymax]` rows. + """ + __slots__ = ( + '_rects', + '_repetition', '_annotations', + ) + + _rects: NDArray[numpy.float64] + + @property + def rects(self) -> NDArray[numpy.float64]: + return self._rects + + @rects.setter + def rects(self, val: ArrayLike) -> None: + self._rects = _normalize_rects(val) + + @property + def offset(self) -> NDArray[numpy.float64]: + return numpy.zeros(2) + + @offset.setter + def offset(self, val: ArrayLike) -> None: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + + def set_offset(self, val: ArrayLike) -> Self: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + return self + + def translate(self, offset: ArrayLike) -> Self: + delta = numpy.asarray(offset, dtype=float).reshape(2) + self._rects[:, [0, 2]] += delta[0] + self._rects[:, [1, 3]] += delta[1] + return self + + def __init__( + self, + rects: ArrayLike, + *, + offset: ArrayLike = (0.0, 0.0), + rotation: float = 0.0, + repetition: Repetition | None = None, + annotations: annotations_t = None, + raw: bool = False, + ) -> None: + if raw: + assert isinstance(rects, numpy.ndarray) + self._rects = rects + self._repetition = repetition + self._annotations = annotations + else: + self.rects = rects + self.repetition = repetition + self.annotations = annotations + if rotation: + self.rotate(rotation) + if numpy.any(offset): + self.translate(offset) + + @classmethod + def _from_raw( + cls, + *, + rects: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._rects = rects + new._repetition = repetition + new._annotations = annotations + return new + + @property + def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: + for rect in self._rects: + xmin, ymin, xmax, ymax = rect + yield numpy.array([ + [xmin, ymin], + [xmin, ymax], + [xmax, ymax], + [xmax, ymin], + ], dtype=float) + + def __deepcopy__(self, memo: dict | None = None) -> Self: + memo = {} if memo is None else memo + new = copy.copy(self) + new._rects = self._rects.copy() + new._repetition = copy.deepcopy(self._repetition, memo) + new._annotations = copy.deepcopy(self._annotations) + return new + + def _sorted_rects(self) -> NDArray[numpy.float64]: + if self._rects.shape[0] <= 1: + return self._rects + order = numpy.lexsort((self._rects[:, 3], self._rects[:, 2], self._rects[:, 1], self._rects[:, 0])) + return self._rects[order] + + def __eq__(self, other: Any) -> bool: + return ( + type(self) is type(other) + and numpy.array_equal(self._sorted_rects(), other._sorted_rects()) + and self.repetition == other.repetition + and annotations_eq(self.annotations, other.annotations) + ) + + def __lt__(self, other: Shape) -> bool: + if type(self) is not type(other): + if repr(type(self)) != repr(type(other)): + return repr(type(self)) < repr(type(other)) + return id(type(self)) < id(type(other)) + + other = cast('RectCollection', other) + self_rects = self._sorted_rects() + other_rects = other._sorted_rects() + if not numpy.array_equal(self_rects, other_rects): + min_len = min(self_rects.shape[0], other_rects.shape[0]) + eq_mask = self_rects[:min_len] != other_rects[:min_len] + eq_lt = self_rects[:min_len] < other_rects[:min_len] + eq_lt_masked = eq_lt[eq_mask] + if eq_lt_masked.size > 0: + return bool(eq_lt_masked.flat[0]) + return self_rects.shape[0] < other_rects.shape[0] + if self.repetition != other.repetition: + return rep2key(self.repetition) < rep2key(other.repetition) + return annotations_lt(self.annotations, other.annotations) + + def to_polygons( + self, + num_vertices: int | None = None, # unused # noqa: ARG002 + max_arclen: float | None = None, # unused # noqa: ARG002 + ) -> list[Polygon]: + return [ + Polygon( + vertices=vertices, + repetition=copy.deepcopy(self.repetition), + annotations=copy.deepcopy(self.annotations), + ) + for vertices in self.polygon_vertices + ] + + def get_bounds_single(self) -> NDArray[numpy.float64] | None: + if self._rects.size == 0: + return None + mins = self._rects[:, :2].min(axis=0) + maxs = self._rects[:, 2:].max(axis=0) + return numpy.vstack((mins, maxs)) + + def rotate(self, theta: float) -> Self: + quarter_turns = int(numpy.rint(theta / (pi / 2))) + if not numpy.isclose(theta, quarter_turns * (pi / 2)): + raise PatternError('RectCollection only supports Manhattan rotations') + turns = quarter_turns % 4 + if turns == 0 or self._rects.size == 0: + return self + + corners = numpy.stack(( + self._rects[:, [0, 1]], + self._rects[:, [0, 3]], + self._rects[:, [2, 3]], + self._rects[:, [2, 1]], + ), axis=1) + flat = corners.reshape(-1, 2) + if turns == 1: + rotated = numpy.column_stack((-flat[:, 1], flat[:, 0])) + elif turns == 2: + rotated = -flat + else: + rotated = numpy.column_stack((flat[:, 1], -flat[:, 0])) + corners = rotated.reshape(corners.shape) + self._rects[:, 0] = corners[:, :, 0].min(axis=1) + self._rects[:, 1] = corners[:, :, 1].min(axis=1) + self._rects[:, 2] = corners[:, :, 0].max(axis=1) + self._rects[:, 3] = corners[:, :, 1].max(axis=1) + return self + + def mirror(self, axis: int = 0) -> Self: + if axis not in (0, 1): + raise PatternError('Axis must be 0 or 1') + if axis == 0: + self._rects[:, [1, 3]] *= -1 + else: + self._rects[:, [0, 2]] *= -1 + _renormalize_rects_in_place(self._rects) + return self + + def scale_by(self, c: float) -> Self: + self._rects *= c + _renormalize_rects_in_place(self._rects) + return self + + def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + rects = self._sorted_rects() + centers = 0.5 * (rects[:, :2] + rects[:, 2:]) + offset = centers.mean(axis=0) + zeroed = rects.copy() + zeroed[:, [0, 2]] -= offset[0] + zeroed[:, [1, 3]] -= offset[1] + normed = zeroed / norm_value + return ( + (type(self), normed.data.tobytes()), + (offset, 1.0, 0.0, False), + lambda: RectCollection(rects=normed * norm_value), + ) + + def __repr__(self) -> str: + if self._rects.size == 0: + return '' + centers = 0.5 * (self._rects[:, :2] + self._rects[:, 2:]) + centroid = centers.mean(axis=0) + return f'' diff --git a/masque/test/test_rect_collection.py b/masque/test/test_rect_collection.py new file mode 100644 index 0000000..449f4fa --- /dev/null +++ b/masque/test/test_rect_collection.py @@ -0,0 +1,70 @@ +import copy + +import numpy +import pytest +from numpy.testing import assert_allclose, assert_equal + +from ..error import PatternError +from ..shapes import Polygon, RectCollection + + +def test_rect_collection_init_and_to_polygons() -> None: + rects = RectCollection([[10, 10, 12, 12], [0, 0, 5, 5]]) + assert_equal(rects.rects, [[0, 0, 5, 5], [10, 10, 12, 12]]) + + polys = rects.to_polygons() + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_equal(polys[0].vertices, [[0, 0], [0, 5], [5, 5], [5, 0]]) + + +def test_rect_collection_rejects_invalid_rects() -> None: + with pytest.raises(PatternError): + RectCollection([[0, 0, 1]]) + with pytest.raises(PatternError): + RectCollection([[5, 0, 1, 2]]) + with pytest.raises(PatternError): + RectCollection([[0, 5, 1, 2]]) + + +def test_rect_collection_raw_constructor_matches_public() -> None: + raw = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'1': ['rects']}, + ) + public = RectCollection( + [[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'1': ['rects']}, + ) + assert raw == public + assert_equal(raw.get_bounds_single(), [[0, 0], [12, 12]]) + + +def test_rect_collection_manhattan_transforms() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + + mirrored = copy.deepcopy(rects).mirror(1) + assert_equal(mirrored.rects, [[-2, 0, 0, 4], [-12, 20, -10, 22]]) + + scaled = copy.deepcopy(rects).scale_by(-2) + assert_equal(scaled.rects, [[-4, -8, 0, 0], [-24, -44, -20, -40]]) + + rotated = copy.deepcopy(rects).rotate(numpy.pi / 2) + assert_equal(rotated.rects, [[-4, 0, 0, 2], [-22, 10, -20, 12]]) + + +def test_rect_collection_non_manhattan_rotation_raises() -> None: + rects = RectCollection([[0, 0, 2, 4]]) + with pytest.raises(PatternError, match='Manhattan rotations'): + rects.rotate(numpy.pi / 4) + + +def test_rect_collection_normalized_form_rebuild_is_independent() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + _intrinsic, extrinsic, rebuild = rects.normalized_form(2) + + clone = rebuild() + clone.rects[:] = [[1, 1, 2, 2], [3, 3, 4, 4]] + + assert_allclose(extrinsic[0], [6, 11.5]) + assert_equal(rects.rects, [[0, 0, 2, 4], [10, 20, 12, 22]]) From 27f4f0e86ec0421998266e6bd093169b6807ac4e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:39 -0700 Subject: [PATCH 09/36] [Polygon / PolyCollection] add raw constructors --- masque/shapes/poly_collection.py | 16 ++++++++++++++++ masque/shapes/polygon.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 6c23da7..0b5543f 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -119,6 +119,22 @@ class PolyCollection(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertex_lists: NDArray[numpy.float64], + vertex_offsets: NDArray[numpy.integer[Any]], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertex_lists = vertex_lists + new._vertex_offsets = vertex_offsets + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index a743de2..42c2f78 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -131,6 +131,20 @@ class Polygon(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Polygon': memo = {} if memo is None else memo new = copy.copy(self) From 3f63599abe174b21ba053a9065445a26621ef6d5 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:52 -0700 Subject: [PATCH 10/36] [gdsii_arrow] more performance work --- masque/file/gdsii_arrow.py | 356 ++++++++++++++++++++++++----- masque/shapes/poly_collection.py | 13 +- masque/test/test_file_roundtrip.py | 29 ++- masque/test/test_gdsii_arrow.py | 264 ++++++++++++++++++--- 4 files changed, 562 insertions(+), 100 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index f6a7739..c5e7681 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -45,7 +45,7 @@ from pyarrow.cffi import ffi from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, PolyCollection +from ..shapes import Polygon, Path, PolyCollection, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -276,6 +276,15 @@ def read_arrow( poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), ) + def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('rect_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_off = batches.values.field('rects').offsets.to_numpy() // 4, + ) + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: boundaries = libarr['cells'].values.field('boundary_props') return dict( @@ -288,22 +297,46 @@ def read_arrow( prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), ) - rf = libarr['cells'].values.field('refs') - refs = dict( - offsets = rf.offsets.to_numpy(), - targets = rf.values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), - invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), - scale = rf.values.field('mag').fill_null(1).to_numpy(), - rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), - rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), - rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), - prop_off = rf.values.field('properties').offsets.to_numpy(), - prop_key = rf.values.field('properties').values.field('key').to_numpy(), - prop_val = rf.values.field('properties').values.field('value').to_pylist(), - ) + def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + prop_off = values.field('properties').offsets.to_numpy(), + prop_key = values.field('properties').values.field('key').to_numpy(), + prop_val = values.field('properties').values.field('value').to_pylist(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem txt = libarr['cells'].values.field('texts') texts = dict( @@ -317,11 +350,15 @@ def read_arrow( ) elements = dict( + srefs = get_refs(libarr, 'srefs', has_repetition=False), + arefs = get_refs(libarr, 'arefs', has_repetition=True), + sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False), + aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True), + rect_batches = get_rect_batches(libarr), boundary_batches = get_boundary_batches(libarr), boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), texts = texts, - refs = refs, ) paths = libarr['cells'].values.field('paths') @@ -344,10 +381,14 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() + _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) - _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _srefs_to_mrefs(pat, global_args, elements['srefs'], cc) + _arefs_to_mrefs(pat, global_args, elements['arefs'], cc) + _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc) + _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) mlib[name] = pat @@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: return library_info -def _grefs_to_mrefs( +def _srefs_to_mrefs( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: cell_names = global_args['cell_names'] - elem_off = elem['offsets'] # which elements belong to each cell - xy = elem['xy'] - prop_key = elem['prop_key'] - prop_val = elem['prop_val'] - targets = elem['targets'] - + elem_off = elem['offsets'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - elem_targets = targets[elem_slc][:elem_count] - elem_xy = xy[elem_slc][:elem_count] - elem_invert_y = elem['invert_y'][elem_slc][:elem_count] - elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] - elem_scale = elem['scale'][elem_slc][:elem_count] - elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] - elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] - elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] - rep_valid = elem['rep_valid'][elem_slc][:elem_count] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] raw_mode = global_args['raw_mode'] + _append_plain_refs_sorted( + pat=pat, + cell_names=cell_names, + elem_targets=elem_targets, + elem_xy=elem_xy, + elem_invert_y=elem_invert_y, + elem_angle_rad=elem_angle_rad, + elem_scale=elem_scale, + raw_mode=raw_mode, + ) + + +def _append_plain_refs_sorted( + *, + pat: Pattern, + cell_names: list[str], + elem_targets: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.integer[Any]], + elem_invert_y: NDArray[numpy.bool_ | numpy.bool], + elem_angle_rad: NDArray[numpy.floating[Any]], + elem_scale: NDArray[numpy.floating[Any]], + raw_mode: bool, + ) -> None: + elem_count = len(elem_targets) + if elem_count == 0: + return + + make_ref = Ref._from_raw if raw_mode else Ref + + target_start = 0 + while target_start < elem_count: + target_id = elem_targets[target_start] + target_stop = target_start + 1 + while target_stop < elem_count and elem_targets[target_stop] == target_id: + target_stop += 1 + + append_refs = pat.refs[cell_names[target_id]].extend + append_refs( + make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=None, + ) + for ee in range(target_start, target_stop) + ) + + target_start = target_stop + + +def _arefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + elem_xy0 = elem['xy0'][start:stop] + elem_xy1 = elem['xy1'][start:stop] + elem_counts = elem['counts'][start:stop] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + if len(elem_targets) == 0: + return + + target = None + append_ref: Callable[[Ref], Any] | None = None + for ee in range(len(elem_targets)): + if target != elem_targets[ee]: + target = elem_targets[ee] + append_ref = pat.refs[cell_names[target]].append + assert append_ref is not None + a_count, b_count = elem_counts[ee] + append_ref(make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=None, + )) + + +def _sref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): - target = cell_names[elem_targets[ee]] - offset = elem_xy[ee] - mirr = elem_invert_y[ee] - rot = elem_angle_rad[ee] - mag = elem_scale[ee] - - rep: None | Grid = None - if rep_valid[ee]: - a_vector = elem_rep_xy0[ee] - b_vector = elem_rep_xy1[ee] - a_count, b_count = elem_rep_counts[ee] - if raw_mode: - rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - else: - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - else: - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - pat.refs[target].append(ref) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) + + +def _aref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] + elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] + elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + for ee in range(elem_count): + a_count, b_count = elem_counts[ee] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) def _texts_to_labels( @@ -520,16 +712,53 @@ def _boundary_batches_to_polygons( for bb in range(batch_count): layer = layer_tups[elem_layer_inds[bb]] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] - vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=None) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(poly) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(polys) +def _rect_batches_to_rectcollections( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + rect_arr = elem['rect_arr'] + rect_off = elem['rect_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) + elem_rect_off = rect_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] + if raw_mode: + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) + else: + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + pat.shapes[layer].append(rect_collection) + + def _boundary_props_to_polygons( pat: Pattern, global_args: dict[str, Any], @@ -558,7 +787,10 @@ def _boundary_props_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) pat.shapes[layer].append(poly) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 0b5543f..f90c29a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -34,7 +34,7 @@ class PolyCollection(Shape): _vertex_lists: NDArray[numpy.float64] """ 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """ - _vertex_offsets: NDArray[numpy.intp] + _vertex_offsets: NDArray[numpy.integer[Any]] """ 1D NDArray specifying the starting offset for each polygon """ @property @@ -45,7 +45,7 @@ class PolyCollection(Shape): return self._vertex_lists @property - def vertex_offsets(self) -> NDArray[numpy.intp]: + def vertex_offsets(self) -> NDArray[numpy.integer[Any]]: """ Starting offset (in `vertex_lists`) for each polygon """ @@ -63,7 +63,7 @@ class PolyCollection(Shape): chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]), strict=True, ): - yield slice(ii, ff) + yield slice(int(ii), int(ff)) @property def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: @@ -105,6 +105,7 @@ class PolyCollection(Shape): if raw: assert isinstance(vertex_lists, numpy.ndarray) assert isinstance(vertex_offsets, numpy.ndarray) + assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) self._vertex_lists = vertex_lists self._vertex_offsets = vertex_offsets self._repetition = repetition @@ -148,7 +149,7 @@ class PolyCollection(Shape): return ( type(self) is type(other) and numpy.array_equal(self._vertex_lists, other._vertex_lists) - and numpy.array_equal(self._vertex_offsets, other._vertex_offsets) + and numpy.array_equal(self.vertex_offsets, other.vertex_offsets) and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -231,11 +232,11 @@ class PolyCollection(Shape): # TODO: normalize mirroring? - return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()), + return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()), (offset, scale / norm_value, rotation, False), lambda: PolyCollection( vertex_lists=rotated_vertices * norm_value, - vertex_offsets=self._vertex_offsets.copy(), + vertex_offsets=self.vertex_offsets.copy(), ), ) diff --git a/masque/test/test_file_roundtrip.py b/masque/test/test_file_roundtrip.py index 2cfb0d1..283a863 100644 --- a/masque/test/test_file_roundtrip.py +++ b/masque/test/test_file_roundtrip.py @@ -5,7 +5,7 @@ from numpy.testing import assert_allclose from ..pattern import Pattern from ..library import Library -from ..shapes import Path as MPath, Circle, Polygon +from ..shapes import Path as MPath, Circle, Polygon, RectCollection from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: @@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: assert poly.repetition is not None assert isinstance(poly.repetition, Grid) assert poly.repetition.a_count == 5 + + +def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None: + from ..file import gdsii + + lib = Library() + pat = Pattern() + pat.shapes[(5, 0)].append( + RectCollection( + rects=[[0, 0, 10, 5], [20, -5, 30, 10]], + annotations={'1': ['rects']}, + ) + ) + lib['rects'] = pat + + gds_file = tmp_path / 'rect_collection.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + read_lib, _ = gdsii.readfile(gds_file) + polys = read_lib['rects'].shapes[(5, 0)] + + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]]) + assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]]) + assert polys[0].annotations == {'1': ['rects']} + assert polys[1].annotations == {'1': ['rects']} diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index cec48c8..2ef4d83 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -9,7 +9,7 @@ from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid -from ..shapes import Path as MPath +from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection from ..file import gdsii, gdsii_arrow from ..file.gdsii_perf import write_fixture @@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: return tuple(tuple(row.tolist()) for row in arr) +def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int) + rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])] + rev = arr[::-1] + rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0])) + return min(rows) + + def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: if isinstance(shape, MPath): cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) @@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...] keys.append(( 'polygon', layer, - _coord_key(poly.vertices), + _canonical_polygon_key(poly.vertices), _coord_key(poly.offset), _annotations_key(poly.annotations), )) return keys -def _ref_key(target: str, ref: object) -> tuple[object, ...]: - repetition = None - if ref.repetition is not None: - repetition = ( - _coord_key(ref.repetition.a_vector), - int(ref.repetition.a_count), - _coord_key(ref.repetition.b_vector), - int(ref.repetition.b_count), - ) - return ( - target, - _coord_key(ref.offset), - round(float(ref.rotation), 8), - round(float(ref.scale), 8), - bool(ref.mirrored), - repetition, - _annotations_key(ref.annotations), - ) +def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]: + keys = [] + for transform in ref.as_transforms(): + keys.append(( + target, + _coord_key(transform[:2]), + round(float(transform[2]), 8), + round(float(transform[4]), 8), + bool(int(round(float(transform[3])))), + _annotations_key(ref.annotations), + )) + return keys def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: @@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]: for shape in shapes: shape_keys.extend(_shape_key(shape, layer)) - ref_keys = [ - _ref_key(target, ref) - for target, refs in pattern.refs.items() - for ref in refs - ] + ref_keys: list[tuple[object, ...]] = [] + for target, refs in pattern.refs.items(): + for ref in refs: + ref_keys.extend(_ref_keys(target, ref)) label_keys = [ _label_key(layer, label) @@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library: ) lib['sibling'] = sibling + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(20, 0)) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2)) + fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2)) + fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']}) + fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2), + annotations={'20': ['fanout-aref']}) + lib['fanout'] = fanout + top = Pattern() top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.ref('fanout', offset=(250, -75)) top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) lib['top'] = top @@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 +def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, _ = gdsii.readfile(gds_file) + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + assert _library_summary(arrow_lib) == _library_summary(canonical_lib) + + decoded_ref = arrow_lib['top'].refs['leaf'][0] + assert decoded_ref.repetition is None + + +def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'plain_srefs.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + fanout = arrow_lib['fanout'] + + plain_leaf_refs = [ + ref + for ref in fanout.refs['leaf'] + if ref.annotations is None and ref.repetition is None + ] + assert len(plain_leaf_refs) == 2 + assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs) + + +def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref_schema.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + srefs = cells.field('srefs')[top_index].as_py() + arefs = cells.field('arefs')[top_index].as_py() + + assert len(srefs) == 1 + assert len(arefs) == 0 + assert cell_names[srefs[0]['target']] == 'leaf' + + def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_batches.gds' @@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + rect_batches = cells.field('rect_batches')[leaf_index].as_py() boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() boundary_props = cells.field('boundary_props')[leaf_index].as_py() - assert len(boundary_batches) == 2 + assert len(rect_batches) == 2 + assert len(boundary_batches) == 0 assert len(boundary_props) == 2 - batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} - assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] - assert len(batch_by_layer[(1, 0)]['vertices']) == 16 - assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] - assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches} + assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10] + assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10] props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} assert sorted(props_by_layer) == [(1, 0), (2, 0)] @@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' +def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None: + lib = Library() + top = Pattern() + top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]])) + top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]]) + top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]]) + lib['top'] = top + + gds_file = tmp_path / 'arrow_rect_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + rect_batches = cells.field('rect_batches')[top_index].as_py() + boundary_batches = cells.field('boundary_batches')[top_index].as_py() + + assert len(rect_batches) == 1 + assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0) + assert rect_batches[0]['rects'] == [ + 0, 0, 10, 10, + 20, 0, 30, 10, + 40, 0, 50, 10, + 60, 0, 70, 10, + ] + + assert len(boundary_batches) == 1 + assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0) + assert boundary_batches[0]['vertex_offsets'] == [0, 3] + + +def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_ref_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout') + + srefs = cells.field('srefs')[fanout_index].as_py() + arefs = cells.field('arefs')[fanout_index].as_py() + sref_props = cells.field('sref_props')[fanout_index].as_py() + aref_props = cells.field('aref_props')[fanout_index].as_py() + + sref_target_ids = [entry['target'] for entry in srefs] + sref_targets = [cell_names[target] for target in sref_target_ids] + assert sorted(sref_targets) == ['child', 'leaf', 'leaf'] + assert sref_target_ids == sorted(sref_target_ids) + sref_by_target = {} + for entry in srefs: + sref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in sref_by_target['child']] == [True] + assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1]) + assert len(sref_by_target['leaf']) == 2 + + aref_target_ids = [entry['target'] for entry in arefs] + aref_targets = [cell_names[target] for target in aref_target_ids] + assert sorted(aref_targets) == ['child', 'leaf', 'leaf'] + assert aref_target_ids == sorted(aref_target_ids) + aref_by_target = {} + for entry in arefs: + aref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in aref_by_target['child']] == [True] + assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2]) + assert len(aref_by_target['leaf']) == 2 + + assert len(sref_props) == 1 + assert cell_names[sref_props[0]['target']] == 'leaf' + assert sref_props[0]['properties'][0]['value'] == 'fanout-sref' + + assert len(aref_props) == 1 + assert cell_names[aref_props[0]['target']] == 'child' + assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), @@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_poly = Polygon._from_raw( + vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]), + annotations={'1': ['poly']}, + ) + public_poly = Polygon( + vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], + annotations={'1': ['poly']}, + ) + assert raw_poly == public_poly + + raw_poly_collection = PolyCollection._from_raw( + vertex_lists=numpy.array([ + [0.0, 0.0], [2.0, 0.0], [2.0, 2.0], + [10.0, 10.0], [12.0, 10.0], [12.0, 12.0], + ]), + vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32), + annotations={'2': ['pc']}, + ) + public_poly_collection = PolyCollection( + vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]], + vertex_offsets=[0, 3], + annotations={'2': ['pc']}, + ) + assert raw_poly_collection == public_poly_collection + assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)] + + raw_rect_collection = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'3': ['rects']}, + ) + public_rect_collection = RectCollection( + rects=[[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'3': ['rects']}, + ) + assert raw_rect_collection == public_rect_collection + raw_ref_empty = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, From 37d462525cea9272de2e8e3bfe602c49611e9358 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:55:30 -0700 Subject: [PATCH 11/36] [shapes] move to per-shape purpose-built _from_raw constructors --- masque/file/gdsii.py | 44 ++++++++----- masque/file/gdsii_arrow.py | 27 ++++++-- masque/shapes/arc.py | 49 ++++++++------ masque/shapes/circle.py | 32 +++++---- masque/shapes/ellipse.py | 38 ++++++----- masque/shapes/path.py | 47 ++++++++------ masque/shapes/poly_collection.py | 18 ++---- masque/shapes/polygon.py | 13 +--- masque/shapes/rect_collection.py | 13 +--- masque/shapes/text.py | 49 ++++++++------ masque/test/test_raw_constructors.py | 97 ++++++++++++++++++++++++++++ 11 files changed, 288 insertions(+), 139 deletions(-) create mode 100644 masque/test/test_raw_constructors.py diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 323dabd..1d8c3d1 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -323,26 +323,40 @@ def _gpath_to_mpath(gpath: klamath.library.Path, raw_mode: bool) -> tuple[layer_ else: raise PatternError(f'Unrecognized path type: {gpath.path_type}') - mpath = Path( - vertices=gpath.xy.astype(float), - width=gpath.width, - cap=cap, - offset=numpy.zeros(2), - annotations=_properties_to_annotations(gpath.properties), - raw=raw_mode, - ) + vertices = gpath.xy.astype(float) + annotations = _properties_to_annotations(gpath.properties) + cap_extensions = None if cap == Path.Cap.SquareCustom: - mpath.cap_extensions = gpath.extension + cap_extensions = numpy.asarray(gpath.extension, dtype=float) + + if raw_mode: + mpath = Path._from_raw( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + mpath = Path( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + offset=numpy.zeros(2), + annotations=annotations, + ) return gpath.layer, mpath def _boundary_to_polygon(boundary: klamath.library.Boundary, raw_mode: bool) -> tuple[layer_t, Polygon]: - return boundary.layer, Polygon( - vertices=boundary.xy[:-1].astype(float), - offset=numpy.zeros(2), - annotations=_properties_to_annotations(boundary.properties), - raw=raw_mode, - ) + vertices = boundary.xy[:-1].astype(float) + annotations = _properties_to_annotations(boundary.properties) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=numpy.zeros(2), annotations=annotations) + return boundary.layer, poly def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.Reference]: diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index c5e7681..58493c1 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -680,8 +680,23 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, - width=width, cap=cap,cap_extensions=cap_extensions) + if raw_mode: + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + path = Path( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + offset=ZERO_OFFSET, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -718,13 +733,13 @@ def _boundary_batches_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=None) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(poly) else: if raw_mode: polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(polys) @@ -755,7 +770,7 @@ def _rect_batches_to_rectcollections( if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(rect_collection) @@ -790,7 +805,7 @@ def _boundary_props_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=annotations) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index 00c5714..9d5f65d 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -159,27 +159,36 @@ class Arc(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(angles, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._angles = angles - self._width = width - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.angles = angles - self.width = width - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.angles = angles + self.width = width + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + angles: NDArray[numpy.float64], + width: float, + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Arc': + new = cls.__new__(cls) + new._radii = radii + new._angles = angles + new._width = width + new._offset = offset + new._rotation = rotation % (2 * pi) + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Arc': memo = {} if memo is None else memo diff --git a/masque/shapes/circle.py b/masque/shapes/circle.py index 19f3f2b..d7591db 100644 --- a/masque/shapes/circle.py +++ b/masque/shapes/circle.py @@ -50,19 +50,27 @@ class Circle(PositionableImpl, Shape): offset: ArrayLike = (0.0, 0.0), repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._radius = radius - self._offset = offset - self._repetition = repetition - self._annotations = annotations - else: - self.radius = radius - self.offset = offset - self.repetition = repetition - self.annotations = annotations + self.radius = radius + self.offset = offset + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radius: float, + offset: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Circle': + new = cls.__new__(cls) + new._radius = radius + new._offset = offset + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Circle': memo = {} if memo is None else memo diff --git a/masque/shapes/ellipse.py b/masque/shapes/ellipse.py index 38799da..52a3297 100644 --- a/masque/shapes/ellipse.py +++ b/masque/shapes/ellipse.py @@ -95,22 +95,30 @@ class Ellipse(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._radii = radii + new._offset = offset + new._rotation = rotation % pi + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo diff --git a/masque/shapes/path.py b/masque/shapes/path.py index 7038309..a1e04af 100644 --- a/masque/shapes/path.py +++ b/masque/shapes/path.py @@ -201,34 +201,43 @@ class Path(Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: self._cap_extensions = None # Since .cap setter might access it - if raw: - assert isinstance(vertices, numpy.ndarray) - assert isinstance(cap_extensions, numpy.ndarray) or cap_extensions is None - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - self._width = width - self._cap = cap - self._cap_extensions = cap_extensions + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations + self._cap = cap + if cap == PathCap.SquareCustom and cap_extensions is None: + self._cap_extensions = numpy.zeros(2) else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations - self._cap = cap - if cap == PathCap.SquareCustom and cap_extensions is None: - self._cap_extensions = numpy.zeros(2) - else: - self.cap_extensions = cap_extensions - self.width = width + self.cap_extensions = cap_extensions + self.width = width if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + width: float, + cap: PathCap, + cap_extensions: NDArray[numpy.float64] | None = None, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._width = width + new._cap = cap + new._cap_extensions = cap_extensions + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Path': memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index f90c29a..f1c840a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -100,21 +100,11 @@ class PolyCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertex_lists, numpy.ndarray) - assert isinstance(vertex_offsets, numpy.ndarray) - assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) - self._vertex_lists = vertex_lists - self._vertex_offsets = vertex_offsets - self._repetition = repetition - self._annotations = annotations - else: - self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) - self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) - self.repetition = repetition - self.annotations = annotations + self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) + self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index 42c2f78..06e5c2b 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -115,17 +115,10 @@ class Polygon(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertices, numpy.ndarray) - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py index 96becfd..eaf028f 100644 --- a/masque/shapes/rect_collection.py +++ b/masque/shapes/rect_collection.py @@ -86,17 +86,10 @@ class RectCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(rects, numpy.ndarray) - self._rects = rects - self._repetition = repetition - self._annotations = annotations - else: - self.rects = rects - self.repetition = repetition - self.annotations = annotations + self.rects = rects + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/text.py b/masque/shapes/text.py index 5047dc4..c078879 100644 --- a/masque/shapes/text.py +++ b/masque/shapes/text.py @@ -73,27 +73,40 @@ class Text(PositionableImpl, RotatableImpl, Shape): mirrored: bool = False, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._offset = offset - self._string = string - self._height = height - self._rotation = rotation - self._mirrored = mirrored - self._repetition = repetition - self._annotations = annotations - else: - self.offset = offset - self.string = string - self.height = height - self.rotation = rotation - self.mirrored = mirrored - self.repetition = repetition - self.annotations = annotations + self.offset = offset + self.string = string + self.height = height + self.rotation = rotation + self.mirrored = mirrored + self.repetition = repetition + self.annotations = annotations self.font_path = font_path + @classmethod + def _from_raw( + cls, + *, + string: str, + height: float, + font_path: str, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._string = string + new._height = height + new._rotation = rotation % (2 * pi) + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations + new.font_path = font_path + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/test/test_raw_constructors.py b/masque/test/test_raw_constructors.py new file mode 100644 index 0000000..2f86ba0 --- /dev/null +++ b/masque/test/test_raw_constructors.py @@ -0,0 +1,97 @@ +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from ..shapes import Arc, Circle, Ellipse, Path, Text + + +def test_circle_raw_constructor_matches_public() -> None: + raw = Circle._from_raw( + radius=5.0, + offset=numpy.array([1.0, 2.0]), + annotations={'1': ['circle']}, + ) + public = Circle( + radius=5.0, + offset=(1.0, 2.0), + annotations={'1': ['circle']}, + ) + assert raw == public + + +def test_ellipse_raw_constructor_matches_public() -> None: + raw = Ellipse._from_raw( + radii=numpy.array([3.0, 5.0]), + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + public = Ellipse( + radii=(3.0, 5.0), + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + assert raw == public + + +def test_arc_raw_constructor_matches_public() -> None: + raw = Arc._from_raw( + radii=numpy.array([10.0, 6.0]), + angles=numpy.array([0.0, pi / 2]), + width=2.0, + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + public = Arc( + radii=(10.0, 6.0), + angles=(0.0, pi / 2), + width=2.0, + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + assert raw == public + + +def test_path_raw_constructor_matches_public() -> None: + raw = Path._from_raw( + vertices=numpy.array([[0.0, 0.0], [10.0, 0.0], [10.0, 5.0]]), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=numpy.array([1.0, 3.0]), + annotations={'4': ['path']}, + ) + public = Path( + vertices=((0.0, 0.0), (10.0, 0.0), (10.0, 5.0)), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=(1.0, 3.0), + annotations={'4': ['path']}, + ) + assert raw == public + assert raw.cap_extensions is not None + assert_allclose(raw.cap_extensions, [1.0, 3.0]) + + +def test_text_raw_constructor_matches_public() -> None: + raw = Text._from_raw( + string='RAW', + height=12.0, + font_path='font.otf', + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + public = Text( + string='RAW', + height=12.0, + font_path='font.otf', + offset=(1.0, 2.0), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + assert raw == public From 5c1050b0fff906ec8e32c820239dadf8b74d1905 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:31 -0700 Subject: [PATCH 12/36] [arrow] add lazy arrow reader --- examples/profile_gdsii_readers.py | 12 + masque/file/gdsii_arrow.py | 95 ++- masque/file/gdsii_lazy_arrow.py | 960 +++++++++++++++++++++++++++ masque/test/test_gdsii_arrow.py | 12 + masque/test/test_gdsii_lazy_arrow.py | 174 +++++ 5 files changed, 1237 insertions(+), 16 deletions(-) create mode 100644 masque/file/gdsii_lazy_arrow.py create mode 100644 masque/test/test_gdsii_lazy_arrow.py diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index 129a391..fb7c99e 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -56,6 +56,18 @@ def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: return _summarize_library(path, elapsed_s, info, lib) if stage == 'arrow_import': + if hasattr(module, 'readfile_arrow'): + libarr, _info = module.readfile_arrow(path) + elapsed_s = time.perf_counter() - start + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': 1, + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + arrow_arr = module._read_to_arrow(path) elapsed_s = time.perf_counter() - start return _summarize_arrow_import(path, elapsed_s, arrow_arr) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 58493c1..b97005b 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -53,7 +53,20 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +ffi.cdef( + """ + void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); + void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + void read_cells_bytes( + uint8_t* data, + size_t size, + uint64_t* ranges, + size_t range_count, + struct ArrowArray* array, + struct ArrowSchema* schema + ); + """ +) clib: Any | None = None @@ -189,12 +202,36 @@ def _read_to_arrow( ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + +def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + return pyarrow.Array._import_from_c(iptr_array, iptr_schema) - return arrow_arr + +def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + + +def _read_selected_cells_to_arrow( + buffer: bytes | mmap.mmap | memoryview, + ranges: NDArray[numpy.uint64], + ) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) + cranges = ffi.from_buffer('uint64_t[]', flat_ranges) + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) def readfile( @@ -211,6 +248,9 @@ def readfile( filename: Filename to save to. *args: passed to `read()` **kwargs: passed to `read()` + + For callers that can consume Arrow directly, prefer `readfile_arrow()` + to skip Python `Pattern` construction entirely. """ arrow_arr = _read_to_arrow(filename) assert len(arrow_arr) == 1 @@ -220,6 +260,28 @@ def readfile( return results +def readfile_arrow( + filename: str | pathlib.Path, + ) -> tuple[pyarrow.StructScalar, dict[str, Any]]: + """ + Read a GDSII file into the native Arrow representation without converting + it into `masque.Library` / `Pattern` objects. + + This is the lowest-overhead public read path exposed by this module. + + Args: + filename: Filename to read. + + Returns: + - Arrow struct scalar for the library payload + - dict of GDSII library info + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + libarr = arrow_arr[0] + return libarr, _read_header(libarr) + + def read_arrow( libarr: pyarrow.Array, raw_mode: bool = True, @@ -247,7 +309,7 @@ def read_arrow( library_info = _read_header(libarr) layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) - layer_tups = [tuple(pair) for pair in layer_names_np] + layer_tups = [(int(pair[0]), int(pair[1])) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() @@ -379,7 +441,7 @@ def read_arrow( mlib = Library() for cc in range(len(libarr['cells'])): - name = cell_names[cell_ids[cc]] + name = cell_names[int(cell_ids[cc])] pat = Pattern() _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) @@ -459,7 +521,7 @@ def _append_plain_refs_sorted( target_start = 0 while target_start < elem_count: - target_id = elem_targets[target_start] + target_id = int(elem_targets[target_start]) target_stop = target_start + 1 while target_stop < elem_count and elem_targets[target_stop] == target_id: target_stop += 1 @@ -513,9 +575,10 @@ def _arefs_to_mrefs( target = None append_ref: Callable[[Ref], Any] | None = None for ee in range(len(elem_targets)): - if target != elem_targets[ee]: - target = elem_targets[ee] - append_ref = pat.refs[cell_names[target]].append + target_id = int(elem_targets[ee]) + if target != target_id: + target = target_id + append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] append_ref(make_ref( @@ -564,7 +627,7 @@ def _sref_props_to_mrefs( repetition=None, annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _aref_props_to_mrefs( @@ -608,7 +671,7 @@ def _aref_props_to_mrefs( repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _texts_to_labels( @@ -633,7 +696,7 @@ def _texts_to_labels( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] offset = elem_xy[ee] string = elem_strings[ee] @@ -669,7 +732,7 @@ def _gpaths_to_mpaths( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] cap_int = elem_path_types[ee] @@ -725,7 +788,7 @@ def _boundary_batches_to_polygons( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] @@ -765,7 +828,7 @@ def _rect_batches_to_rectcollections( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) @@ -799,7 +862,7 @@ def _boundary_props_to_polygons( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) if raw_mode: diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py new file mode 100644 index 0000000..9a03960 --- /dev/null +++ b/masque/file/gdsii_lazy_arrow.py @@ -0,0 +1,960 @@ +""" +Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. + +This module is intentionally separate from `gdsii_arrow` so the eager read path +keeps its current behavior and performance profile. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping, Sequence +import copy +import gzip +import logging +import mmap +import pathlib + +import numpy +from numpy.typing import NDArray +import pyarrow +import klamath + +from . import gdsii, gdsii_arrow +from .utils import is_gzipped, tmpfile +from ..error import LibraryError +from ..library import ILibrary, ILibraryView, Library, LibraryView, dangling_mode_t +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _StructRange: + start: int + end: int + + +@dataclass +class _SourceBuffer: + path: pathlib.Path + data: bytes | mmap.mmap + handle: IO[bytes] | None = None + + def raw_slice(self, start: int, end: int) -> bytes: + return self.data[start:end] + + +@dataclass +class _ScanRefs: + offsets: NDArray[numpy.integer[Any]] + targets: NDArray[numpy.integer[Any]] + xy: NDArray[numpy.int32] + xy0: NDArray[numpy.int32] + xy1: NDArray[numpy.int32] + counts: NDArray[numpy.int64] + invert_y: NDArray[numpy.bool_ | numpy.bool] + angle_rad: NDArray[numpy.floating[Any]] + scale: NDArray[numpy.floating[Any]] + + +@dataclass(frozen=True) +class _CellScan: + cell_id: int + struct_range: _StructRange + ref_start: int + ref_stop: int + children: set[str] + + +@dataclass +class _ScanPayload: + libarr: pyarrow.StructScalar + library_info: dict[str, Any] + cell_names: list[str] + cell_order: list[str] + cells: dict[str, _CellScan] + refs: _ScanRefs + + +@dataclass +class _SourceLayer: + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + layer_index: int + source_name: str + + +def is_available() -> bool: + return gdsii_arrow.is_available() + + +def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: + return gdsii_arrow._read_header(libarr) + + +def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: + if is_gzipped(path): + with gzip.open(path, mode='rb') as stream: + data = stream.read() + return _SourceBuffer(path=path, data=data) + + handle = path.open(mode='rb', buffering=0) + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + return _SourceBuffer(path=path, data=mapped, handle=handle) + + +def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: + library_info = _read_header(libarr) + cell_names = libarr['cell_names'].as_py() + + cells = libarr['cells'] + cell_values = cells.values + cell_ids = cell_values.field('id').to_numpy() + struct_starts = cell_values.field('struct_start_offset').to_numpy() + struct_ends = cell_values.field('struct_end_offset').to_numpy() + + refs = cell_values.field('refs') + ref_values = refs.values + ref_offsets = refs.offsets.to_numpy() + targets = ref_values.field('target').to_numpy() + xy = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) + xy0 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) + xy1 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) + counts = gdsii_arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) + invert_y = ref_values.field('invert_y').to_numpy(zero_copy_only=False) + angle_rad = ref_values.field('angle_rad').to_numpy() + scale = ref_values.field('scale').to_numpy() + + ref_payload = _ScanRefs( + offsets=ref_offsets, + targets=targets, + xy=xy, + xy0=xy0, + xy1=xy1, + counts=counts, + invert_y=invert_y, + angle_rad=angle_rad, + scale=scale, + ) + + cell_order = [cell_names[int(cell_id)] for cell_id in cell_ids] + cell_scan: dict[str, _CellScan] = {} + for cc, name in enumerate(cell_order): + ref_start = int(ref_offsets[cc]) + ref_stop = int(ref_offsets[cc + 1]) + children = { + cell_names[int(target)] + for target in targets[ref_start:ref_stop] + } + cell_scan[name] = _CellScan( + cell_id=int(cell_ids[cc]), + struct_range=_StructRange(int(struct_starts[cc]), int(struct_ends[cc])), + ref_start=ref_start, + ref_stop=ref_stop, + children=children, + ) + + return _ScanPayload( + libarr=libarr, + library_info=library_info, + cell_names=cell_names, + cell_order=cell_order, + cells=cell_scan, + refs=ref_payload, + ) + + +def _pattern_children(pat: Pattern) -> set[str]: + return {child for child, refs in pat.refs.items() if child is not None and refs} + + +def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: + if not pat.refs: + return pat + pat.refs = map_targets(pat.refs, remap) + return pat + + +def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: + if isinstance(source, ILibraryView): + return source + return LibraryView(source) + + +def _source_order(source: ILibraryView) -> list[str]: + if isinstance(source, ArrowLibrary): + return list(source.source_order()) + return list(source.keys()) + + +def _make_ref_rows( + xy: NDArray[numpy.integer[Any]], + angle_rad: NDArray[numpy.floating[Any]], + invert_y: NDArray[numpy.bool_ | numpy.bool], + scale: NDArray[numpy.floating[Any]], + ) -> NDArray[numpy.float64]: + rows = numpy.empty((len(xy), 5), dtype=float) + rows[:, :2] = xy + rows[:, 2] = angle_rad + rows[:, 3] = invert_y.astype(float) + rows[:, 4] = scale + return rows + + +def _expand_aref_row( + xy: NDArray[numpy.integer[Any]], + xy0: NDArray[numpy.integer[Any]], + xy1: NDArray[numpy.integer[Any]], + counts: NDArray[numpy.integer[Any]], + angle_rad: float, + invert_y: bool, + scale: float, + ) -> NDArray[numpy.float64]: + a_count = int(counts[0]) + b_count = int(counts[1]) + aa, bb = numpy.meshgrid(numpy.arange(a_count), numpy.arange(b_count), indexing='ij') + displacements = aa.reshape(-1, 1) * xy0[None, :] + bb.reshape(-1, 1) * xy1[None, :] + rows = numpy.empty((displacements.shape[0], 5), dtype=float) + rows[:, :2] = xy + displacements + rows[:, 2] = angle_rad + rows[:, 3] = float(invert_y) + rows[:, 4] = scale + return rows + + +class ArrowLibrary(ILibraryView): + """ + Read-only library backed by the native lazy Arrow scan schema. + + Materializing a cell via `__getitem__` caches a real `Pattern` for that cell. + Cached cells are treated as edited for future writes from this module. + """ + + path: pathlib.Path + library_info: dict[str, Any] + + def __init__( + self, + *, + path: pathlib.Path, + payload: _ScanPayload, + source: _SourceBuffer, + ) -> None: + self.path = path + self.library_info = payload.library_info + self._payload = payload + self._source = source + self._cache: dict[str, Pattern] = {} + + @classmethod + def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: + path = pathlib.Path(filename).expanduser().resolve() + source = _open_source_buffer(path) + scan_arr = gdsii_arrow._scan_buffer_to_arrow(source.data) + assert len(scan_arr) == 1 + payload = _extract_scan_payload(scan_arr[0]) + return cls(path=path, payload=payload, source=source) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._payload.cell_order) + + def __len__(self) -> int: + return len(self._payload.cell_order) + + def __contains__(self, key: object) -> bool: + return key in self._payload.cells + + def source_order(self) -> tuple[str, ...]: + return tuple(self._payload.cell_order) + + def raw_struct_bytes(self, name: str) -> bytes: + struct_range = self._payload.cells[name].struct_range + return self._source.raw_slice(struct_range.start, struct_range.end) + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = self._materialize_patterns(names, persist=persist) + return LibraryView(mats) + + def _materialize_patterns( + self, + names: Sequence[str], + *, + persist: bool, + ) -> dict[str, Pattern]: + ordered_names = list(dict.fromkeys(names)) + missing = [name for name in ordered_names if name not in self._payload.cells] + if missing: + raise KeyError(missing[0]) + + materialized: dict[str, Pattern] = {} + uncached = [name for name in ordered_names if name not in self._cache] + if uncached: + ranges = numpy.asarray( + [ + [ + self._payload.cells[name].struct_range.start, + self._payload.cells[name].struct_range.end, + ] + for name in uncached + ], + dtype=numpy.uint64, + ) + arrow_arr = gdsii_arrow._read_selected_cells_to_arrow(self._source.data, ranges) + assert len(arrow_arr) == 1 + selected_lib, _info = gdsii_arrow.read_arrow(arrow_arr[0]) + for name in uncached: + pat = selected_lib[name] + materialized[name] = pat + if persist: + self._cache[name] = pat + + for name in ordered_names: + if name in self._cache: + materialized[name] = self._cache[name] + return materialized + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + return self._materialize_patterns((name,), persist=persist)[name] + + def _raw_children(self, name: str) -> set[str]: + return set(self._payload.cells[name].children) + + def _collect_raw_transforms(self, cell: _CellScan, target_id: int) -> list[NDArray[numpy.float64]]: + refs = self._payload.refs + start = cell.ref_start + stop = cell.ref_stop + if stop <= start: + return [] + + targets = refs.targets[start:stop] + mask = targets == target_id + if not mask.any(): + return [] + + rows: list[NDArray[numpy.float64]] = [] + counts = refs.counts[start:stop] + unit_mask = mask & (counts[:, 0] == 1) & (counts[:, 1] == 1) + if unit_mask.any(): + rows.append(_make_ref_rows( + refs.xy[start:stop][unit_mask], + refs.angle_rad[start:stop][unit_mask], + refs.invert_y[start:stop][unit_mask], + refs.scale[start:stop][unit_mask], + )) + + aref_indices = numpy.nonzero(mask & ~unit_mask)[0] + for idx in aref_indices: + abs_idx = start + int(idx) + rows.append(_expand_aref_row( + xy=refs.xy[abs_idx], + xy0=refs.xy0[abs_idx], + xy1=refs.xy1[abs_idx], + counts=refs.counts[abs_idx], + angle_rad=float(refs.angle_rad[abs_idx]), + invert_y=bool(refs.invert_y[abs_idx]), + scale=float(refs.scale[abs_idx]), + )) + return rows + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._payload.cell_order: + if name in self._cache: + graph[name] = _pattern_children(self._cache[name]) + else: + graph[name] = self._raw_children(name) + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + graph = self.child_graph(dangling='ignore') + names = set(graph) + not_toplevel: set[str] = set() + for children in graph.values(): + not_toplevel |= children + return list(names - not_toplevel) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + target_id = self._payload.cells.get(name) + for parent in parent_graph.get(name, set()): + if parent in self._cache: + for ref in self._cache[parent].refs.get(name, []): + instances[parent].append(ref.as_transforms()) + continue + + if target_id is None or parent not in self._payload.cells: + continue + rows = self._collect_raw_transforms(self._payload.cells[parent], target_id.cell_id) + if rows: + instances[parent].extend(rows) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + full_path = (parent,) + path + result[full_path] = instances + return result + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + view = _coerce_library_view(source) + source_order = _source_order(view) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + + for name in source_order: + visible = name + if visible in self._entries or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._entries or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return rename_map + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + remap = lambda target: None if target is None else self._effective_target(layer, target) + pat = _remap_pattern_targets(source_pat, remap) + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = _pattern_children(entry) + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + +def readfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + lib = ArrowLibrary.from_file(filename) + return lib, lib.library_info + + +def load_libraryfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + return readfile(filename) + + +def _get_write_info( + library: Mapping[str, Pattern] | ILibraryView, + *, + meters_per_unit: float | None, + logical_units_per_unit: float | None, + library_name: str | None, + ) -> tuple[float, float, str]: + if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: + return meters_per_unit, logical_units_per_unit, library_name + + infos: list[dict[str, Any]] = [] + if isinstance(library, ArrowLibrary): + infos.append(library.library_info) + elif isinstance(library, OverlayLibrary): + for layer in library._layers: + if isinstance(layer.library, ArrowLibrary): + infos.append(layer.library.library_info) + + if infos: + unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} + if len(unit_pairs) > 1: + raise LibraryError('Merged lazy GDS sources must have identical units before writing') + info = infos[0] + meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit + logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit + name = info['name'] if library_name is None else library_name + return meters, logical, name + + if meters_per_unit is None or logical_units_per_unit is None or library_name is None: + raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') + return meters_per_unit, logical_units_per_unit, library_name + + +def _can_copy_arrow_cell(library: ArrowLibrary, name: str) -> bool: + return name not in library._cache + + +def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: + layer = library._layers[entry.layer_index] + if not isinstance(layer.library, ArrowLibrary): + return False + if name != entry.source_name: + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(library._effective_target(layer, child) == child for child in children) + + +def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: + elements: list[klamath.elements.Element] = [] + elements += gdsii._shapes_to_elements(pat.shapes) + elements += gdsii._labels_to_texts(pat.labels) + elements += gdsii._mrefs_to_grefs(pat.refs) + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) + + +def write( + library: Mapping[str, Pattern] | ILibraryView, + stream: IO[bytes], + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + meters_per_unit, logical_units_per_unit, library_name = _get_write_info( + library, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + + header = klamath.library.FileHeader( + name=library_name.encode('ASCII'), + user_units_per_db_unit=logical_units_per_unit, + meters_per_db_unit=meters_per_unit, + ) + header.write(stream) + + if isinstance(library, ArrowLibrary): + for name in library.source_order(): + if _can_copy_arrow_cell(library, name): + stream.write(library.raw_struct_bytes(name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + if isinstance(library, OverlayLibrary): + for name in library.source_order(): + entry = library._entries[name] + if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + layer = library._layers[entry.layer_index] + assert isinstance(layer.library, ArrowLibrary) + stream.write(layer.library.raw_struct_bytes(entry.source_name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) + + +def writefile( + library: Mapping[str, Pattern] | ILibraryView, + filename: str | pathlib.Path, + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + path = pathlib.Path(filename) + + with tmpfile(path) as base_stream: + streams: tuple[Any, ...] = (base_stream,) + if path.suffix == '.gz': + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + streams = (stream,) + streams + else: + stream = base_stream + + try: + write( + library, + stream, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + finally: + for ss in streams: + ss.close() diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 2ef4d83..f3f4f6a 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_native.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == manifest.library_name + assert libarr['lib_name'].as_py() == manifest.library_name + assert len(libarr['cells']) == manifest.cells + assert 0 < len(libarr['layers']) <= manifest.layers + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py new file mode 100644 index 0000000..61a99af --- /dev/null +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -0,0 +1,174 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..file import gdsii, gdsii_lazy_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_lazy_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _make_small_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 5], [0, 5]]) + lib['leaf'] = leaf + + mid = Pattern() + mid.ref('leaf', offset=(10, 20)) + mid.ref('leaf', offset=(40, 0), repetition=Grid(a_vector=(12, 0), a_count=2, b_vector=(0, 9), b_count=2)) + lib['mid'] = mid + + top = Pattern() + top.ref('mid', offset=(100, 200)) + lib['top'] = top + return lib + + +def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_lazy.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert lib.top() == 'TOP' + assert 'TOP' in lib.child_graph(dangling='ignore') + + +def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: + gds_file = tmp_path / 'refs.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-refs') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + local = lib.find_refs_local('leaf') + assert set(local) == {'mid'} + assert sum(arr.shape[0] for arr in local['mid']) == 5 + + global_refs = lib.find_refs_global('leaf') + assert {path for path in global_refs} == {('top', 'mid', 'leaf')} + assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 + + +def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: + base_a = Library() + leaf_a = Pattern() + leaf_a.polygon((1, 0), vertices=[[0, 0], [8, 0], [8, 8], [0, 8]]) + base_a['leaf'] = leaf_a + top_a = Pattern() + top_a.ref('leaf', offset=(0, 0)) + base_a['top_a'] = top_a + + base_b = Library() + leaf_b = Pattern() + leaf_b.polygon((2, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + base_b['leaf'] = leaf_b + top_b = Pattern() + top_b.ref('leaf', offset=(20, 30)) + base_b['top_b'] = top_b + + gds_a = tmp_path / 'a.gds' + gds_b = tmp_path / 'b.gds' + gdsii.writefile(base_a, gds_a, meters_per_unit=1e-9, library_name='overlay') + gdsii.writefile(base_b, gds_b, meters_per_unit=1e-9, library_name='overlay') + + lib_a, _ = gdsii_lazy_arrow.readfile(gds_a) + lib_b, _ = gdsii_lazy_arrow.readfile(gds_b) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib_a) + rename_map = overlay.add_source(lib_b, rename_theirs=lambda lib, name: lib.get_name(name)) + renamed_leaf = rename_map['leaf'] + + assert rename_map == {'leaf': renamed_leaf} + assert renamed_leaf != 'leaf' + assert len(lib_a._cache) == 0 + assert len(lib_b._cache) == 0 + + overlay.move_references('leaf', renamed_leaf) + + out_file = tmp_path / 'overlay_out.gds' + gdsii_lazy_arrow.writefile(overlay, out_file) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf', renamed_leaf, 'top_a', 'top_b'} + assert 'top_b' in roundtrip + assert list(roundtrip['top_b'].refs.keys()) == [renamed_leaf] + + +def test_gdsii_writer_accepts_overlay_library(tmp_path: Path) -> None: + gds_file = tmp_path / 'overlay_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-src') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib) + overlay.rename('leaf', 'leaf_copy', move_references=True) + + out_file = tmp_path / 'overlay_via_eager_writer.gds' + gdsii.writefile( + overlay, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf_copy', 'mid', 'top'} + assert list(roundtrip['mid'].refs.keys()) == ['leaf_copy'] + + +def test_svg_writer_uses_detached_materialized_copy(tmp_path: Path) -> None: + pytest.importorskip('svgwrite') + from ..file import svg + from ..shapes import Path as MPath + + gds_file = tmp_path / 'svg_source.gds' + src = _make_small_library() + src['top'].path((3, 0), vertices=[[0, 0], [0, 20]], width=4) + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='svg-src') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + top_pat = lib['top'] + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) + + svg_path = tmp_path / 'lazy.svg' + svg.writefile(lib, 'top', str(svg_path)) + + assert svg_path.exists() + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) From 27e1c23c334c87b2e89d01c133c48d0f54b2ac25 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:45 -0700 Subject: [PATCH 13/36] [svg] avoid mutating the original library --- masque/file/svg.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/masque/file/svg.py b/masque/file/svg.py index 621bcdb..772aa39 100644 --- a/masque/file/svg.py +++ b/masque/file/svg.py @@ -45,6 +45,10 @@ def _make_svg_ids(names: Mapping[str, Pattern]) -> dict[str, str]: return svg_ids +def _detached_library(library: Mapping[str, Pattern]) -> dict[str, Pattern]: + return {name: pat.deepcopy() for name, pat in library.items()} + + def writefile( library: Mapping[str, Pattern], top: str, @@ -53,13 +57,12 @@ def writefile( annotate_ports: bool = False, ) -> None: """ - Write a Pattern to an SVG file, by first calling .polygonize() on it + Write a Pattern to an SVG file, by first calling .polygonize() on a detached + materialized copy to change the shapes into polygons, and then writing patterns as SVG groups (, inside ), polygons as paths (), and refs as elements. - Note that this function modifies the Pattern. - If `custom_attributes` is `True`, a non-standard `pattern_layer` attribute is written to the relevant elements. @@ -71,19 +74,21 @@ def writefile( prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. custom_attributes: Whether to write non-standard `pattern_layer` attribute to the SVG elements. annotate_ports: If True, draw an arrow for each port (similar to `Pattern.visualize(..., ports=True)`). """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize pattern pattern.polygonize() - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) @@ -96,10 +101,10 @@ def writefile( # Create file svg = svgwrite.Drawing(filename, profile='full', viewBox=viewbox_string, debug=(not custom_attributes)) - svg_ids = _make_svg_ids(library) + svg_ids = _make_svg_ids(detached) # Now create a group for each pattern and add in any Boundary and Use elements - for name, pat in library.items(): + for name, pat in detached.items(): svg_group = svg.g(id=svg_ids[name], fill='blue', stroke='red') for layer, shapes in pat.shapes.items(): @@ -158,21 +163,21 @@ def writefile_inverted( box and drawing the polygons with reverse vertex order inside it, all within one `` element. - Note that this function modifies the Pattern. - If you want pattern polygonized with non-default arguments, just call `pattern.polygonize()` prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize and flatten pattern - pattern.polygonize().flatten(library) + pattern.polygonize().flatten(detached) - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) From 9e8a8d6a2216efcdf5b05c1fc5d8482a9297877b Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 21:34:10 -0700 Subject: [PATCH 14/36] [gdsii_arrow] remove non-raw-mode arrow option; fix gzip wrapper --- masque/file/gdsii_arrow.py | 115 ++++++++------------------- masque/test/test_gdsii_arrow.py | 39 +++++++++ masque/test/test_gdsii_lazy_arrow.py | 85 ++++++++++++++++++++ 3 files changed, 158 insertions(+), 81 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index b97005b..a38dee5 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -27,7 +27,6 @@ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable from importlib.machinery import EXTENSION_SUFFIXES import importlib.util -import io import mmap import logging import os @@ -35,11 +34,11 @@ import pathlib import gzip import string import sys +import tempfile from pprint import pformat import numpy from numpy.typing import ArrayLike, NDArray -from numpy.testing import assert_equal import pyarrow from pyarrow.cffi import ffi @@ -70,8 +69,6 @@ ffi.cdef( clib: Any | None = None -ZERO_OFFSET = numpy.zeros(2) - path_cap_map = { 0: Path.Cap.Flush, @@ -195,13 +192,22 @@ def _read_annotations( def _read_to_arrow( filename: str | pathlib.Path, - *args, - **kwargs, ) -> pyarrow.Array: path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + if is_gzipped(path): + with gzip.open(path, mode='rb') as src: + data = src.read() + with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream: + tmp_stream.write(data) + tmp_name = tmp_stream.name + try: + _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + finally: + pathlib.Path(tmp_name).unlink(missing_ok=True) + else: + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) return _import_arrow_array(ptr_array, ptr_schema) @@ -236,18 +242,14 @@ def _read_selected_cells_to_arrow( def readfile( filename: str | pathlib.Path, - *args, - **kwargs, ) -> tuple[Library, dict[str, Any]]: """ - Wrapper for `read()` that takes a filename or path instead of a stream. + Read a GDSII file from a path into `masque.Library` / `Pattern` objects. Will automatically decompress gzipped files. Args: - filename: Filename to save to. - *args: passed to `read()` - **kwargs: passed to `read()` + filename: Filename to read. For callers that can consume Arrow directly, prefer `readfile_arrow()` to skip Python `Pattern` construction entirely. @@ -284,7 +286,6 @@ def readfile_arrow( def read_arrow( libarr: pyarrow.Array, - raw_mode: bool = True, ) -> tuple[Library, dict[str, Any]]: """ # TODO check GDSII file for cycles! @@ -299,8 +300,7 @@ def read_arrow( per database unit Args: - stream: Stream to read from. - raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + libarr: Arrow library payload as returned by `readfile_arrow()`. Returns: - dict of pattern_name:Patterns generated from GDSII structures @@ -436,7 +436,6 @@ def read_arrow( global_args = dict( cell_names = cell_names, layer_tups = layer_tups, - raw_mode = raw_mode, ) mlib = Library() @@ -488,7 +487,6 @@ def _srefs_to_mrefs( elem_invert_y = elem['invert_y'][start:stop] elem_angle_rad = elem['angle_rad'][start:stop] elem_scale = elem['scale'][start:stop] - raw_mode = global_args['raw_mode'] _append_plain_refs_sorted( pat=pat, @@ -498,7 +496,6 @@ def _srefs_to_mrefs( elem_invert_y=elem_invert_y, elem_angle_rad=elem_angle_rad, elem_scale=elem_scale, - raw_mode=raw_mode, ) @@ -511,14 +508,11 @@ def _append_plain_refs_sorted( elem_invert_y: NDArray[numpy.bool_ | numpy.bool], elem_angle_rad: NDArray[numpy.floating[Any]], elem_scale: NDArray[numpy.floating[Any]], - raw_mode: bool, ) -> None: elem_count = len(elem_targets) if elem_count == 0: return - make_ref = Ref._from_raw if raw_mode else Ref - target_start = 0 while target_start < elem_count: target_id = int(elem_targets[target_start]) @@ -528,7 +522,7 @@ def _append_plain_refs_sorted( append_refs = pat.refs[cell_names[target_id]].extend append_refs( - make_ref( + Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -564,10 +558,6 @@ def _arefs_to_mrefs( elem_xy0 = elem['xy0'][start:stop] elem_xy1 = elem['xy1'][start:stop] elem_counts = elem['counts'][start:stop] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid if len(elem_targets) == 0: return @@ -581,12 +571,12 @@ def _arefs_to_mrefs( append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] - append_ref(make_ref( + append_ref(Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=None, )) @@ -613,13 +603,10 @@ def _sref_props_to_mrefs( elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -655,20 +642,16 @@ def _aref_props_to_mrefs( elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid for ee in range(elem_count): a_count, b_count = elem_counts[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) pat.refs[cell_names[int(elem_targets[ee])]].append(ref) @@ -693,7 +676,6 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] @@ -701,10 +683,7 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) - else: - mlabel = Label(string=string, offset=offset, annotations=annotations) + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -730,7 +709,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] @@ -743,23 +721,13 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - path = Path._from_raw( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - annotations=annotations, - ) - else: - path = Path( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - offset=ZERO_OFFSET, - annotations=annotations, - ) + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -786,23 +754,16 @@ def _boundary_batches_to_polygons( elem_poly_off = poly_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=None) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) + poly = Polygon._from_raw(vertices=vertices, annotations=None) pat.shapes[layer].append(poly) else: - if raw_mode: - polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) - else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) pat.shapes[layer].append(polys) @@ -826,14 +787,10 @@ def _rect_batches_to_rectcollections( elem_rect_off = rect_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] - if raw_mode: - rect_collection = RectCollection._from_raw(rects=rects, annotations=None) - else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) pat.shapes[layer].append(rect_collection) @@ -860,15 +817,11 @@ def _boundary_props_to_polygons( prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=annotations) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index f3f4f6a..660818c 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_matches_gdsii_readfile_for_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_native.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -199,6 +211,33 @@ def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> No assert 0 < len(libarr['layers']) <= manifest.layers +def test_gdsii_arrow_readfile_arrow_reads_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'native_payload.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == 'masque-klamath' + assert libarr['lib_name'].as_py() == 'masque-klamath' + assert len(libarr['cells']) == len(lib) + assert len(libarr['layers']) > 0 + + +def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'removed_raw_mode.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, _ = gdsii_arrow.readfile_arrow(gds_file) + + with pytest.raises(TypeError): + gdsii_arrow.readfile(gds_file, raw_mode=False) + + with pytest.raises(TypeError): + gdsii_arrow.read_arrow(libarr, raw_mode=False) + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 61a99af..b30b058 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -34,6 +34,78 @@ def _make_small_library() -> Library: return lib +def _make_complex_ref_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]]) + lib['leaf'] = leaf + + child = Pattern() + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25) + lib['child'] = child + + sibling = Pattern() + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + ) + lib['sibling'] = sibling + + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref( + 'child', + offset=(40, 0), + mirrored=True, + rotation=numpy.pi / 4, + scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2), + ) + lib['fanout'] = fanout + + top = Pattern() + top.ref('child', offset=(500, 600)) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi) + top.ref('fanout', offset=(250, -75)) + lib['top'] = top + + return lib + + +def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: + arr = numpy.asarray(values, dtype=float) + arr = numpy.atleast_2d(arr) + rows = [ + ( + round(float(row[0]), 8), + round(float(row[1]), 8), + round(float(row[2]), 8), + bool(int(round(float(row[3])))), + round(float(row[4]), 8), + ) + for row in arr + ] + return tuple(sorted(rows)) + + +def _local_refs_key(refs: dict[str, list[numpy.ndarray]]) -> dict[str, tuple[tuple[object, ...], ...]]: + return { + parent: _transform_rows_key(numpy.concatenate(transforms)) + for parent, transforms in refs.items() + } + + +def _global_refs_key(refs: dict[tuple[str, ...], numpy.ndarray]) -> dict[tuple[str, ...], tuple[tuple[object, ...], ...]]: + return { + path: _transform_rows_key(transforms) + for path, transforms in refs.items() + } + + def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_lazy.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -62,6 +134,19 @@ def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 +def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None: + gds_file = tmp_path / 'complex_refs.gds' + src = _make_complex_ref_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-complex-refs') + + eager, _ = gdsii.readfile(gds_file) + lazy, _ = gdsii_lazy_arrow.readfile(gds_file) + + for name in ('leaf', 'child'): + assert _local_refs_key(lazy.find_refs_local(name)) == _local_refs_key(eager.find_refs_local(name)) + assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() From 7439e4cae9df160eaabc473b75512f135d863a32 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 21 Apr 2026 21:50:18 -0700 Subject: [PATCH 15/36] [arrow] improve test coverage and error handling --- masque/file/gdsii_arrow.py | 35 ++++++++++--- masque/test/test_gdsii_arrow.py | 59 +++++++++++++++++++++- masque/test/test_gdsii_lazy_arrow.py | 75 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index a38dee5..95229ea 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -37,6 +37,7 @@ import sys import tempfile from pprint import pformat +from klamath.basic import KlamathError import numpy from numpy.typing import ArrayLike, NDArray import pyarrow @@ -54,9 +55,10 @@ logger = logging.getLogger(__name__) ffi.cdef( """ - void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); - void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); - void read_cells_bytes( + const char* last_error_message(void); + int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); + int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + int read_cells_bytes( uint8_t* data, size_t size, uint64_t* ranges, @@ -203,11 +205,11 @@ def _read_to_arrow( tmp_stream.write(data) tmp_name = tmp_stream.name try: - _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path') finally: pathlib.Path(tmp_name).unlink(missing_ok=True) else: - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path') return _import_arrow_array(ptr_array, ptr_schema) @@ -217,12 +219,24 @@ def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: return pyarrow.Array._import_from_c(iptr_array, iptr_schema) +def _call_native(status: int, action: str) -> None: + if status == 0: + return + + err_ptr = _get_clib().last_error_message() + if err_ptr == ffi.NULL: + raise KlamathError(f'{action} failed') + + message = ffi.string(err_ptr).decode(errors='replace') + raise KlamathError(message) + + def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) buf_view = memoryview(buffer) cbuf = ffi.from_buffer('uint8_t[]', buf_view) - _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes') return _import_arrow_array(ptr_array, ptr_schema) @@ -236,7 +250,10 @@ def _read_selected_cells_to_arrow( cbuf = ffi.from_buffer('uint8_t[]', buf_view) flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) cranges = ffi.from_buffer('uint64_t[]', flat_ranges) - _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + _call_native( + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema), + 'read_cells_bytes', + ) return _import_arrow_array(ptr_array, ptr_schema) @@ -713,7 +730,9 @@ def _gpaths_to_mpaths( layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] - cap_int = elem_path_types[ee] + cap_int = int(elem_path_types[ee]) + if cap_int not in path_cap_map: + raise PatternError(f'Unrecognized path type: {cap_int}') cap = path_cap_map[cap_int] if cap_int == 4: cap_extensions = elem_extensions[ee] diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 660818c..b2962f8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -1,11 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') -from .. import Ref, Label +from .. import Ref, Label, PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -175,6 +179,26 @@ def _make_arrow_test_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_roundtrip.gds' @@ -238,6 +262,28 @@ def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: gdsii_arrow.read_arrow(libarr, raw_mode=False) +def test_gdsii_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_arrow + try: + gdsii_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -435,6 +481,17 @@ def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' +def test_gdsii_arrow_invalid_path_type_matches_gdsii(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii_arrow.readfile(gds_file) + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index b30b058..eb62721 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -1,10 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') +from .. import PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -76,6 +81,26 @@ def _make_complex_ref_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: arr = numpy.asarray(values, dtype=float) arr = numpy.atleast_2d(arr) @@ -147,6 +172,38 @@ def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) +def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_lazy_arrow + try: + gdsii_lazy_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_lazy_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + +def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + lib['top'] + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() @@ -165,6 +222,24 @@ def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> Non assert out_file.read_bytes() == gds_file.read_bytes() +def test_gdsii_lazy_arrow_gzipped_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds.gz' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through-gz') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds.gz' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: base_a = Library() leaf_a = Pattern() From e33a0f5ae14ce15eece9f779142d9772bb9cd541 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 27 May 2026 21:19:37 -0700 Subject: [PATCH 16/36] [Tool / SimpleTool / AutoTool] Documentation updates --- masque/builder/tools.py | 221 +++++++++++++++++++++++++++++++--------- 1 file changed, 174 insertions(+), 47 deletions(-) diff --git a/masque/builder/tools.py b/masque/builder/tools.py index c17a7b9..923e4a3 100644 --- a/masque/builder/tools.py +++ b/masque/builder/tools.py @@ -1,9 +1,19 @@ """ Tools are objects which dynamically generate simple single-use devices (e.g. wires or waveguides) -Concrete tools may implement native planning/rendering for `L`, `S`, or `U` routes. -Any unimplemented planning method falls back to the corresponding `trace*()` method, -and `Pather` may further synthesize some routes from simpler primitives when needed. +The `Tool` interface has two layers: + +* `traceL`/`traceS`/`traceU` create concrete single-use geometry immediately. +* `planL`/`planS`/`planU` return an output `Port` plus tool-specific render + data, allowing `Pather(auto_render=False)` to defer geometry creation until + `Tool.render()` is called with a batch of `RenderStep`s. + +Plans are expressed in local tool coordinates: the input port is at `(0, 0)` +with rotation `0`, `length` is measured along the input axis, and positive +`jog` is left of the direction of travel. Concrete tools may implement native +planning/rendering for L, S, and U routes; otherwise the base planning methods +fall back to the corresponding `trace*()` methods. `Pather` may also synthesize +some routes from simpler primitives when a tool does not provide a native route. """ from typing import Literal, Any, Self, cast from collections.abc import Sequence, Callable, Iterator @@ -25,8 +35,11 @@ from ..error import BuildError @dataclass(frozen=True, slots=True) class RenderStep: """ - Representation of a single saved operation, used by deferred `Pather` - instances and passed to `Tool.render()` when `Pather.render()` is called. + A single deferred routing operation. + + `Pather(auto_render=False)` stores these records while routing and later + passes batches of compatible steps to `Tool.render()` when `Pather.render()` + is called. """ opcode: Literal['L', 'S', 'U', 'P'] """ What operation is being performed. @@ -37,10 +50,13 @@ class RenderStep: """ tool: 'Tool | None' - """ The current tool. May be `None` if `opcode='P'` """ + """ Tool that produced this step, or `None` for `opcode='P'`. """ start_port: Port + """ Input-side port before this step is rendered. """ + end_port: Port + """ Output-side port after this step is rendered. """ data: Any """ Arbitrary tool-specific data""" @@ -101,7 +117,9 @@ class RenderStep: def measure_tool_plan(tree: ILibrary, port_names: tuple[str, str]) -> tuple[Port, Any]: """ - Extracts a Port and returns the tree (as data) for tool planning fallbacks. + Measure generated geometry for the base `Tool.plan*()` fallbacks. + + Returns the calculated output port and the original tree as render data. """ pat = tree.top_pattern() in_p = pat[port_names[0]] @@ -113,6 +131,13 @@ def measure_tool_plan(tree: ILibrary, port_names: tuple[str, str]) -> tuple[Port class Tool: """ Interface for path (e.g. wire or waveguide) generation. + + Subclasses may implement immediate `trace*()` methods, deferred + `plan*()`/`render()` methods, or both. The base `plan*()` implementations + call the matching `trace*()` method and measure the resulting ports, so a + simple immediate-rendering tool can implement only `traceL`, `traceS`, or + `traceU` as needed. Tools that support deferred rendering should return + compact, tool-specific data from `plan*()` and consume it in `render()`. """ def traceL( self, @@ -172,7 +197,7 @@ class Tool: """ Create a wire or waveguide that travels exactly `length` distance along the axis of its input port, and `jog` distance on the perpendicular axis. - `jog` is positive when moving left of the direction of travel (from input to ouput port). + `jog` is positive when moving left of the direction of travel (from input to output port). Used by `Pather`. @@ -236,7 +261,7 @@ class Tool: Returns: The calculated output `Port` for the wire, assuming an input port at (0, 0) with rotation 0. - Any tool-specifc data, to be stored in `RenderStep.data`, for use during rendering. + Any tool-specific data, to be stored in `RenderStep.data`, for use during rendering. Raises: BuildError if an impossible or unsupported geometry is requested. @@ -284,7 +309,7 @@ class Tool: Returns: The calculated output `Port` for the wire, assuming an input port at (0, 0) with rotation 0. - Any tool-specifc data, to be stored in `RenderStep.data`, for use during rendering. + Any tool-specific data, to be stored in `RenderStep.data`, for use during rendering. Raises: BuildError if an impossible or unsupported geometry is requested. @@ -312,8 +337,9 @@ class Tool: **kwargs, ) -> Library: """ - Create a wire or waveguide that travels exactly `jog` distance along the axis - perpendicular to its input port (i.e. a U-bend). + Create a wire or waveguide whose output is displaced by `length` along + the input axis and `jog` along the perpendicular axis, while preserving + the input orientation (i.e. a U-bend or jogged U-turn). Used by `Pather`. Tools may leave this unimplemented if they do not support a native U-bend primitive. @@ -328,6 +354,7 @@ class Tool: jog: The total offset from the input to output, along the perpendicular axis. A positive number implies a leftwards shift (i.e. counterclockwise bend followed by a clockwise bend) + length: The total offset from the input to output, along the input axis. in_ptype: The `ptype` of the port into which this wire's input will be `plug`ged. out_ptype: The `ptype` of the port into which this wire's output will be `plug`ged. port_names: The output pattern will have its input port named `port_names[0]` and @@ -351,8 +378,9 @@ class Tool: **kwargs, ) -> tuple[Port, Any]: """ - Plan a wire or waveguide that travels exactly `jog` distance along the axis - perpendicular to its input port (i.e. a U-bend). + Plan a wire or waveguide whose output is displaced by optional `length` + along the input axis and `jog` along the perpendicular axis, while + preserving the input orientation (i.e. a U-bend or jogged U-turn). Used by `Pather` when `auto_render=False`. This is an optional native-planning hook: tools may implement it when they can represent a U-turn directly, otherwise they may rely @@ -374,7 +402,7 @@ class Tool: Returns: The calculated output `Port` for the wire, assuming an input port at (0, 0) with rotation 0. - Any tool-specifc data, to be stored in `RenderStep.data`, for use during rendering. + Any tool-specific data, to be stored in `RenderStep.data`, for use during rendering. Raises: BuildError if an impossible or unsupported geometry is requested. @@ -404,6 +432,11 @@ class Tool: Render the provided `batch` of `RenderStep`s into geometry, returning a tree (a Library with a single topcell). + The base implementation is intended for steps whose plan data came from + the base fallback planners, where `RenderStep.data` is already an + `ILibrary`. Subclasses with native `plan*()` data should generally + override this method. + Args: batch: A sequence of `RenderStep` objects containing the ports and data provided by this tool's `planL`/`planS`/`planU` functions. @@ -464,15 +497,18 @@ abstract_tuple_t = tuple[Abstract, str, str] @dataclass class SimpleTool(Tool, metaclass=ABCMeta): """ - A simple tool which relies on a single pre-rendered `bend` pattern, a function - for generating straight paths, and a table of pre-rendered `transitions` for converting - from non-native ptypes. + Minimal L-route tool built from one straight generator and one bend. + + `SimpleTool` supports straight segments and single-bend L routes through + `planL`/`traceL`/`render`. It does not perform automatic port-type + transitions and does not provide native S or U routes. Use `AutoTool` when + routes need multiple candidate primitives, transitions, S-bends, or U-turns. """ straight: tuple[Callable[[float], Pattern] | Callable[[float], Library], str, str] - """ `create_straight(length: float), in_port_name, out_port_name` """ + """ `(create_straight, in_port_name, out_port_name)` for straight segments. """ bend: abstract_tuple_t # Assumed to be clockwise - """ `clockwise_bend_abstract, in_port_name, out_port_name` """ + """ `(clockwise_bend_abstract, in_port_name, out_port_name)` for L turns. """ default_out_ptype: str """ Default value for out_ptype """ @@ -482,7 +518,7 @@ class SimpleTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class LData: - """ Data for planL """ + """ Deferred render data returned by `planL()`. """ straight_length: float straight_kwargs: dict[str, Any] ccw: SupportsBool | None @@ -608,43 +644,114 @@ class SimpleTool(Tool, metaclass=ABCMeta): @dataclass class AutoTool(Tool, metaclass=ABCMeta): """ - A simple tool which relies on a single pre-rendered `bend` pattern, a function - for generating straight paths, and a table of pre-rendered `transitions` for converting - from non-native ptypes. + A routing tool assembled from reusable path primitives. + + `AutoTool` chooses among prioritized straight generators, pre-rendered bends, + optional native S-bend generators, and pre-rendered transitions to satisfy the + `Tool` planning/rendering interface used by `Pather`. + + Route selection is greedy in the order supplied by `straights`, `bends`, and + `sbends`. For each route, the planner subtracts any transition and bend + overhead from the requested distance, then uses the first candidate whose + remaining straight or jog length falls within that candidate's range. + + `planL` uses one straight and, if `ccw` is not `None`, one bend. `planS` + first tries a straight plus a native S-bend, then a pure native S-bend, and + falls back to a two-L route when no native S-bend candidate fits. `planU` + is implemented as a two-L route. + + Transition keys are `(external_ptype, internal_ptype)`. For example, a + transition keyed by `('m2wire', 'm1wire')` is used when the route is being + attached to an external `m2wire` port but the selected primitive is `m1wire`. + Call `add_complementary_transitions()` to automatically add reversed entries + for any missing opposite directions. + + Straight and S-bend generator functions may return either a `Pattern` or a + single-top `Library`. Extra keyword arguments passed to `trace*()` or + `render()` are forwarded to those generators, along with any keyword + arguments captured during `plan*()`. """ @dataclass(frozen=True, slots=True) class Straight: - """ Description of a straight-path generator """ + """ + Description of a straight-path generator. + + `fn(length, **kwargs)` must return a path whose `in_port_name` and + `out_port_name` ports are separated by `length` along the input axis. + The planner considers this generator only when the required length is in + `length_range`, with an inclusive lower bound and exclusive upper bound. + """ ptype: str + """ Port type produced by this straight segment. """ + fn: Callable[[float], Pattern] | Callable[[float], Library] + """ Generator function called as `fn(length, **kwargs)`. """ + in_port_name: str + """ Name of the input port on the generated pattern. """ + out_port_name: str + """ Name of the output port on the generated pattern. """ + length_range: tuple[float, float] = (0, numpy.inf) + """ Valid generated lengths, as `(inclusive_min, exclusive_max)`. """ @dataclass(frozen=True, slots=True) class SBend: - """ Description of an s-bend generator """ + """ + Description of a native S-bend generator. + + `fn(jog, **kwargs)` is called with a non-negative jog magnitude and must + return a path whose output port faces back toward the input port. For a + negative requested jog, `AutoTool` mirrors the generated S-bend during + rendering. + """ ptype: str + """ Port type produced by this S-bend. """ fn: Callable[[float], Pattern] | Callable[[float], Library] """ - Generator function. `jog` (only argument) is assumed to be left (ccw) relative to travel - and may be negative for a jog in the opposite direction. Won't be called if jog=0. + Generator function called as `fn(abs(jog), **kwargs)`. The generated + geometry is assumed to jog left, i.e. counterclockwise relative to the + direction of travel. This function is not called when the residual jog is + zero. """ in_port_name: str + """ Name of the input port on the generated pattern. """ + out_port_name: str + """ Name of the output port on the generated pattern. """ + jog_range: tuple[float, float] = (0, numpy.inf) + """ Valid residual jog magnitudes, as `(inclusive_min, exclusive_max)`. """ @dataclass(frozen=True, slots=True) class Bend: - """ Description of a pre-rendered bend """ + """ + Description of a pre-rendered L-bend. + + `abstract` must contain `in_port_name` and `out_port_name`. The + `clockwise` flag describes the in-to-out turn direction of that stored + bend. If `mirror` is true, `AutoTool` mirrors the stored bend to realize + the opposite turn direction; otherwise it plugs the bend from the + opposite port where possible. + """ abstract: Abstract + """ Abstract for the reusable bend pattern. """ + in_port_name: str + """ Name of the bend input port. """ + out_port_name: str + """ Name of the bend output port. """ + clockwise: bool = True # Is in-to-out clockwise? + """ Whether the stored bend turns clockwise from input to output. """ + mirror: bool = True # Should we mirror to get the other rotation? + """ Whether to mirror the stored bend to produce the opposite turn. """ @property def in_port(self) -> Port: @@ -656,10 +763,22 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class Transition: - """ Description of a pre-rendered transition """ + """ + Description of a pre-rendered port-type transition. + + `their_port_name` is the external side of the transition and + `our_port_name` is the side compatible with the selected internal + primitive. The transition table key should match that direction: + `(their_ptype, our_ptype)`. + """ abstract: Abstract + """ Abstract for the reusable transition pattern. """ + their_port_name: str + """ Name of the external-side port. """ + our_port_name: str + """ Name of the internal primitive-side port. """ @property def our_port(self) -> Port: @@ -674,7 +793,7 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class LPlan: - """ Template for an L-path configuration """ + """ Candidate L-route configuration before final straight length is known. """ straight: 'AutoTool.Straight' bend: 'AutoTool.Bend | None' in_trans: 'AutoTool.Transition | None' @@ -687,7 +806,7 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class LData: - """ Data for planL """ + """ Deferred render data returned by `planL()`. """ straight_length: float straight: 'AutoTool.Straight' straight_kwargs: dict[str, Any] @@ -758,7 +877,7 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class SData: - """ Data for planS """ + """ Deferred render data for native-S routes returned by `planS()`. """ straight_length: float straight: 'AutoTool.Straight' gen_kwargs: dict[str, Any] @@ -770,7 +889,7 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass(frozen=True, slots=True) class UData: - """ Data for planU or planS (double-L) """ + """ Deferred render data for `planU()` or double-L `planS()` routes. """ ldata0: 'AutoTool.LData' ldata1: 'AutoTool.LData' straight2: 'AutoTool.Straight' @@ -834,21 +953,27 @@ class AutoTool(Tool, metaclass=ABCMeta): raise BuildError(f"Failed to find a valid double-L configuration for {length=}, {jog=}") straights: list[Straight] - """ List of straight-generators to choose from, in order of priority """ + """ Straight generators to choose from, in priority order. """ bends: list[Bend] - """ List of bends to choose from, in order of priority """ + """ L-bend primitives to choose from, in priority order. """ sbends: list[SBend] - """ List of S-bend generators to choose from, in order of priority """ + """ Native S-bend generators to choose from, in priority order. """ transitions: dict[tuple[str, str], Transition] - """ `{(external_ptype, internal_ptype): Transition, ...}` """ + """ Mapping from `(external_ptype, internal_ptype)` to transition primitive. """ default_out_ptype: str - """ Default value for out_ptype """ + """ Output port type used when a zero-length route provides no primitive ptype. """ def add_complementary_transitions(self) -> Self: + """ + Add reversed transition entries for any missing opposite directions. + + Existing explicit entries are preserved. The method mutates + `self.transitions` and returns `self` for fluent construction. + """ for iioo in list(self.transitions.keys()): ooii = (iioo[1], iioo[0]) self.transitions.setdefault(ooii, self.transitions[iioo].reversed()) @@ -928,7 +1053,7 @@ class AutoTool(Tool, metaclass=ABCMeta): straight_kwargs: dict[str, Any], ) -> ILibrary: """ - Render an L step into a preexisting tree + Render an L step into an existing tree. """ pat = tree.top_pattern() if data.in_transition: @@ -1061,7 +1186,7 @@ class AutoTool(Tool, metaclass=ABCMeta): gen_kwargs: dict[str, Any], ) -> ILibrary: """ - Render an L step into a preexisting tree + Render a native-S step into an existing tree. """ pat = tree.top_pattern() if data.in_transition: @@ -1207,19 +1332,21 @@ class AutoTool(Tool, metaclass=ABCMeta): @dataclass class PathTool(Tool, metaclass=ABCMeta): """ - A tool which draws `Path` geometry elements. + Tool that renders routes directly as `Pattern.path()` geometry. - If `planL` / `render` are used, the `Path` elements can cover >2 vertices; - with `path` only individual rectangles will be drawn. + `PathTool` supports L and S routes. Immediate `traceL()` and `traceS()` + create one path element per route, while deferred `render()` combines a + compatible batch of L/S `RenderStep`s into one multi-vertex path. U routes + are left to `Pather` synthesis or to a different tool. """ layer: layer_t - """ Layer to draw on """ + """ Layer to draw generated path geometry on. """ width: float - """ `Path` width """ + """ Width of generated path geometry. """ ptype: str = 'unk' - """ ptype for any ports in patterns generated by this tool """ + """ Port type for generated input and output ports. """ #@dataclass(frozen=True, slots=True) #class LData: From e4a52b2c907abe453f463c82fadf86d4fdd0c821 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 28 May 2026 21:45:06 -0700 Subject: [PATCH 17/36] [AutoTool] fix output transition offset --- masque/builder/tools.py | 2 +- masque/test/test_autotool_refactor.py | 43 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/masque/builder/tools.py b/masque/builder/tools.py index 923e4a3..af49d44 100644 --- a/masque/builder/tools.py +++ b/masque/builder/tools.py @@ -1014,7 +1014,7 @@ class AutoTool(Tool, metaclass=ABCMeta): return numpy.zeros(2) orot = out_transition.our_port.rotation assert orot is not None - otrans_dxy = rotation_matrix_2d(pi - orot - bend_angle) @ (out_transition.their_port.offset - out_transition.our_port.offset) + otrans_dxy = rotation_matrix_2d(bend_angle - orot - pi) @ (out_transition.their_port.offset - out_transition.our_port.offset) return otrans_dxy def planL( diff --git a/masque/test/test_autotool_refactor.py b/masque/test/test_autotool_refactor.py index 3109447..2f560cb 100644 --- a/masque/test/test_autotool_refactor.py +++ b/masque/test/test_autotool_refactor.py @@ -132,6 +132,49 @@ def test_autotool_planL_selection(multi_bend_tool) -> None: assert data.straight_length == 13 assert_allclose(p.offset, [15, 2]) + +@pytest.mark.parametrize("ccw", [False, True]) +def test_autotool_traceL_matches_plan_with_post_bend_transition(ccw: bool) -> None: + lib = Library() + + bend_pat = Pattern() + bend_pat.ports["A"] = Port((0, 0), 0, ptype="core") + bend_pat.ports["B"] = Port((2, -2), pi / 2, ptype="core") + lib["core_bend"] = bend_pat + + trans_pat = Pattern() + trans_pat.ports["CORE"] = Port((0, 0), 0, ptype="core") + trans_pat.ports["EXT"] = Port((3, 1), pi, ptype="ext") + lib["out_trans"] = trans_pat + + tool = AutoTool( + straights=[ + AutoTool.Straight( + ptype="core", + fn=lambda length: make_straight(length, ptype="core"), + in_port_name="A", + out_port_name="B", + length_range=(0, 1e8), + ), + ], + bends=[ + AutoTool.Bend(lib.abstract("core_bend"), "A", "B", clockwise=True, mirror=True), + ], + sbends=[], + transitions={ + ("ext", "core"): AutoTool.Transition(lib.abstract("out_trans"), "EXT", "CORE"), + }, + default_out_ptype="core", + ) + + plan_port, data = tool.planL(ccw, 10, out_ptype="ext") + + assert data.out_transition is not None + + tree = tool.traceL(ccw, 10, out_ptype="ext") + assert_trace_matches_plan(plan_port, tree) + + def test_autotool_planU_consistency(multi_bend_tool) -> None: tool, lib = multi_bend_tool From 51c7fa9add5fc2701a31b718cd5d36287b9b1c01 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 15 Jun 2026 18:40:51 -0700 Subject: [PATCH 18/36] [Arc] add angle_ref to enable focus-based arcs --- masque/shapes/arc.py | 177 ++++++++++++++++++++++++----- masque/test/test_shape_advanced.py | 21 ++++ masque/test/test_shapes.py | 32 ++++++ 3 files changed, 204 insertions(+), 26 deletions(-) diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index 00c5714..fe258c4 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -1,6 +1,7 @@ from typing import Any, cast import copy import functools +from enum import Enum import numpy from numpy import pi @@ -13,18 +14,37 @@ from ..utils import is_scalar, annotations_t, annotations_lt, annotations_eq, re from ..traits import PositionableImpl +@functools.total_ordering +class ArcAngleRef(Enum): + Center = 'center' + FocusPos = 'focus_pos' + FocusNeg = 'focus_neg' + + def __lt__(self, other: Any) -> bool: + if self.__class__ is not other.__class__: + return self.__class__.__name__ < other.__class__.__name__ + order = { + ArcAngleRef.Center: 0, + ArcAngleRef.FocusPos: 1, + ArcAngleRef.FocusNeg: 2, + } + return order[self] < order[other] + + @functools.total_ordering class Arc(PositionableImpl, Shape): """ - An elliptical arc, formed by cutting off an elliptical ring with two rays which exit from its - center. It has a position, two radii, a start and stop angle, a rotation, and a width. + An elliptical arc, formed by cutting off an elliptical ring with two rays. + By default the rays exit from its center, but they can optionally exit from one of the + foci of the nominal ellipse. It has a position, two radii, a start and stop angle, + a rotation, and a width. The radii define an ellipse; the ring is formed with radii +/- width/2. The rotation gives the angle from x-axis, counterclockwise, to the first (x) radius. The start and stop angle are measured counterclockwise from the first (x) radius. """ __slots__ = ( - '_radii', '_angles', '_width', '_rotation', + '_radii', '_angles', '_width', '_rotation', '_angle_ref', # Inherited '_offset', '_repetition', '_annotations', ) @@ -41,6 +61,11 @@ class Arc(PositionableImpl, Shape): _width: float """ Width of the arc """ + _angle_ref: ArcAngleRef + """ Origin used by start/stop rays """ + + AngleRef = ArcAngleRef + # radius properties @property def radii(self) -> NDArray[numpy.float64]: @@ -113,6 +138,18 @@ class Arc(PositionableImpl, Shape): def stop_angle(self, val: float) -> None: self.angles = (self.angles[0], val) + # Angle reference property + @property + def angle_ref(self) -> ArcAngleRef: + """ + Origin used to interpret start and stop angle rays. + """ + return self._angle_ref + + @angle_ref.setter + def angle_ref(self, val: ArcAngleRef | str) -> None: + self._angle_ref = ArcAngleRef(val) + # Rotation property @property def rotation(self) -> float: @@ -159,6 +196,7 @@ class Arc(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, + angle_ref: ArcAngleRef | str = ArcAngleRef.Center, raw: bool = False, ) -> None: if raw: @@ -170,6 +208,7 @@ class Arc(PositionableImpl, Shape): self._width = width self._offset = offset self._rotation = rotation + self._angle_ref = ArcAngleRef(angle_ref) self._repetition = repetition self._annotations = annotations else: @@ -178,6 +217,7 @@ class Arc(PositionableImpl, Shape): self.width = width self.offset = offset self.rotation = rotation + self.angle_ref = angle_ref self.repetition = repetition self.annotations = annotations @@ -199,6 +239,7 @@ class Arc(PositionableImpl, Shape): and numpy.array_equal(self.angles, other.angles) and self.width == other.width and self.rotation == other.rotation + and self.angle_ref == other.angle_ref and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -215,6 +256,8 @@ class Arc(PositionableImpl, Shape): return tuple(self.radii) < tuple(other.radii) if not numpy.array_equal(self.angles, other.angles): return tuple(self.angles) < tuple(other.angles) + if self.angle_ref != other.angle_ref: + return self.angle_ref < other.angle_ref if not numpy.array_equal(self.offset, other.offset): return tuple(self.offset) < tuple(other.offset) if self.rotation != other.rotation: @@ -370,6 +413,11 @@ class Arc(PositionableImpl, Shape): return self def mirror(self, axis: int = 0) -> 'Arc': + if self.angle_ref != ArcAngleRef.Center: + x_major = self.radius_x > self.radius_y + y_major = self.radius_y > self.radius_x + if (axis == 0 and y_major) or (axis == 1 and x_major): + self._swap_focus_ref() self.rotation *= -1 self.rotation += axis * pi self.angles *= -1 @@ -381,6 +429,7 @@ class Arc(PositionableImpl, Shape): return self def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + angle_ref = self.angle_ref if self.radius_x < self.radius_y: radii = self.radii / self.radius_x scale = self.radius_x @@ -391,23 +440,26 @@ class Arc(PositionableImpl, Shape): scale = self.radius_y rotation = self.rotation + pi / 2 angles = self.angles - pi / 2 + angle_ref = _swapped_focus_ref(angle_ref) delta_angle = angles[1] - angles[0] start_angle = angles[0] % (2 * pi) if start_angle >= pi: start_angle -= pi rotation += pi + angle_ref = _swapped_focus_ref(angle_ref) norm_angles = (start_angle, start_angle + delta_angle) rotation %= 2 * pi width = self.width - return ((type(self), tuple(radii.tolist()), norm_angles, width / norm_value), + return ((type(self), tuple(radii.tolist()), norm_angles, width / norm_value, angle_ref.value), (self.offset, scale / norm_value, rotation, False), lambda: Arc( radii=radii * norm_value, angles=norm_angles, width=width * norm_value, + angle_ref=angle_ref, )) def get_cap_edges(self) -> NDArray[numpy.float64]: @@ -418,27 +470,16 @@ class Arc(PositionableImpl, Shape): [[x2, y2], [x3, y3]]], would create this arc from its corresponding ellipse. ``` """ - a_ranges = cast('_array2x2_t', self._angles_to_parameters()) + a_ranges = self._angles_to_parameters() - mins = [] - maxs = [] - for aa, sgn in zip(a_ranges, (-1, +1), strict=True): - wh = sgn * self.width / 2 - rx = self.radius_x + wh - ry = self.radius_y + wh - - sin_r = numpy.sin(self.rotation) - cos_r = numpy.cos(self.rotation) - sin_a = numpy.sin(aa) - cos_a = numpy.cos(aa) - - # arc endpoints - xn, xp = sorted(rx * cos_r * cos_a - ry * sin_r * sin_a) - yn, yp = sorted(rx * sin_r * cos_a + ry * cos_r * sin_a) - - mins.append([xn, yn]) - maxs.append([xp, yp]) - return numpy.array([mins, maxs]) + self.offset + cuts = [] + for index in range(2): + edge = [] + for aa, sgn in zip(a_ranges, (-1, +1), strict=True): + wh = sgn * self.width / 2 + edge.append(self._point_on_edge(self.radius_x + wh, self.radius_y + wh, aa[index])) + cuts.append(edge) + return numpy.array(cuts) + self.offset def _angles_to_parameters(self) -> NDArray[numpy.float64]: """ @@ -459,7 +500,7 @@ class Arc(PositionableImpl, Shape): rx = self.radius_x + wh ry = self.radius_y + wh - a0, a1 = (numpy.arctan2(rx * numpy.sin(ai), ry * numpy.cos(ai)) for ai in self.angles) + a0, a1 = (self._angle_to_parameter(ai, rx, ry) for ai in self.angles) sign = numpy.sign(d_angle) if sign != numpy.sign(a1 - a0): a1 += sign * 2 * pi @@ -467,9 +508,93 @@ class Arc(PositionableImpl, Shape): aa.append((a0, a1)) return numpy.array(aa, dtype=float) + def _angle_to_parameter(self, angle: float, rx: float, ry: float) -> float: + """ + Convert an angle-reference ray to the ellipse parameter for one boundary edge. + + Center-referenced arcs convert the ray angle from polar coordinates about the origin. + Focus-referenced arcs solve the forward ray/ellipse intersection from the selected + nominal focus and return the parameter `t` for `[rx*cos(t), ry*sin(t)]`. + """ + if self.angle_ref == ArcAngleRef.Center: + return numpy.arctan2(rx * numpy.sin(angle), ry * numpy.cos(angle)) + + focus = self._focus_point() + if rx <= 0 or ry <= 0: + raise PatternError('Focus-referenced arc boundary radii must be positive') + + fx, fy = focus + origin_position = fx * fx / (rx * rx) + fy * fy / (ry * ry) + if origin_position >= 1: + raise PatternError('Focus-referenced arc ray origin must be inside both arc boundary ellipses') + + dx = numpy.cos(angle) + dy = numpy.sin(angle) + aa = dx * dx / (rx * rx) + dy * dy / (ry * ry) + bb = 2 * (fx * dx / (rx * rx) + fy * dy / (ry * ry)) + cc = origin_position - 1 + determinant = bb * bb - 4 * aa * cc + if determinant < 0: + raise PatternError('Focus-referenced arc ray does not intersect boundary ellipse') + + roots = numpy.array(( + (-bb - numpy.sqrt(determinant)) / (2 * aa), + (-bb + numpy.sqrt(determinant)) / (2 * aa), + )) + positive_roots = roots[roots > 0] + if positive_roots.size != 1: + raise PatternError('Focus-referenced arc ray must have exactly one forward boundary intersection') + + point = focus + positive_roots[0] * numpy.array((dx, dy)) + return numpy.arctan2(point[1] / ry, point[0] / rx) + + def _focus_point(self) -> NDArray[numpy.float64]: + """ + Return the selected nominal focus in the arc's unrotated local coordinates. + + `FocusPos` and `FocusNeg` select opposite directions along the major axis. Circles + have coincident foci, so both focus modes intentionally collapse to the center. + """ + if self.angle_ref == ArcAngleRef.Center or self.radius_x == self.radius_y: + return numpy.zeros(2) + + sign = 1 if self.angle_ref == ArcAngleRef.FocusPos else -1 + if self.radius_x > self.radius_y: + return numpy.array((sign * numpy.sqrt(self.radius_x * self.radius_x - self.radius_y * self.radius_y), 0.0)) + return numpy.array((0.0, sign * numpy.sqrt(self.radius_y * self.radius_y - self.radius_x * self.radius_x))) + + def _point_on_edge(self, rx: float, ry: float, tt: float) -> NDArray[numpy.float64]: + """ + Return a rotated local-space point on a boundary ellipse, before applying offset. + """ + sin_r = numpy.sin(self.rotation) + cos_r = numpy.cos(self.rotation) + return numpy.array(( + rx * numpy.cos(tt) * cos_r - ry * numpy.sin(tt) * sin_r, + rx * numpy.cos(tt) * sin_r + ry * numpy.sin(tt) * cos_r, + )) + + def _swap_focus_ref(self) -> None: + """ + Swap `focus_pos` and `focus_neg`, leaving center-referenced arcs unchanged. + """ + self.angle_ref = _swapped_focus_ref(self.angle_ref) + def __repr__(self) -> str: angles = f' a°{numpy.rad2deg(self.angles)}' rotation = f' r°{numpy.rad2deg(self.rotation):g}' if self.rotation != 0 else '' - return f'' + angle_ref = f' ref={self.angle_ref.value}' if self.angle_ref != ArcAngleRef.Center else '' + return f'' + + +def _swapped_focus_ref(angle_ref: ArcAngleRef) -> ArcAngleRef: + """ + Return the opposite focus reference, or center for center-referenced arcs. + """ + if angle_ref == ArcAngleRef.FocusPos: + return ArcAngleRef.FocusNeg + if angle_ref == ArcAngleRef.FocusNeg: + return ArcAngleRef.FocusPos + return angle_ref _array2x2_t = tuple[tuple[float, float], tuple[float, float]] diff --git a/masque/test/test_shape_advanced.py b/masque/test/test_shape_advanced.py index 689df2a..8e35841 100644 --- a/masque/test/test_shape_advanced.py +++ b/masque/test/test_shape_advanced.py @@ -111,6 +111,14 @@ def test_rotated_arc_bounds_match_polygonized_geometry() -> None: assert_allclose(bounds, poly_bounds, atol=1e-3) +def test_rotated_focus_arc_bounds_match_polygonized_geometry() -> None: + arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, rotation=pi / 4, + offset=(100, 200), angle_ref=Arc.AngleRef.FocusPos) + bounds = arc.get_bounds_single() + poly_bounds = arc.to_polygons(num_vertices=8192)[0].get_bounds_single() + assert_allclose(bounds, poly_bounds, atol=1e-3) + + def test_curve_polygonizers_clamp_large_max_arclen() -> None: for shape in ( Circle(radius=10), @@ -128,6 +136,19 @@ def test_arc_polygonization_rejects_nan_implied_arclen() -> None: arc.to_polygons(num_vertices=24) +def test_focus_arc_rejects_focus_outside_inner_boundary() -> None: + arc = Arc(radii=(10, 5), angles=(0, 1), width=6, angle_ref=Arc.AngleRef.FocusPos) + with pytest.raises(PatternError, match='inside both arc boundary ellipses'): + arc.to_polygons(num_vertices=24) + + +def test_focus_arc_max_arclen_limits_segments() -> None: + arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, angle_ref=Arc.AngleRef.FocusNeg) + v = arc.to_polygons(max_arclen=2)[0].vertices + dist = numpy.sqrt(numpy.sum(numpy.diff(v, axis=0, append=v[:1]) ** 2, axis=1)) + assert numpy.all(dist <= 2.000001) + + def test_ellipse_integer_radii_scale_cleanly() -> None: ellipse = Ellipse(radii=(10, 20)) ellipse.scale_by(0.5) diff --git a/masque/test/test_shapes.py b/masque/test/test_shapes.py index b19d6bc..e453f7c 100644 --- a/masque/test/test_shapes.py +++ b/masque/test/test_shapes.py @@ -78,6 +78,30 @@ def test_arc_to_polygons() -> None: assert_allclose(bounds, [[0, 0], [11, 11]], atol=1e-10) +def test_arc_focus_to_polygons() -> None: + a = Arc(radii=(10, 6), angles=(-0.4, 0.7), width=1, angle_ref=Arc.AngleRef.FocusPos) + polys = a.to_polygons(num_vertices=32) + assert len(polys) == 1 + + focus = numpy.array([8.0, 0.0]) + cuts = a.get_cap_edges() + for angle, cut in zip(a.angles, cuts, strict=True): + direction = numpy.array([numpy.cos(angle), numpy.sin(angle)]) + for point in cut: + delta = point - focus + assert_allclose(direction[0] * delta[1] - direction[1] * delta[0], 0, atol=1e-10) + assert numpy.dot(direction, delta) > 0 + + +def test_arc_circle_focus_matches_center() -> None: + center = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) + focus = Arc(radii=(10, 10), angles=(0, pi / 2), width=2, angle_ref=Arc.AngleRef.FocusPos) + + assert_allclose(focus.to_polygons(num_vertices=32)[0].vertices, + center.to_polygons(num_vertices=32)[0].vertices, + atol=1e-10) + + def test_shape_mirror() -> None: e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) e.mirror(0) # Mirror across x axis (axis 0): in-place relative to offset @@ -91,6 +115,14 @@ def test_shape_mirror() -> None: # For Arc, mirror(0) negates rotation and angles assert_allclose(a.angles, [0, -pi / 4], atol=1e-10) + a = Arc(radii=(10, 5), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) + a.mirror(1) + assert a.angle_ref == Arc.AngleRef.FocusNeg + + a = Arc(radii=(5, 10), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) + a.mirror(0) + assert a.angle_ref == Arc.AngleRef.FocusNeg + def test_shape_flip_across() -> None: e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) From 4d57936da84a3603e4e460906ec371ece496213f Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 15 Jun 2026 19:59:21 -0700 Subject: [PATCH 19/36] [test] refactor tests --- masque/test/helpers.py | 27 + masque/test/test_advanced_routing.py | 77 -- masque/test/test_arc.py | 87 ++ masque/test/test_autotool.py | 81 -- ..._refactor.py => test_autotool_planning.py} | 145 +-- masque/test/test_boolean.py | 7 +- masque/test/test_circle.py | 17 + masque/test/test_curve_polygonization.py | 26 + masque/test/test_dxf.py | 22 +- masque/test/test_ellipse.py | 29 + ...dtrip.py => test_file_format_roundtrip.py} | 43 +- masque/test/test_manhattanize.py | 17 + masque/test/test_path.py | 28 +- masque/test/test_pather.py | 108 -- masque/test/test_pather_api.py | 936 ------------------ masque/test/test_pather_autotool.py | 127 +++ masque/test/test_pather_constraints.py | 213 ++++ masque/test/test_pather_core.py | 305 ++++++ masque/test/test_pather_place_plug.py | 122 +++ masque/test/test_pather_rendering.py | 312 ++++++ masque/test/test_pather_trace_into.py | 189 ++++ masque/test/test_poly_collection.py | 89 ++ masque/test/test_renderpather.py | 199 ---- masque/test/test_repetition.py | 6 +- masque/test/test_shape_advanced.py | 265 ----- masque/test/test_shape_ordering.py | 15 + masque/test/test_shape_transforms.py | 44 + masque/test/test_shapes.py | 174 ---- masque/test/test_text.py | 47 + masque/test/test_utils.py | 6 - masque/test/test_visualize.py | 2 - 31 files changed, 1734 insertions(+), 2031 deletions(-) create mode 100644 masque/test/helpers.py delete mode 100644 masque/test/test_advanced_routing.py create mode 100644 masque/test/test_arc.py delete mode 100644 masque/test/test_autotool.py rename masque/test/{test_autotool_refactor.py => test_autotool_planning.py} (73%) create mode 100644 masque/test/test_circle.py create mode 100644 masque/test/test_curve_polygonization.py create mode 100644 masque/test/test_ellipse.py rename masque/test/{test_file_roundtrip.py => test_file_format_roundtrip.py} (70%) create mode 100644 masque/test/test_manhattanize.py delete mode 100644 masque/test/test_pather.py delete mode 100644 masque/test/test_pather_api.py create mode 100644 masque/test/test_pather_autotool.py create mode 100644 masque/test/test_pather_constraints.py create mode 100644 masque/test/test_pather_core.py create mode 100644 masque/test/test_pather_place_plug.py create mode 100644 masque/test/test_pather_rendering.py create mode 100644 masque/test/test_pather_trace_into.py create mode 100644 masque/test/test_poly_collection.py delete mode 100644 masque/test/test_renderpather.py delete mode 100644 masque/test/test_shape_advanced.py create mode 100644 masque/test/test_shape_ordering.py create mode 100644 masque/test/test_shape_transforms.py delete mode 100644 masque/test/test_shapes.py create mode 100644 masque/test/test_text.py diff --git a/masque/test/helpers.py b/masque/test/helpers.py new file mode 100644 index 0000000..32b2f66 --- /dev/null +++ b/masque/test/helpers.py @@ -0,0 +1,27 @@ +from typing import Any + +import numpy +from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_allclose + + +def closed_edge_lengths(vertices: ArrayLike) -> NDArray[numpy.float64]: + """ + Return lengths for each edge of an implicitly closed vertex loop. + """ + vv = numpy.asarray(vertices, dtype=float) + return numpy.sqrt(numpy.sum(numpy.diff(vv, axis=0, append=vv[:1]) ** 2, axis=1)) + + +def assert_closed_edges_within(vertices: ArrayLike, max_len: float, *, atol: float = 1e-6) -> None: + """ + Assert that every edge in an implicitly closed vertex loop is no longer than `max_len`. + """ + assert numpy.all(closed_edge_lengths(vertices) <= max_len + atol) + + +def assert_bounds_close(shape_or_polygon: Any, expected: ArrayLike, *, atol: float = 1e-10) -> None: + """ + Assert that an object's single-shape bounds match `expected`. + """ + assert_allclose(shape_or_polygon.get_bounds_single(), expected, atol=atol) diff --git a/masque/test/test_advanced_routing.py b/masque/test/test_advanced_routing.py deleted file mode 100644 index 0008172..0000000 --- a/masque/test/test_advanced_routing.py +++ /dev/null @@ -1,77 +0,0 @@ -import pytest -from numpy.testing import assert_equal -from numpy import pi - -from ..builder import Pather -from ..builder.tools import PathTool -from ..library import Library -from ..ports import Port - - -@pytest.fixture -def advanced_pather() -> tuple[Pather, PathTool, Library]: - lib = Library() - # Simple PathTool: 2um width on layer (1,0) - tool = PathTool(layer=(1, 0), width=2, ptype="wire") - p = Pather(lib, tools=tool, auto_render=True, auto_render_append=False) - return p, tool, lib - - -def test_path_into_straight(advanced_pather: tuple[Pather, PathTool, Library]) -> None: - p, _tool, _lib = advanced_pather - # Facing ports - p.ports["src"] = Port((0, 0), 0, ptype="wire") # Facing East (into device) - # Forward (+pi relative to port) is West (-x). - # Put destination at (-20, 0) pointing East (pi). - p.ports["dst"] = Port((-20, 0), pi, ptype="wire") - - p.trace_into("src", "dst") - - assert "src" not in p.ports - assert "dst" not in p.ports - # Pather._traceL adds a Reference to the generated pattern - assert len(p.pattern.refs) == 1 - - -def test_path_into_bend(advanced_pather: tuple[Pather, PathTool, Library]) -> None: - p, _tool, _lib = advanced_pather - # Source at (0,0) rot 0 (facing East). Forward is West (-x). - p.ports["src"] = Port((0, 0), 0, ptype="wire") - # Destination at (-20, -20) rot pi (facing West). Forward is East (+x). - # Wait, src forward is -x. dst is at -20, -20. - # To use a single bend, dst should be at some -x, -y and its rotation should be 3pi/2 (facing South). - # Forward for South is North (+y). - p.ports["dst"] = Port((-20, -20), 3 * pi / 2, ptype="wire") - - p.trace_into("src", "dst") - - assert "src" not in p.ports - assert "dst" not in p.ports - # `trace_into()` now batches its internal legs before auto-rendering so the operation - # can roll back cleanly on later failures. - assert len(p.pattern.refs) == 1 - - -def test_path_into_sbend(advanced_pather: tuple[Pather, PathTool, Library]) -> None: - p, _tool, _lib = advanced_pather - # Facing but offset ports - p.ports["src"] = Port((0, 0), 0, ptype="wire") # Forward is West (-x) - p.ports["dst"] = Port((-20, -10), pi, ptype="wire") # Facing East (rot pi) - - p.trace_into("src", "dst") - - assert "src" not in p.ports - assert "dst" not in p.ports - - -def test_path_into_thru(advanced_pather: tuple[Pather, PathTool, Library]) -> None: - p, _tool, _lib = advanced_pather - p.ports["src"] = Port((0, 0), 0, ptype="wire") - p.ports["dst"] = Port((-20, 0), pi, ptype="wire") - p.ports["other"] = Port((10, 10), 0) - - p.trace_into("src", "dst", thru="other") - - assert "src" in p.ports - assert_equal(p.ports["src"].offset, [10, 10]) - assert "other" not in p.ports diff --git a/masque/test/test_arc.py b/masque/test/test_arc.py new file mode 100644 index 0000000..dc23144 --- /dev/null +++ b/masque/test/test_arc.py @@ -0,0 +1,87 @@ +import pytest +import numpy +from numpy import pi +from numpy.testing import assert_equal, assert_allclose + +from ..error import PatternError +from ..shapes import Arc +from .helpers import assert_closed_edges_within + + +def test_arc_init() -> None: + a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2, offset=(0, 0)) + assert_equal(a.radii, [10, 10]) + assert_equal(a.angles, [0, pi / 2]) + assert a.width == 2 + +def test_arc_to_polygons() -> None: + a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) + polys = a.to_polygons(num_vertices=32) + assert len(polys) == 1 + + # Quarter-circle ring section with outer radius 11 and inner radius 9. + bounds = polys[0].get_bounds_single() + assert_allclose(bounds, [[0, 0], [11, 11]], atol=1e-10) + +def test_arc_focus_to_polygons() -> None: + a = Arc(radii=(10, 6), angles=(-0.4, 0.7), width=1, angle_ref=Arc.AngleRef.FocusPos) + polys = a.to_polygons(num_vertices=32) + assert len(polys) == 1 + + focus = numpy.array([8.0, 0.0]) + cuts = a.get_cap_edges() + for angle, cut in zip(a.angles, cuts, strict=True): + direction = numpy.array([numpy.cos(angle), numpy.sin(angle)]) + for point in cut: + delta = point - focus + assert_allclose(direction[0] * delta[1] - direction[1] * delta[0], 0, atol=1e-10) + assert numpy.dot(direction, delta) > 0 + +def test_arc_circle_focus_matches_center() -> None: + center = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) + focus = Arc(radii=(10, 10), angles=(0, pi / 2), width=2, angle_ref=Arc.AngleRef.FocusPos) + + assert_allclose(focus.to_polygons(num_vertices=32)[0].vertices, + center.to_polygons(num_vertices=32)[0].vertices, + atol=1e-10) + +def test_arc_edge_cases() -> None: + a = Arc(radii=(10, 10), angles=(0, 3 * pi), width=2) + a.to_polygons(num_vertices=64) + bounds = a.get_bounds_single() + assert_allclose(bounds, [[-11, -11], [11, 11]], atol=1e-10) + +def test_rotated_arc_bounds_match_polygonized_geometry() -> None: + arc = Arc(radii=(10, 20), angles=(0, pi), width=2, rotation=pi / 4, offset=(100, 200)) + bounds = arc.get_bounds_single() + poly_bounds = arc.to_polygons(num_vertices=8192)[0].get_bounds_single() + assert_allclose(bounds, poly_bounds, atol=1e-3) + +def test_rotated_focus_arc_bounds_match_polygonized_geometry() -> None: + arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, rotation=pi / 4, + offset=(100, 200), angle_ref=Arc.AngleRef.FocusPos) + bounds = arc.get_bounds_single() + poly_bounds = arc.to_polygons(num_vertices=8192)[0].get_bounds_single() + assert_allclose(bounds, poly_bounds, atol=1e-3) + +def test_arc_polygonization_rejects_nan_implied_arclen() -> None: + arc = Arc(radii=(10, 20), angles=(0, numpy.nan), width=2) + with pytest.raises(PatternError, match='valid max_arclen'): + arc.to_polygons(num_vertices=24) + +def test_focus_arc_rejects_focus_outside_inner_boundary() -> None: + arc = Arc(radii=(10, 5), angles=(0, 1), width=6, angle_ref=Arc.AngleRef.FocusPos) + with pytest.raises(PatternError, match='inside both arc boundary ellipses'): + arc.to_polygons(num_vertices=24) + +def test_focus_arc_max_arclen_limits_segments() -> None: + arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, angle_ref=Arc.AngleRef.FocusNeg) + assert_closed_edges_within(arc.to_polygons(max_arclen=2)[0].vertices, 2) + +def test_arc_rejects_zero_radii_up_front() -> None: + with pytest.raises(PatternError, match='Radii must be positive'): + Arc(radii=(0, 5), angles=(0, 1), width=1) + with pytest.raises(PatternError, match='Radii must be positive'): + Arc(radii=(5, 0), angles=(0, 1), width=1) + with pytest.raises(PatternError, match='Radii must be positive'): + Arc(radii=(0, 0), angles=(0, 1), width=1) diff --git a/masque/test/test_autotool.py b/masque/test/test_autotool.py deleted file mode 100644 index e03994e..0000000 --- a/masque/test/test_autotool.py +++ /dev/null @@ -1,81 +0,0 @@ -import pytest -from numpy.testing import assert_allclose -from numpy import pi - -from ..builder import Pather -from ..builder.tools import AutoTool -from ..library import Library -from ..pattern import Pattern -from ..ports import Port - - -def make_straight(length: float, width: float = 2, ptype: str = "wire") -> Pattern: - pat = Pattern() - pat.rect((1, 0), xmin=0, xmax=length, yctr=0, ly=width) - pat.ports["in"] = Port((0, 0), 0, ptype=ptype) - pat.ports["out"] = Port((length, 0), pi, ptype=ptype) - return pat - - -@pytest.fixture -def autotool_setup() -> tuple[Pather, AutoTool, Library]: - lib = Library() - - # Define a simple bend - bend_pat = Pattern() - # 2x2 bend from (0,0) rot 0 to (2, -2) rot pi/2 (Clockwise) - bend_pat.ports["in"] = Port((0, 0), 0, ptype="wire") - bend_pat.ports["out"] = Port((2, -2), pi / 2, ptype="wire") - lib["bend"] = bend_pat - lib.abstract("bend") - - # Define a transition (e.g., via) - via_pat = Pattern() - via_pat.ports["m1"] = Port((0, 0), 0, ptype="wire_m1") - via_pat.ports["m2"] = Port((1, 0), pi, ptype="wire_m2") - lib["via"] = via_pat - via_abs = lib.abstract("via") - - tool_m1 = AutoTool( - straights=[ - AutoTool.Straight(ptype="wire_m1", fn=lambda length: make_straight(length, ptype="wire_m1"), in_port_name="in", out_port_name="out") - ], - bends=[], - sbends=[], - transitions={("wire_m2", "wire_m1"): AutoTool.Transition(via_abs, "m2", "m1")}, - default_out_ptype="wire_m1", - ) - - p = Pather(lib, tools=tool_m1) - # Start with an m2 port - p.ports["start"] = Port((0, 0), pi, ptype="wire_m2") - - return p, tool_m1, lib - - -def test_autotool_transition(autotool_setup: tuple[Pather, AutoTool, Library]) -> None: - p, _tool, _lib = autotool_setup - - # Route m1 from an m2 port. Should trigger via. - # length 10. Via length is 1. So straight m1 should be 9. - p.straight("start", 10) - - # Start at (0,0) rot pi (facing West). - # Forward (+pi relative to port) is East (+x). - # Via: m2(1,0)pi -> m1(0,0)0. - # Plug via m2 into start(0,0)pi: transformation rot=mod(pi-pi-pi, 2pi)=pi. - # rotate via by pi: m2 at (0,0), m1 at (-1, 0) rot pi. - # Then straight m1 of length 9 from (-1, 0) rot pi -> ends at (8, 0) rot pi. - # Wait, (length, 0) relative to (-1, 0) rot pi: - # transform (9, 0) by pi: (-9, 0). - # (-1, 0) + (-9, 0) = (-10, 0)? No. - # Let's re-calculate. - # start (0,0) rot pi. Direction East. - # via m2 is at (0,0), m1 is at (1,0). - # When via is plugged into start: m2 goes to (0,0). - # since start is pi and m2 is pi, rotation is 0. - # so via m1 is at (1,0) rot 0. - # then straight m1 length 9 from (1,0) rot 0: ends at (10, 0) rot 0. - - assert_allclose(p.ports["start"].offset, [10, 0], atol=1e-10) - assert p.ports["start"].ptype == "wire_m1" diff --git a/masque/test/test_autotool_refactor.py b/masque/test/test_autotool_planning.py similarity index 73% rename from masque/test/test_autotool_refactor.py rename to masque/test/test_autotool_planning.py index 2f560cb..1c98cde 100644 --- a/masque/test/test_autotool_refactor.py +++ b/masque/test/test_autotool_planning.py @@ -1,12 +1,61 @@ import pytest -from numpy.testing import assert_allclose from numpy import pi +from numpy.testing import assert_allclose from masque.builder.tools import AutoTool +from masque.builder.pather import Pather +from masque.library import Library from masque.pattern import Pattern from masque.ports import Port -from masque.library import Library -from masque.builder.pather import Pather + + +def _make_transition_straight(length: float, width: float = 2, ptype: str = "wire") -> Pattern: + pat = Pattern() + pat.rect((1, 0), xmin=0, xmax=length, yctr=0, ly=width) + pat.ports["in"] = Port((0, 0), 0, ptype=ptype) + pat.ports["out"] = Port((length, 0), pi, ptype=ptype) + return pat + + +@pytest.fixture +def autotool_setup() -> tuple[Pather, AutoTool, Library]: + lib = Library() + + bend_pat = Pattern() + bend_pat.ports["in"] = Port((0, 0), 0, ptype="wire") + bend_pat.ports["out"] = Port((2, -2), pi / 2, ptype="wire") + lib["bend"] = bend_pat + lib.abstract("bend") + + via_pat = Pattern() + via_pat.ports["m1"] = Port((0, 0), 0, ptype="wire_m1") + via_pat.ports["m2"] = Port((1, 0), pi, ptype="wire_m2") + lib["via"] = via_pat + via_abs = lib.abstract("via") + + tool_m1 = AutoTool( + straights=[ + AutoTool.Straight(ptype="wire_m1", fn=lambda length: _make_transition_straight(length, ptype="wire_m1"), in_port_name="in", out_port_name="out") + ], + bends=[], + sbends=[], + transitions={("wire_m2", "wire_m1"): AutoTool.Transition(via_abs, "m2", "m1")}, + default_out_ptype="wire_m1", + ) + + p = Pather(lib, tools=tool_m1) + p.ports["start"] = Port((0, 0), pi, ptype="wire_m2") + + return p, tool_m1, lib + +def test_autotool_transition(autotool_setup: tuple[Pather, AutoTool, Library]) -> None: + p, _tool, _lib = autotool_setup + + p.straight("start", 10) + + # Via length is 1, so the remaining wire_m1 straight length is 9. + assert_allclose(p.ports["start"].offset, [10, 0], atol=1e-10) + assert p.ports["start"].ptype == "wire_m1" def make_straight(length, width=2, ptype="wire"): pat = Pattern() @@ -17,15 +66,13 @@ def make_straight(length, width=2, ptype="wire"): def make_bend(R, width=2, ptype="wire", clockwise=True): pat = Pattern() - # 90 degree arc approximation (just two rects for start and end) + # Rectangular approximation of a 90 degree bend. if clockwise: - # (0,0) rot 0 to (R, -R) rot pi/2 pat.rect((1, 0), xmin=0, xmax=R, yctr=0, ly=width) pat.rect((1, 0), xctr=R, lx=width, ymin=-R, ymax=0) pat.ports["A"] = Port((0, 0), 0, ptype=ptype) pat.ports["B"] = Port((R, -R), pi/2, ptype=ptype) else: - # (0,0) rot 0 to (R, R) rot -pi/2 pat.rect((1, 0), xmin=0, xmax=R, yctr=0, ly=width) pat.rect((1, 0), xctr=R, lx=width, ymin=0, ymax=R) pat.ports["A"] = Port((0, 0), 0, ptype=ptype) @@ -36,18 +83,14 @@ def make_bend(R, width=2, ptype="wire", clockwise=True): def multi_bend_tool(): lib = Library() - # Bend 1: R=2 lib["b1"] = make_bend(2, ptype="wire") b1_abs = lib.abstract("b1") - # Bend 2: R=5 lib["b2"] = make_bend(5, ptype="wire") b2_abs = lib.abstract("b2") tool = AutoTool( straights=[ - # Straight 1: only for length < 10 AutoTool.Straight(ptype="wire", fn=make_straight, in_port_name="A", out_port_name="B", length_range=(0, 10)), - # Straight 2: for length >= 10 AutoTool.Straight(ptype="wire", fn=lambda l: make_straight(l, width=4), in_port_name="A", out_port_name="B", length_range=(10, 1e8)) ], bends=[ @@ -60,7 +103,6 @@ def multi_bend_tool(): ) return tool, lib - @pytest.fixture def asymmetric_transition_tool() -> AutoTool: lib = Library() @@ -102,7 +144,6 @@ def asymmetric_transition_tool() -> AutoTool: default_out_ptype="core", ).add_complementary_transitions() - def assert_trace_matches_plan(plan_port: Port, tree: Library, port_names: tuple[str, str] = ("A", "B")) -> None: pat = tree.top_pattern() out_port = pat[port_names[1]] @@ -113,20 +154,15 @@ def assert_trace_matches_plan(plan_port: Port, tree: Library, port_names: tuple[ assert_allclose(rot, plan_port.rotation) assert out_port.ptype == plan_port.ptype - def test_autotool_planL_selection(multi_bend_tool) -> None: tool, _ = multi_bend_tool - # Small length: should pick straight 1 and bend 1 (R=2) - # L = straight + R. If L=5, straight=3. p, data = tool.planL(True, 5) assert data.straight.length_range == (0, 10) assert data.straight_length == 3 assert data.bend.abstract.name == "b1" assert_allclose(p.offset, [5, 2]) - # Large length: should pick straight 2 and bend 1 (R=2) - # If L=15, straight=13. p, data = tool.planL(True, 15) assert data.straight.length_range == (10, 1e8) assert data.straight_length == 13 @@ -178,11 +214,6 @@ def test_autotool_traceL_matches_plan_with_post_bend_transition(ccw: bool) -> No def test_autotool_planU_consistency(multi_bend_tool) -> None: tool, lib = multi_bend_tool - # length=10, jog=20. - # U-turn: Straight1 -> Bend1 -> Straight_mid -> Straight3(0) -> Bend2 - # X = L1_total - R2 = length - # Y = R1 + L2_mid + R2 = jog - p, data = tool.planU(20, length=10) assert data.ldata0.straight_length == 7 assert data.ldata0.bend.abstract.name == "b2" @@ -190,7 +221,6 @@ def test_autotool_planU_consistency(multi_bend_tool) -> None: assert data.ldata1.straight_length == 0 assert data.ldata1.bend.abstract.name == "b1" - def test_autotool_traceU_matches_plan_with_asymmetric_transition(asymmetric_transition_tool: AutoTool) -> None: tool = asymmetric_transition_tool @@ -202,14 +232,9 @@ def test_autotool_traceU_matches_plan_with_asymmetric_transition(asymmetric_tran tree = tool.traceU(12, length=0, in_ptype="core") assert_trace_matches_plan(plan_port, tree) - def test_autotool_planS_double_L(multi_bend_tool) -> None: tool, lib = multi_bend_tool - # length=20, jog=10. S-bend (ccw1, cw2) - # X = L1_total + R2 = length - # Y = R1 + L2_mid + R2 = jog - p, data = tool.planS(20, 10) assert_allclose(p.offset, [20, 10]) assert_allclose(p.rotation, pi) @@ -218,7 +243,6 @@ def test_autotool_planS_double_L(multi_bend_tool) -> None: assert data.ldata1.straight_length == 0 assert data.l2_length == 6 - def test_autotool_traceS_double_l_matches_plan_with_asymmetric_transition(asymmetric_transition_tool: AutoTool) -> None: tool = asymmetric_transition_tool @@ -231,7 +255,6 @@ def test_autotool_traceS_double_l_matches_plan_with_asymmetric_transition(asymme tree = tool.traceS(4, 10, in_ptype="core") assert_trace_matches_plan(plan_port, tree) - def test_autotool_planS_pure_sbend_with_transition_dx() -> None: lib = Library() @@ -285,65 +308,3 @@ def test_autotool_planS_pure_sbend_with_transition_dx() -> None: assert data.straight_length == 0 assert data.jog_remaining == 4 assert data.in_transition is not None - - -def test_renderpather_autotool_double_L(multi_bend_tool) -> None: - tool, lib = multi_bend_tool - rp = Pather(lib, tools=tool, auto_render=False) - rp.ports["A"] = Port((0,0), 0, ptype="wire") - - # This should trigger double-L fallback in planS - rp.jog("A", 10, length=20) - - # port_rot=0 -> forward is -x. jog=10 (left) is -y. - assert_allclose(rp.ports["A"].offset, [-20, -10]) - assert_allclose(rp.ports["A"].rotation, 0) # jog rot is pi relative to input, input rot is pi relative to port. - # Wait, planS returns out_port at (length, jog) rot pi relative to input (0,0) rot 0. - # Input rot relative to port is pi. - # Rotate (length, jog) rot pi by pi: (-length, -jog) rot 0. Correct. - - rp.render() - assert len(rp.pattern.refs) > 0 - -def test_pather_uturn_fallback_no_heuristic(multi_bend_tool) -> None: - tool, lib = multi_bend_tool - - class BasicTool(AutoTool): - def planU(self, *args, **kwargs): - raise NotImplementedError() - - tool_basic = BasicTool( - straights=tool.straights, - bends=tool.bends, - sbends=tool.sbends, - transitions=tool.transitions, - default_out_ptype=tool.default_out_ptype - ) - - p = Pather(lib, tools=tool_basic) - p.ports["A"] = Port((0,0), 0, ptype="wire") # facing West (Actually East points Inwards, West is Extension) - - # uturn jog=10, length=5. - # R=2. L1 = 5+2=7. L2 = 10-2=8. - p.uturn("A", 10, length=5) - - # port_rot=0 -> forward is -x. jog=10 (left) is -y. - # L1=7 along -x -> (-7, 0). Bend1 (ccw) -> rot -pi/2 (South). - # L2=8 along -y -> (-7, -8). Bend2 (ccw) -> rot 0 (East). - # wait. CCW turn from facing South (-y): turn towards East (+x). - # Wait. - # Input facing -x. CCW turn -> face -y. - # Input facing -y. CCW turn -> face +x. - # So final rotation is 0. - # Bend1 (ccw) relative to -x: global offset is (-7, -2)? - # Let's re-run my manual calculation. - # Port rot 0. Wire input rot pi. Wire output relative to input: - # L1=7, R1=2, CCW=True. Output (7, 2) rot pi/2. - # Rotate wire by pi: output (-7, -2) rot 3pi/2. - # Second turn relative to (-7, -2) rot 3pi/2: - # local output (8, 2) rot pi/2. - # global: (-7, -2) + 8*rot(3pi/2)*x + 2*rot(3pi/2)*y - # = (-7, -2) + 8*(0, -1) + 2*(1, 0) = (-7, -2) + (0, -8) + (2, 0) = (-5, -10). - # YES! ACTUAL result was (-5, -10). - assert_allclose(p.ports["A"].offset, [-5, -10]) - assert_allclose(p.ports["A"].rotation, pi) diff --git a/masque/test/test_boolean.py b/masque/test/test_boolean.py index 0249c64..e2fd7ea 100644 --- a/masque/test/test_boolean.py +++ b/masque/test/test_boolean.py @@ -48,12 +48,7 @@ def test_layer_as_polygons_flatten() -> None: polys = parent.layer_as_polygons((1, 0), flatten=True, library=lib) assert len(polys) == 1 - # Original child at (0,0) with rot pi/2 is still at (0,0) in its own space? - # No, ref.as_pattern(child) will apply the transform. - # Child (0,0), (1,0), (1,1) rotated pi/2 around (0,0) -> (0,0), (0,1), (-1,1) - # Then offset by (10,10) -> (10,10), (10,11), (9,11) - - # Let's verify the vertices + # Child vertices are rotated by the ref and then translated by the ref offset. expected = numpy.array([[10, 10], [10, 11], [9, 11]]) assert_allclose(polys[0].vertices, expected, atol=1e-10) diff --git a/masque/test/test_circle.py b/masque/test/test_circle.py new file mode 100644 index 0000000..1b4158e --- /dev/null +++ b/masque/test/test_circle.py @@ -0,0 +1,17 @@ +from numpy.testing import assert_equal, assert_allclose + +from ..shapes import Circle, Polygon + + +def test_circle_init() -> None: + c = Circle(radius=10, offset=(5, 5)) + assert c.radius == 10 + assert_equal(c.offset, [5, 5]) + +def test_circle_to_polygons() -> None: + c = Circle(radius=10) + polys = c.to_polygons(num_vertices=32) + assert len(polys) == 1 + assert isinstance(polys[0], Polygon) + bounds = polys[0].get_bounds_single() + assert_allclose(bounds, [[-10, -10], [10, 10]], atol=1e-10) diff --git a/masque/test/test_curve_polygonization.py b/masque/test/test_curve_polygonization.py new file mode 100644 index 0000000..aaf4070 --- /dev/null +++ b/masque/test/test_curve_polygonization.py @@ -0,0 +1,26 @@ +from numpy import pi + +from ..shapes import Arc, Circle, Ellipse +from .helpers import assert_closed_edges_within + + +def test_shape_arclen() -> None: + e = Ellipse(radii=(10, 5)) + polys = e.to_polygons(max_arclen=5) + v = polys[0].vertices + assert_closed_edges_within(v, 5) + assert len(v) > 10 + + a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) + polys = a.to_polygons(max_arclen=2) + assert_closed_edges_within(polys[0].vertices, 2) + +def test_curve_polygonizers_clamp_large_max_arclen() -> None: + for shape in ( + Circle(radius=10), + Ellipse(radii=(10, 20)), + Arc(radii=(10, 20), angles=(0, 1), width=2), + ): + polys = shape.to_polygons(num_vertices=None, max_arclen=1e9) + assert len(polys) == 1 + assert len(polys[0].vertices) >= 3 diff --git a/masque/test/test_dxf.py b/masque/test/test_dxf.py index f6dd177..442b62f 100644 --- a/masque/test/test_dxf.py +++ b/masque/test/test_dxf.py @@ -26,19 +26,16 @@ def test_dxf_roundtrip(tmp_path: Path): lib = Library() pat = Pattern() - # 1. Polygon (closed) poly_verts = numpy.array([[0, 0], [10, 0], [10, 10], [0, 10]]) pat.polygon("1", vertices=poly_verts) - # 2. Path (open, 3 points) path_verts = numpy.array([[20, 0], [30, 0], [30, 10]]) pat.path("2", vertices=path_verts, width=2) - # 3. Path (open, 2 points) - Testing the fix for 2-point polylines + # Two-point paths remain paths rather than being polygonized. path2_verts = numpy.array([[40, 0], [50, 10]]) - pat.path("3", vertices=path2_verts, width=0) # width 0 to be sure it's not a polygonized path if we're not careful + pat.path("3", vertices=path2_verts, width=0) - # 4. Ref with Grid repetition (Manhattan) subpat = Pattern() subpat.polygon("sub", vertices=[[0, 0], [1, 0], [1, 1]]) lib["sub"] = subpat @@ -52,38 +49,29 @@ def test_dxf_roundtrip(tmp_path: Path): read_lib, _ = dxf.readfile(dxf_file) - # In DXF read, the top level is usually called "Model" top_pat = read_lib.get("Model") or read_lib.get("top") or list(read_lib.values())[0] - # Verify Polygon polys = [s for s in top_pat.shapes["1"] if isinstance(s, Polygon)] assert len(polys) >= 1 poly_read = polys[0] assert _matches_closed_vertices(poly_read.vertices, poly_verts) - # Verify 3-point Path paths = [s for s in top_pat.shapes["2"] if isinstance(s, MPath)] assert len(paths) >= 1 path_read = paths[0] assert _matches_open_path(path_read.vertices, path_verts) assert path_read.width == 2 - # Verify 2-point Path paths2 = [s for s in top_pat.shapes["3"] if isinstance(s, MPath)] assert len(paths2) >= 1 path2_read = paths2[0] assert _matches_open_path(path2_read.vertices, path2_verts) assert path2_read.width == 0 - # Verify Ref with Grid - # Finding the sub pattern name might be tricky because of how DXF stores blocks - # but "sub" should be in read_lib assert "sub" in read_lib - # Check refs in the top pattern found_grid = False for target, reflist in top_pat.refs.items(): - # DXF names might be case-insensitive or modified, but ezdxf usually preserves them if target.upper() == "SUB": for ref in reflist: if isinstance(ref.repetition, Grid): @@ -95,16 +83,12 @@ def test_dxf_roundtrip(tmp_path: Path): assert found_grid, f"Manhattan Grid repetition should have been preserved. Targets: {list(top_pat.refs.keys())}" def test_dxf_manhattan_precision(tmp_path: Path): - # Test that float precision doesn't break Manhattan grid detection lib = Library() sub = Pattern() sub.polygon("1", vertices=[[0, 0], [1, 0], [1, 1]]) lib["sub"] = sub top = Pattern() - # 90 degree rotation: in masque the grid is NOT rotated, so it stays [[10,0],[0,10]] - # In DXF, an array with rotation 90 has basis vectors [[0,10],[-10,0]]. - # So a masque grid [[10,0],[0,10]] with ref rotation 90 matches a DXF array. angle = numpy.pi / 2 # 90 degrees top.ref("sub", offset=(0, 0), rotation=angle, repetition=Grid(a_vector=(10, 0), a_count=2, b_vector=(0, 10), b_count=2)) @@ -114,7 +98,7 @@ def test_dxf_manhattan_precision(tmp_path: Path): dxf_file = tmp_path / "precision.dxf" dxf.writefile(lib, "top", dxf_file) - # If the isclose() fix works, this should still be a Grid when read back + # Near-integer rotated basis vectors round-trip as a Manhattan Grid. read_lib, _ = dxf.readfile(dxf_file) read_top = read_lib.get("Model") or read_lib.get("top") or list(read_lib.values())[0] diff --git a/masque/test/test_ellipse.py b/masque/test/test_ellipse.py new file mode 100644 index 0000000..1125fb1 --- /dev/null +++ b/masque/test/test_ellipse.py @@ -0,0 +1,29 @@ +from numpy import pi +from numpy.testing import assert_equal, assert_allclose + +from ..shapes import Ellipse + + +def test_ellipse_init() -> None: + e = Ellipse(radii=(10, 5), offset=(1, 2), rotation=pi / 4) + assert_equal(e.radii, [10, 5]) + assert_equal(e.offset, [1, 2]) + assert e.rotation == pi / 4 + +def test_ellipse_to_polygons() -> None: + e = Ellipse(radii=(10, 5)) + polys = e.to_polygons(num_vertices=64) + assert len(polys) == 1 + bounds = polys[0].get_bounds_single() + assert_allclose(bounds, [[-10, -5], [10, 5]], atol=1e-10) + +def test_rotated_ellipse_bounds_match_polygonized_geometry() -> None: + ellipse = Ellipse(radii=(10, 20), rotation=pi / 4, offset=(100, 200)) + bounds = ellipse.get_bounds_single() + poly_bounds = ellipse.to_polygons(num_vertices=8192)[0].get_bounds_single() + assert_allclose(bounds, poly_bounds, atol=1e-3) + +def test_ellipse_integer_radii_scale_cleanly() -> None: + ellipse = Ellipse(radii=(10, 20)) + ellipse.scale_by(0.5) + assert_allclose(ellipse.radii, [5, 10]) diff --git a/masque/test/test_file_roundtrip.py b/masque/test/test_file_format_roundtrip.py similarity index 70% rename from masque/test/test_file_roundtrip.py rename to masque/test/test_file_format_roundtrip.py index 2cfb0d1..6bd079d 100644 --- a/masque/test/test_file_roundtrip.py +++ b/masque/test/test_file_format_roundtrip.py @@ -11,45 +11,31 @@ from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: lib = Library() - # 1. Polygons pat_poly = Pattern() pat_poly.polygon((1, 0), vertices=[[0, 0], [10, 0], [5, 10]]) lib["polygons"] = pat_poly - # 2. Paths with different endcaps pat_paths = Pattern() - # Flush pat_paths.path((2, 0), vertices=[[0, 0], [20, 0]], width=2, cap=MPath.Cap.Flush) - # Square pat_paths.path((2, 1), vertices=[[0, 10], [20, 10]], width=2, cap=MPath.Cap.Square) - # Circle (Only for GDS) if for_gds: pat_paths.path((2, 2), vertices=[[0, 20], [20, 20]], width=2, cap=MPath.Cap.Circle) - # SquareCustom pat_paths.path((2, 3), vertices=[[0, 30], [20, 30]], width=2, cap=MPath.Cap.SquareCustom, cap_extensions=(1, 5)) lib["paths"] = pat_paths - # 3. Circles (only for OASIS or polygonized for GDS) pat_circles = Pattern() if for_gds: - # GDS writer calls to_polygons() for non-supported shapes, - # but we can also pre-polygonize pat_circles.shapes[(3, 0)].append(Circle(radius=5, offset=(10, 10)).to_polygons()[0]) else: pat_circles.shapes[(3, 0)].append(Circle(radius=5, offset=(10, 10))) lib["circles"] = pat_circles - # 4. Refs with repetitions pat_refs = Pattern() - # Simple Ref pat_refs.ref("polygons", offset=(0, 0)) - # Ref with Grid repetition pat_refs.ref("polygons", offset=(100, 0), repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 20), b_count=2)) - # Ref with Arbitrary repetition pat_refs.ref("polygons", offset=(0, 100), repetition=Arbitrary(displacements=[[0, 0], [10, 20], [30, -10]])) lib["refs"] = pat_refs - # 5. Shapes with repetitions (OASIS only, must be wrapped for GDS) pat_rep_shapes = Pattern() poly_rep = Polygon(vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], repetition=Grid(a_vector=(10, 0), a_count=5)) pat_rep_shapes.shapes[(4, 0)].append(poly_rep) @@ -68,16 +54,10 @@ def test_gdsii_full_roundtrip(tmp_path: Path) -> None: read_lib, _ = gdsii.readfile(gds_file) - # Check existence for name in lib: assert name in read_lib - # Check Paths read_paths = read_lib["paths"] - # Check caps (GDS stores them as path_type) - # Order might be different depending on how they were written, - # but here they should match the order they were added if dict order is preserved. - # Actually, they are grouped by layer. p_flush = cast("MPath", read_paths.shapes[(2, 0)][0]) assert p_flush.cap == MPath.Cap.Flush @@ -92,20 +72,16 @@ def test_gdsii_full_roundtrip(tmp_path: Path) -> None: assert p_custom.cap_extensions is not None assert_allclose(p_custom.cap_extensions, (1, 5)) - # Check Refs with repetitions read_refs = read_lib["refs"] assert len(read_refs.refs["polygons"]) >= 3 # Simple, Grid (becomes 1 AREF), Arbitrary (becomes 3 SREFs) - # AREF check arefs = [r for r in read_refs.refs["polygons"] if r.repetition is not None] assert len(arefs) == 1 assert isinstance(arefs[0].repetition, Grid) assert arefs[0].repetition.a_count == 3 assert arefs[0].repetition.b_count == 2 - # Check wrapped shapes - # lib.wrap_repeated_shapes() created new patterns - # Original pattern "rep_shapes" now should have a Ref + # GDS stores repeated shapes through refs created by wrap_repeated_shapes(). assert len(read_lib["rep_shapes"].refs) > 0 def test_oasis_full_roundtrip(tmp_path: Path) -> None: @@ -117,34 +93,17 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: read_lib, _ = oasis.readfile(oas_file) - # Check existence for name in lib: assert name in read_lib - # Check Circle read_circles = read_lib["circles"] assert isinstance(read_circles.shapes[(3, 0)][0], Circle) assert read_circles.shapes[(3, 0)][0].radius == 5 - # Check Path caps read_paths = read_lib["paths"] assert cast("MPath", read_paths.shapes[(2, 0)][0]).cap == MPath.Cap.Flush assert cast("MPath", read_paths.shapes[(2, 1)][0]).cap == MPath.Cap.Square - # OASIS HalfWidth is Square. masque's Square is also HalfWidth extension. - # Wait, Circle cap in OASIS? - # masque/file/oasis.py: - # path_cap_map = { - # PathExtensionScheme.Flush: Path.Cap.Flush, - # PathExtensionScheme.HalfWidth: Path.Cap.Square, - # PathExtensionScheme.Arbitrary: Path.Cap.SquareCustom, - # } - # It seems Circle cap is NOT supported in OASIS by masque currently. - # Let's verify what happens with Circle cap in OASIS write. - # _shapes_to_elements in oasis.py: - # path_type = next(k for k, v in path_cap_map.items() if v == shape.cap) - # This will raise StopIteration if Circle is not in path_cap_map. - # Check Shape repetition read_rep_shapes = read_lib["rep_shapes"] poly = read_rep_shapes.shapes[(4, 0)][0] assert poly.repetition is not None diff --git a/masque/test/test_manhattanize.py b/masque/test/test_manhattanize.py new file mode 100644 index 0000000..d0f4fa7 --- /dev/null +++ b/masque/test/test_manhattanize.py @@ -0,0 +1,17 @@ +import pytest +import numpy + +from ..shapes import Polygon + + +def test_manhattanize() -> None: + pytest.importorskip("float_raster") + pytest.importorskip("skimage.measure") + poly = Polygon([[0, 5], [5, 10], [10, 5], [5, 0]]) + grid = numpy.arange(0, 11, 1) + + manhattan_polys = poly.manhattanize(grid, grid) + assert len(manhattan_polys) >= 1 + for mp in manhattan_polys: + dv = numpy.diff(mp.vertices, axis=0) + assert numpy.all((dv[:, 0] == 0) | (dv[:, 1] == 0)) diff --git a/masque/test/test_path.py b/masque/test/test_path.py index 1cdd872..f8685f4 100644 --- a/masque/test/test_path.py +++ b/masque/test/test_path.py @@ -1,6 +1,6 @@ from numpy.testing import assert_equal, assert_allclose -from ..shapes import Path +from ..shapes import Path, Path as MPath def test_path_init() -> None: @@ -14,7 +14,6 @@ def test_path_to_polygons_flush() -> None: p = Path(vertices=[[0, 0], [10, 0]], width=2, cap=Path.Cap.Flush) polys = p.to_polygons() assert len(polys) == 1 - # Rectangle from (0, -1) to (10, 1) bounds = polys[0].get_bounds_single() assert_equal(bounds, [[0, -1], [10, 1]]) @@ -23,8 +22,6 @@ def test_path_to_polygons_square() -> None: p = Path(vertices=[[0, 0], [10, 0]], width=2, cap=Path.Cap.Square) polys = p.to_polygons() assert len(polys) == 1 - # Square cap adds width/2 = 1 to each end - # Rectangle from (-1, -1) to (11, 1) bounds = polys[0].get_bounds_single() assert_equal(bounds, [[-1, -1], [11, 1]]) @@ -32,11 +29,8 @@ def test_path_to_polygons_square() -> None: def test_path_to_polygons_circle() -> None: p = Path(vertices=[[0, 0], [10, 0]], width=2, cap=Path.Cap.Circle) polys = p.to_polygons(num_vertices=32) - # Path.to_polygons for Circle cap returns 1 polygon for the path + polygons for the caps assert len(polys) >= 3 - # Combined bounds should be from (-1, -1) to (11, 1) - # But wait, Path.get_bounds_single() handles this more directly bounds = p.get_bounds_single() assert_equal(bounds, [[-1, -1], [11, 1]]) @@ -45,32 +39,21 @@ def test_path_custom_cap() -> None: p = Path(vertices=[[0, 0], [10, 0]], width=2, cap=Path.Cap.SquareCustom, cap_extensions=(5, 10)) polys = p.to_polygons() assert len(polys) == 1 - # Extends 5 units at start, 10 at end - # Starts at -5, ends at 20 bounds = polys[0].get_bounds_single() assert_equal(bounds, [[-5, -1], [20, 1]]) def test_path_bend() -> None: - # L-shaped path p = Path(vertices=[[0, 0], [10, 0], [10, 10]], width=2) polys = p.to_polygons() assert len(polys) == 1 bounds = polys[0].get_bounds_single() - # Outer corner at (11, -1) is not right. - # Segments: (0,0)-(10,0) and (10,0)-(10,10) - # Corners of segment 1: (0,1), (10,1), (10,-1), (0,-1) - # Corners of segment 2: (9,0), (9,10), (11,10), (11,0) - # Bounds should be [[-1 (if start is square), -1], [11, 11]]? - # Flush cap start at (0,0) with width 2 means y from -1 to 1. - # Vertical segment end at (10,10) with width 2 means x from 9 to 11. - # So bounds should be x: [0, 11], y: [-1, 10] assert_equal(bounds, [[0, -1], [11, 10]]) def test_path_mirror() -> None: p = Path(vertices=[[10, 5], [20, 10]], width=2) - p.mirror(0) # Mirror across x axis (y -> -y) + p.mirror(0) assert_equal(p.vertices, [[10, -5], [20, -10]]) @@ -109,3 +92,10 @@ def test_path_normalized_form_distinguishes_custom_caps() -> None: p2 = Path(vertices=[[0, 0], [10, 0]], width=2, cap=Path.Cap.SquareCustom, cap_extensions=(3, 4)) assert p1.normalized_form(1)[0] != p2.normalized_form(1)[0] + + +def test_path_edge_cases() -> None: + p = MPath(vertices=[[0, 0], [0, 0], [10, 0]], width=2) + polys = p.to_polygons() + assert len(polys) == 1 + assert_equal(polys[0].get_bounds_single(), [[0, -1], [10, 1]]) diff --git a/masque/test/test_pather.py b/masque/test/test_pather.py deleted file mode 100644 index 47cae29..0000000 --- a/masque/test/test_pather.py +++ /dev/null @@ -1,108 +0,0 @@ -import pytest -from numpy.testing import assert_equal, assert_allclose -from numpy import pi - -from ..builder import Pather -from ..builder.tools import PathTool -from ..library import Library -from ..ports import Port - - -@pytest.fixture -def pather_setup() -> tuple[Pather, PathTool, Library]: - lib = Library() - # Simple PathTool: 2um width on layer (1,0) - tool = PathTool(layer=(1, 0), width=2, ptype="wire") - p = Pather(lib, tools=tool) - # Add an initial port facing North (pi/2) - # Port rotation points INTO device. So "North" rotation means device is North of port. - # Pathing "forward" moves South. - p.ports["start"] = Port((0, 0), pi / 2, ptype="wire") - return p, tool, lib - - -def test_pather_straight(pather_setup: tuple[Pather, PathTool, Library]) -> None: - p, tool, lib = pather_setup - # Route 10um "forward" - p.straight("start", 10) - - # port rot pi/2 (North). Travel +pi relative to port -> South. - assert_allclose(p.ports["start"].offset, [0, -10], atol=1e-10) - assert p.ports["start"].rotation is not None - assert_allclose(p.ports["start"].rotation, pi / 2, atol=1e-10) - - -def test_pather_bend(pather_setup: tuple[Pather, PathTool, Library]) -> None: - p, tool, lib = pather_setup - # Start (0,0) rot pi/2 (North). - # Path 10um "forward" (South), then turn Clockwise (ccw=False). - # Facing South, turn Right -> West. - p.cw("start", 10) - - # PathTool.planL(ccw=False, length=10) returns out_port at (10, -1) relative to (0,0) rot 0. - # Transformed by port rot pi/2 (North) + pi (to move "forward" away from device): - # Transformation rot = pi/2 + pi = 3pi/2. - # (10, -1) rotated 3pi/2: (x,y) -> (y, -x) -> (-1, -10). - - assert_allclose(p.ports["start"].offset, [-1, -10], atol=1e-10) - # North (pi/2) + CW (90 deg) -> West (pi)? - # Actual behavior results in 0 (East) - apparently rotation is flipped. - assert p.ports["start"].rotation is not None - assert_allclose(p.ports["start"].rotation, 0, atol=1e-10) - - -def test_pather_path_to(pather_setup: tuple[Pather, PathTool, Library]) -> None: - p, tool, lib = pather_setup - # start at (0,0) rot pi/2 (North) - # path "forward" (South) to y=-50 - p.straight("start", y=-50) - assert_equal(p.ports["start"].offset, [0, -50]) - - -def test_pather_mpath(pather_setup: tuple[Pather, PathTool, Library]) -> None: - p, tool, lib = pather_setup - p.ports["A"] = Port((0, 0), pi / 2, ptype="wire") - p.ports["B"] = Port((10, 0), pi / 2, ptype="wire") - - # Path both "forward" (South) to y=-20 - p.straight(["A", "B"], ymin=-20) - assert_equal(p.ports["A"].offset, [0, -20]) - assert_equal(p.ports["B"].offset, [10, -20]) - - -def test_pather_at_chaining(pather_setup: tuple[Pather, PathTool, Library]) -> None: - p, tool, lib = pather_setup - # Fluent API test - p.at("start").straight(10).ccw(10) - # 10um South -> (0, -10) rot pi/2 - # then 10um South and turn CCW (Facing South, CCW is East) - # PathTool.planL(ccw=True, length=10) -> out_port=(10, 1) rot -pi/2 relative to rot 0 - # Transform (10, 1) by 3pi/2: (x,y) -> (y, -x) -> (1, -10) - # (0, -10) + (1, -10) = (1, -20) - assert_allclose(p.ports["start"].offset, [1, -20], atol=1e-10) - # pi/2 (North) + CCW (90 deg) -> 0 (East)? - # Actual behavior results in pi (West). - assert p.ports["start"].rotation is not None - assert_allclose(p.ports["start"].rotation, pi, atol=1e-10) - - -def test_pather_dead_ports() -> None: - lib = Library() - tool = PathTool(layer=(1, 0), width=1) - p = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool) - p.set_dead() - - # Path with negative length (impossible for PathTool, would normally raise BuildError) - p.straight("in", -10) - - # Port 'in' should be updated by dummy extension despite tool failure - # port_rot=0, forward is -x. path(-10) means moving -10 in -x direction -> +10 in x. - assert_allclose(p.ports["in"].offset, [10, 0], atol=1e-10) - - # Downstream path should work correctly using the dummy port location - p.straight("in", 20) - # 10 + (-20) = -10 - assert_allclose(p.ports["in"].offset, [-10, 0], atol=1e-10) - - # Verify no geometry - assert not p.pattern.has_shapes() diff --git a/masque/test/test_pather_api.py b/masque/test/test_pather_api.py deleted file mode 100644 index d01ed44..0000000 --- a/masque/test/test_pather_api.py +++ /dev/null @@ -1,936 +0,0 @@ -from typing import Any - -import pytest -import numpy -from numpy import pi -from masque import Pather, Library, Pattern, Port -from masque.builder.tools import PathTool, Tool -from masque.error import BuildError, PortError, PatternError - -def test_pather_trace_basic() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - - # Port rotation 0 points in +x (INTO device). - # To extend it, we move in -x direction. - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - # Trace single port - p.at('A').trace(None, 5000) - assert numpy.allclose(p.pattern.ports['A'].offset, (-5000, 0)) - - # Trace with bend - p.at('A').trace(True, 5000) # CCW bend - # Port was at (-5000, 0) rot 0. - # New wire starts at (-5000, 0) rot 0. - # Output port of wire before rotation: (5000, 500) rot -pi/2 - # Rotate by pi (since dev port rot is 0 and tool port rot is 0): - # (-5000, -500) rot pi - pi/2 = pi/2 - # Add to start: (-10000, -500) rot pi/2 - assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, -500)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, pi/2) - -def test_pather_trace_to() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - # Trace to x=-10000 - p.at('A').trace_to(None, x=-10000) - assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) - - # Trace to position=-20000 - p.at('A').trace_to(None, p=-20000) - assert numpy.allclose(p.pattern.ports['A'].offset, (-20000, 0)) - -def test_pather_bundle_trace() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - - p.pattern.ports['A'] = Port((0, 0), rotation=0) - p.pattern.ports['B'] = Port((0, 2000), rotation=0) - - # Straight bundle - all should align to same x - p.at(['A', 'B']).straight(xmin=-10000) - assert numpy.isclose(p.pattern.ports['A'].offset[0], -10000) - assert numpy.isclose(p.pattern.ports['B'].offset[0], -10000) - - # Bundle with bend - p.at(['A', 'B']).ccw(xmin=-20000, spacing=2000) - # Traveling in -x direction. CCW turn turns towards -y. - # A is at y=0, B is at y=2000. - # Rotation center is at y = -R. - # A is closer to center than B. So A is inner, B is outer. - # xmin is coordinate of innermost bend (A). - assert numpy.isclose(p.pattern.ports['A'].offset[0], -20000) - # B's bend is further out (more negative x) - assert numpy.isclose(p.pattern.ports['B'].offset[0], -22000) - -def test_pather_each_bound() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - - p.pattern.ports['A'] = Port((0, 0), rotation=0) - p.pattern.ports['B'] = Port((-1000, 2000), rotation=0) - - # Each should move by 5000 (towards -x) - p.at(['A', 'B']).trace(None, each=5000) - assert numpy.allclose(p.pattern.ports['A'].offset, (-5000, 0)) - assert numpy.allclose(p.pattern.ports['B'].offset, (-6000, 2000)) - -def test_selection_management() -> None: - lib = Library() - p = Pather(lib) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - p.pattern.ports['B'] = Port((0, 0), rotation=0) - - pp = p.at('A') - assert pp.ports == ['A'] - - pp.select('B') - assert pp.ports == ['A', 'B'] - - pp.deselect('A') - assert pp.ports == ['B'] - - pp.select(['A']) - assert pp.ports == ['B', 'A'] - - pp.drop() - assert 'A' not in p.pattern.ports - assert 'B' not in p.pattern.ports - assert pp.ports == [] - -def test_mark_fork() -> None: - lib = Library() - p = Pather(lib) - p.pattern.ports['A'] = Port((100, 200), rotation=1) - - pp = p.at('A') - pp.mark('B') - assert 'B' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['B'].offset, (100, 200)) - assert p.pattern.ports['B'].rotation == 1 - assert pp.ports == ['A'] # mark keeps current selection - - pp.fork('C') - assert 'C' in p.pattern.ports - assert pp.ports == ['C'] # fork switches to new name - - -def test_mark_fork_reject_overwrite_and_duplicate_targets() -> None: - lib = Library() - - p_mark = Pather(lib, pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0), - 'C': Port((2, 0), rotation=0), - })) - with pytest.raises(PortError, match='overwrite existing ports'): - p_mark.at('A').mark('C') - assert numpy.allclose(p_mark.pattern.ports['C'].offset, (2, 0)) - - p_fork = Pather(lib, pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0), - 'B': Port((1, 0), rotation=0), - })) - pp = p_fork.at(['A', 'B']) - with pytest.raises(PortError, match='targets would collide'): - pp.fork({'A': 'X', 'B': 'X'}) - assert set(p_fork.pattern.ports) == {'A', 'B'} - assert pp.ports == ['A', 'B'] - - -def test_mark_fork_dead_overwrite_and_duplicate_targets() -> None: - lib = Library() - p = Pather(lib, pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0), - 'B': Port((1, 0), rotation=0), - 'C': Port((2, 0), rotation=0), - })) - p.set_dead() - - p.at('A').mark('C') - assert numpy.allclose(p.pattern.ports['C'].offset, (0, 0)) - - pp = p.at(['A', 'B']) - pp.fork({'A': 'X', 'B': 'X'}) - assert numpy.allclose(p.pattern.ports['X'].offset, (1, 0)) - assert pp.ports == ['X'] - - -def test_mark_fork_reject_missing_sources() -> None: - lib = Library() - p = Pather(lib, pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0), - 'B': Port((1, 0), rotation=0), - })) - - with pytest.raises(PortError, match='selected ports'): - p.at(['A', 'B']).mark({'Z': 'C'}) - - with pytest.raises(PortError, match='selected ports'): - p.at(['A', 'B']).fork({'Z': 'C'}) - -def test_rename() -> None: - lib = Library() - p = Pather(lib) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - p.at('A').rename('B') - assert 'A' not in p.pattern.ports - assert 'B' in p.pattern.ports - - p.pattern.ports['C'] = Port((0, 0), rotation=0) - pp = p.at(['B', 'C']) - pp.rename({'B': 'D', 'C': 'E'}) - assert 'B' not in p.pattern.ports - assert 'C' not in p.pattern.ports - assert 'D' in p.pattern.ports - assert 'E' in p.pattern.ports - assert set(pp.ports) == {'D', 'E'} - -def test_renderpather_uturn_fallback() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - rp = Pather(lib, tools=tool, auto_render=False) - rp.pattern.ports['A'] = Port((0, 0), rotation=0) - - # PathTool doesn't implement planU, so it should fall back to two planL calls - rp.at('A').uturn(offset=10000, length=5000) - - # Two steps should be added - assert len(rp.paths['A']) == 2 - assert rp.paths['A'][0].opcode == 'L' - assert rp.paths['A'][1].opcode == 'L' - - rp.render() - assert rp.pattern.ports['A'].rotation is not None - assert numpy.isclose(rp.pattern.ports['A'].rotation, pi) - -def test_autotool_uturn() -> None: - from masque.builder.tools import AutoTool - lib = Library() - - # Setup AutoTool with a simple straight and a bend - def make_straight(length: float) -> Pattern: - pat = Pattern() - pat.rect(layer='M1', xmin=0, xmax=length, yctr=0, ly=1000) - pat.ports['in'] = Port((0, 0), 0) - pat.ports['out'] = Port((length, 0), pi) - return pat - - bend_pat = Pattern() - bend_pat.polygon(layer='M1', vertices=[(0, -500), (0, 500), (1000, -500)]) - bend_pat.ports['in'] = Port((0, 0), 0) - bend_pat.ports['out'] = Port((500, -500), pi/2) - lib['bend'] = bend_pat - - tool = AutoTool( - straights=[AutoTool.Straight(ptype='wire', fn=make_straight, in_port_name='in', out_port_name='out')], - bends=[AutoTool.Bend(abstract=lib.abstract('bend'), in_port_name='in', out_port_name='out', clockwise=True)], - sbends=[], - transitions={}, - default_out_ptype='wire' - ) - - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), 0) - - # CW U-turn (jog < 0) - # R = 500. jog = -2000. length = 1000. - # p0 = planL(length=1000) -> out at (1000, -500) rot pi/2 - # R2 = 500. - # l2_length = abs(-2000) - abs(-500) - 500 = 1000. - p.at('A').uturn(offset=-2000, length=1000) - - # Final port should be at (-1000, 2000) rot pi - # Start: (0,0) rot 0. Wire direction is rot + pi = pi (West, -x). - # Tool planU returns (length, jog) = (1000, -2000) relative to (0,0) rot 0. - # Rotation of pi transforms (1000, -2000) to (-1000, 2000). - # Final rotation: 0 + pi = pi. - assert numpy.allclose(p.pattern.ports['A'].offset, (-1000, 2000)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, pi) - -def test_pather_trace_into() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - - # 1. Straight connector - p.pattern.ports['A'] = Port((0, 0), rotation=0) - p.pattern.ports['B'] = Port((-10000, 0), rotation=pi) - p.at('A').trace_into('B', plug_destination=False) - assert 'B' in p.pattern.ports - assert 'A' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) - - # 2. Single bend - p.pattern.ports['C'] = Port((0, 0), rotation=0) - p.pattern.ports['D'] = Port((-5000, 5000), rotation=pi/2) - p.at('C').trace_into('D', plug_destination=False) - assert 'D' in p.pattern.ports - assert 'C' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['C'].offset, (-5000, 5000)) - - # 3. Jog (S-bend) - p.pattern.ports['E'] = Port((0, 0), rotation=0) - p.pattern.ports['F'] = Port((-10000, 2000), rotation=pi) - p.at('E').trace_into('F', plug_destination=False) - assert 'F' in p.pattern.ports - assert 'E' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['E'].offset, (-10000, 2000)) - - # 4. U-bend (0 deg angle) - p.pattern.ports['G'] = Port((0, 0), rotation=0) - p.pattern.ports['H'] = Port((-10000, 2000), rotation=0) - p.at('G').trace_into('H', plug_destination=False) - assert 'H' in p.pattern.ports - assert 'G' in p.pattern.ports - # A U-bend with length=-travel=10000 and jog=-2000 from (0,0) rot 0 - # ends up at (-10000, 2000) rot pi. - assert numpy.allclose(p.pattern.ports['G'].offset, (-10000, 2000)) - assert p.pattern.ports['G'].rotation is not None - assert numpy.isclose(p.pattern.ports['G'].rotation, pi) - - # 5. Vertical straight connector - p.pattern.ports['I'] = Port((0, 0), rotation=pi / 2) - p.pattern.ports['J'] = Port((0, -10000), rotation=3 * pi / 2) - p.at('I').trace_into('J', plug_destination=False) - assert 'J' in p.pattern.ports - assert 'I' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['I'].offset, (0, -10000)) - assert p.pattern.ports['I'].rotation is not None - assert numpy.isclose(p.pattern.ports['I'].rotation, pi / 2) - - -def test_pather_trace_into_dead_updates_ports_without_geometry() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.pattern.ports['B'] = Port((-10000, 0), rotation=pi, ptype='wire') - p.set_dead() - - p.trace_into('A', 'B', plug_destination=False) - - assert set(p.pattern.ports) == {'A', 'B'} - assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - assert len(p.paths['A']) == 0 - assert not p.pattern.has_shapes() - assert not p.pattern.has_refs() - - -def test_pather_dead_fallback_preserves_out_ptype() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.set_dead() - - p.straight('A', -1000, out_ptype='other') - - assert numpy.allclose(p.pattern.ports['A'].offset, (1000, 0)) - assert p.pattern.ports['A'].ptype == 'other' - assert len(p.paths['A']) == 0 - - -def test_pather_dead_place_overwrites_colliding_ports_last_wins() -> None: - lib = Library() - p = Pather(lib, pattern=Pattern(ports={ - 'A': Port((5, 5), rotation=0), - 'keep': Port((9, 9), rotation=0), - })) - p.set_dead() - - other = Pattern() - other.ports['X'] = Port((1, 0), rotation=0) - other.ports['Y'] = Port((2, 0), rotation=pi / 2) - - p.place(other, port_map={'X': 'A', 'Y': 'A'}) - - assert set(p.pattern.ports) == {'A', 'keep'} - assert numpy.allclose(p.pattern.ports['A'].offset, (2, 0)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, pi / 2) - - -def test_pather_dead_plug_overwrites_colliding_outputs_last_wins() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000, ptype='wire') - p = Pather(lib, tools=tool, pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0, ptype='wire'), - 'B': Port((99, 99), rotation=0, ptype='wire'), - })) - p.set_dead() - - other = Pattern() - other.ports['in'] = Port((0, 0), rotation=pi, ptype='wire') - other.ports['X'] = Port((10, 0), rotation=0, ptype='wire') - other.ports['Y'] = Port((20, 0), rotation=0, ptype='wire') - - p.plug(other, map_in={'A': 'in'}, map_out={'X': 'B', 'Y': 'B'}) - - assert 'A' not in p.pattern.ports - assert 'B' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['B'].offset, (20, 0)) - assert p.pattern.ports['B'].rotation is not None - assert numpy.isclose(p.pattern.ports['B'].rotation, 0) - - -def test_pather_dead_rename_overwrites_colliding_ports_last_wins() -> None: - p = Pather(Library(), pattern=Pattern(ports={ - 'A': Port((0, 0), rotation=0), - 'B': Port((1, 0), rotation=0), - 'C': Port((2, 0), rotation=0), - })) - p.set_dead() - - p.rename_ports({'A': 'C', 'B': 'C'}) - - assert set(p.pattern.ports) == {'C'} - assert numpy.allclose(p.pattern.ports['C'].offset, (1, 0)) - - -def test_pather_jog_failed_fallback_is_atomic() -> None: - lib = Library() - tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='shorter than required bend'): - p.jog('A', 1.5, length=1.5) - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert p.pattern.ports['A'].rotation == 0 - assert len(p.paths['A']) == 0 - - -def test_pather_jog_accepts_sub_width_offset_when_length_is_sufficient() -> None: - lib = Library() - tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.jog('A', 1.5, length=5) - - assert numpy.allclose(p.pattern.ports['A'].offset, (-5, -1.5)) - assert p.pattern.ports['A'].rotation == 0 - assert len(p.paths['A']) == 0 - - -def test_pather_jog_length_solved_from_single_position_bound() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.jog('A', 2, x=-6) - assert numpy.allclose(p.pattern.ports['A'].offset, (-6, -2)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - - q = Pather(Library(), tools=tool) - q.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - q.jog('A', 2, p=-6) - assert numpy.allclose(q.pattern.ports['A'].offset, (-6, -2)) - - -def test_pather_jog_requires_length_or_one_position_bound() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='requires either length'): - p.jog('A', 2) - - with pytest.raises(BuildError, match='exactly one positional bound'): - p.jog('A', 2, x=-6, p=-6) - - -def test_pather_trace_to_rejects_conflicting_position_bounds() -> None: - tool = PathTool(layer='M1', width=1, ptype='wire') - - for kwargs in ({'x': -5, 'y': 2}, {'y': 2, 'x': -5}, {'p': -7, 'x': -5}): - p = Pather(Library(), tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - with pytest.raises(BuildError, match='exactly one positional bound'): - p.trace_to('A', None, **kwargs) - - p = Pather(Library(), tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - with pytest.raises(BuildError, match='length cannot be combined'): - p.trace_to('A', None, x=-5, length=3) - - -def test_pather_trace_rejects_length_with_bundle_bound() -> None: - p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='length cannot be combined'): - p.trace('A', None, length=5, xmin=-100) - - -@pytest.mark.parametrize('kwargs', ({'xmin': -10, 'xmax': -20}, {'xmax': -20, 'xmin': -10})) -def test_pather_trace_rejects_multiple_bundle_bounds(kwargs: dict[str, int]) -> None: - p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.pattern.ports['B'] = Port((0, 5), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='exactly one bundle bound'): - p.trace(['A', 'B'], None, **kwargs) - - -def test_pather_jog_rejects_length_with_position_bound() -> None: - p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='length cannot be combined'): - p.jog('A', 2, length=5, x=-999) - - -@pytest.mark.parametrize('kwargs', ({'x': -999}, {'xmin': -10})) -def test_pather_uturn_rejects_routing_bounds(kwargs: dict[str, int]) -> None: - p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='Unsupported routing bounds for uturn'): - p.uturn('A', 4, **kwargs) - - -def test_pather_uturn_none_length_defaults_to_zero() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.uturn('A', 4) - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, -4)) - assert p.pattern.ports['A'].rotation is not None - assert numpy.isclose(p.pattern.ports['A'].rotation, pi) - - -def test_pather_trace_into_failure_rolls_back_ports_and_paths() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.pattern.ports['B'] = Port((-5, 5), rotation=pi / 2, ptype='wire') - - with pytest.raises(BuildError, match='does not match path ptype'): - p.trace_into('A', 'B', plug_destination=False, out_ptype='other') - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - assert numpy.allclose(p.pattern.ports['B'].offset, (-5, 5)) - assert numpy.isclose(p.pattern.ports['B'].rotation, pi / 2) - assert len(p.paths['A']) == 0 - - -def test_pather_trace_into_rename_failure_rolls_back_ports_and_paths() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.pattern.ports['B'] = Port((-10, 0), rotation=pi, ptype='wire') - p.pattern.ports['other'] = Port((3, 4), rotation=0, ptype='wire') - - with pytest.raises(PortError, match='overwritten'): - p.trace_into('A', 'B', plug_destination=False, thru='other') - - assert set(p.pattern.ports) == {'A', 'B', 'other'} - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert numpy.allclose(p.pattern.ports['B'].offset, (-10, 0)) - assert numpy.allclose(p.pattern.ports['other'].offset, (3, 4)) - assert len(p.paths['A']) == 0 - - -@pytest.mark.parametrize( - ('dst', 'kwargs', 'match'), - ( - (Port((-5, 5), rotation=pi / 2, ptype='wire'), {'x': -99}, r'trace_to\(\) arguments: x'), - (Port((-10, 2), rotation=pi, ptype='wire'), {'length': 1}, r'jog\(\) arguments: length'), - (Port((-10, 2), rotation=0, ptype='wire'), {'length': 1}, r'uturn\(\) arguments: length'), - ), -) -def test_pather_trace_into_rejects_reserved_route_kwargs( - dst: Port, - kwargs: dict[str, Any], - match: str, - ) -> None: - lib = Library() - tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.pattern.ports['B'] = dst - - with pytest.raises(BuildError, match=match): - p.trace_into('A', 'B', plug_destination=False, **kwargs) - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - assert numpy.allclose(p.pattern.ports['B'].offset, dst.offset) - assert dst.rotation is not None - assert p.pattern.ports['B'].rotation is not None - assert numpy.isclose(p.pattern.ports['B'].rotation, dst.rotation) - assert len(p.paths['A']) == 0 - - -def test_pather_two_l_fallback_validation_rejects_out_ptype_sensitive_jog() -> None: - class OutPtypeSensitiveTool(Tool): - def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): - radius = 1 if out_ptype is None else 2 - if ccw is None: - rotation = pi - jog = 0 - elif bool(ccw): - rotation = -pi / 2 - jog = radius - else: - rotation = pi / 2 - jog = -radius - ptype = out_ptype or in_ptype or 'wire' - return Port((length, jog), rotation=rotation, ptype=ptype), {'ccw': ccw, 'length': length} - - p = Pather(Library(), tools=OutPtypeSensitiveTool()) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='fallback via two planL'): - p.jog('A', 5, length=10, out_ptype='wide') - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - assert len(p.paths['A']) == 0 - - -def test_pather_two_l_fallback_validation_rejects_out_ptype_sensitive_uturn() -> None: - class OutPtypeSensitiveTool(Tool): - def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): - radius = 1 if out_ptype is None else 2 - if ccw is None: - rotation = pi - jog = 0 - elif bool(ccw): - rotation = -pi / 2 - jog = radius - else: - rotation = pi / 2 - jog = -radius - ptype = out_ptype or in_ptype or 'wire' - return Port((length, jog), rotation=rotation, ptype=ptype), {'ccw': ccw, 'length': length} - - p = Pather(Library(), tools=OutPtypeSensitiveTool()) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='fallback via two planL'): - p.uturn('A', 5, length=10, out_ptype='wide') - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert numpy.isclose(p.pattern.ports['A'].rotation, 0) - assert len(p.paths['A']) == 0 - - -def test_tool_planL_fallback_accepts_custom_port_names() -> None: - class DummyTool(Tool): - def traceL(self, ccw, length, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: - lib = Library() - pat = Pattern() - pat.ports[port_names[0]] = Port((0, 0), 0, ptype='wire') - pat.ports[port_names[1]] = Port((length, 0), pi, ptype='wire') - lib['top'] = pat - return lib - - out_port, _ = DummyTool().planL(None, 5, port_names=('X', 'Y')) - assert numpy.allclose(out_port.offset, (5, 0)) - assert numpy.isclose(out_port.rotation, pi) - - -def test_tool_planS_fallback_accepts_custom_port_names() -> None: - class DummyTool(Tool): - def traceS(self, length, jog, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: - lib = Library() - pat = Pattern() - pat.ports[port_names[0]] = Port((0, 0), 0, ptype='wire') - pat.ports[port_names[1]] = Port((length, jog), pi, ptype='wire') - lib['top'] = pat - return lib - - out_port, _ = DummyTool().planS(5, 2, port_names=('X', 'Y')) - assert numpy.allclose(out_port.offset, (5, 2)) - assert numpy.isclose(out_port.rotation, pi) - - -def test_pather_uturn_failed_fallback_is_atomic() -> None: - lib = Library() - tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - with pytest.raises(BuildError, match='shorter than required bend'): - p.uturn('A', 1.5, length=0) - - assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) - assert p.pattern.ports['A'].rotation == 0 - assert len(p.paths['A']) == 0 - - -def test_pather_render_auto_renames_single_use_tool_children() -> None: - class FullTreeTool(Tool): - def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 - ptype = out_ptype or in_ptype or 'wire' - return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} - - def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 - tree = Library() - top = Pattern(ports={ - port_names[0]: Port((0, 0), 0, ptype='wire'), - port_names[1]: Port((1, 0), pi, ptype='wire'), - }) - child = Pattern(annotations={'batch': [len(batch)]}) - top.ref('_seg') - tree['_top'] = top - tree['_seg'] = child - return tree - - lib = Library() - p = Pather(lib, tools=FullTreeTool(), auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.straight('A', 10) - p.render() - p.straight('A', 10) - p.render() - - assert len(lib) == 2 - assert set(lib.keys()) == set(p.pattern.refs.keys()) - assert len(set(p.pattern.refs.keys())) == 2 - assert all(name.startswith('_seg') for name in lib) - assert p.pattern.referenced_patterns() <= set(lib.keys()) - - -def test_tool_render_fallback_preserves_segment_subtrees() -> None: - class TraceTreeTool(Tool): - def traceL(self, ccw, length, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001 - tree = Library() - top = Pattern(ports={ - port_names[0]: Port((0, 0), 0, ptype='wire'), - port_names[1]: Port((length, 0), pi, ptype='wire'), - }) - child = Pattern(annotations={'length': [length]}) - top.ref('_seg') - tree['_top'] = top - tree['_seg'] = child - return tree - - lib = Library() - p = Pather(lib, tools=TraceTreeTool(), auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.straight('A', 10) - p.render() - - assert '_seg' in lib - assert '_seg' in p.pattern.refs - assert p.pattern.referenced_patterns() <= set(lib.keys()) - - -def test_pather_render_rejects_missing_single_use_tool_refs() -> None: - class MissingSingleUseTool(Tool): - def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 - ptype = out_ptype or in_ptype or 'wire' - return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} - - def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 - tree = Library() - top = Pattern(ports={ - port_names[0]: Port((0, 0), 0, ptype='wire'), - port_names[1]: Port((1, 0), pi, ptype='wire'), - }) - top.ref('_seg') - tree['_top'] = top - return tree - - lib = Library() - lib['_seg'] = Pattern(annotations={'stale': [1]}) - p = Pather(lib, tools=MissingSingleUseTool(), auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - p.straight('A', 10) - - with pytest.raises(BuildError, match='missing single-use refs'): - p.render() - - assert list(lib.keys()) == ['_seg'] - assert not p.pattern.refs - - -def test_pather_render_allows_missing_non_single_use_tool_refs() -> None: - class SharedRefTool(Tool): - def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 - ptype = out_ptype or in_ptype or 'wire' - return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} - - def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 - tree = Library() - top = Pattern(ports={ - port_names[0]: Port((0, 0), 0, ptype='wire'), - port_names[1]: Port((1, 0), pi, ptype='wire'), - }) - top.ref('shared') - tree['_top'] = top - return tree - - lib = Library() - lib['shared'] = Pattern(annotations={'shared': [1]}) - p = Pather(lib, tools=SharedRefTool(), auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') - - p.straight('A', 10) - p.render() - - assert 'shared' in p.pattern.refs - assert p.pattern.referenced_patterns() <= set(lib.keys()) - - -def test_renderpather_rename_to_none_keeps_pending_geometry_without_port() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - rp = Pather(lib, tools=tool, auto_render=False) - rp.pattern.ports['A'] = Port((0, 0), rotation=0) - - rp.at('A').straight(5000) - rp.rename_ports({'A': None}) - - assert 'A' not in rp.pattern.ports - assert len(rp.paths['A']) == 1 - - rp.render() - assert rp.pattern.has_shapes() - assert 'A' not in rp.pattern.ports - - -def test_pather_place_treeview_resolves_once() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool) - - tree = {'child': Pattern(ports={'B': Port((1, 0), pi)})} - - p.place(tree) - - assert len(lib) == 1 - assert 'child' in lib - assert 'child' in p.pattern.refs - assert 'B' in p.pattern.ports - - -def test_pather_plug_treeview_resolves_once() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - tree = {'child': Pattern(ports={'B': Port((0, 0), pi)})} - - p.plug(tree, {'A': 'B'}) - - assert len(lib) == 1 - assert 'child' in lib - assert 'child' in p.pattern.refs - assert 'A' not in p.pattern.ports - - -def test_pather_failed_plug_does_not_add_break_marker() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.annotations = {'k': [1]} - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - p.at('A').trace(None, 5000) - assert [step.opcode for step in p.paths['A']] == ['L'] - - other = Pattern( - annotations={'k': [2]}, - ports={'X': Port((0, 0), pi), 'Y': Port((5, 0), 0)}, - ) - - with pytest.raises(PatternError, match='Annotation keys overlap'): - p.plug(other, {'A': 'X'}, map_out={'Y': 'Z'}, append=True) - - assert [step.opcode for step in p.paths['A']] == ['L'] - assert set(p.pattern.ports) == {'A'} - - -def test_pather_place_reused_deleted_name_keeps_break_marker() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - p.at('A').straight(5000) - p.rename_ports({'A': None}) - - other = Pattern(ports={'X': Port((-5000, 0), rotation=0)}) - p.place(other, port_map={'X': 'A'}, append=True) - p.at('A').straight(2000) - - assert [step.opcode for step in p.paths['A']] == ['L', 'P', 'L'] - - p.render() - assert p.pattern.has_shapes() - assert 'A' in p.pattern.ports - assert numpy.allclose(p.pattern.ports['A'].offset, (-7000, 0)) - - -def test_pather_plug_reused_deleted_name_keeps_break_marker() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - p.pattern.ports['B'] = Port((0, 0), rotation=0) - - p.at('A').straight(5000) - p.rename_ports({'A': None}) - - other = Pattern( - ports={ - 'X': Port((0, 0), rotation=pi), - 'Y': Port((-5000, 0), rotation=0), - }, - ) - p.plug(other, {'B': 'X'}, map_out={'Y': 'A'}, append=True) - p.at('A').straight(2000) - - assert [step.opcode for step in p.paths['A']] == ['L', 'P', 'L'] - - p.render() - assert p.pattern.has_shapes() - assert 'A' in p.pattern.ports - assert 'B' not in p.pattern.ports - assert numpy.allclose(p.pattern.ports['A'].offset, (-7000, 0)) - - -def test_pather_failed_plugged_does_not_add_break_marker() -> None: - lib = Library() - tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) - p.pattern.ports['A'] = Port((0, 0), rotation=0) - - p.at('A').straight(5000) - assert [step.opcode for step in p.paths['A']] == ['L'] - - with pytest.raises(PortError, match='Connection destination ports were not found'): - p.plugged({'A': 'missing'}) - - assert [step.opcode for step in p.paths['A']] == ['L'] - assert set(p.paths) == {'A'} diff --git a/masque/test/test_pather_autotool.py b/masque/test/test_pather_autotool.py new file mode 100644 index 0000000..6ad553a --- /dev/null +++ b/masque/test/test_pather_autotool.py @@ -0,0 +1,127 @@ +import pytest +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from masque import Pather, Library, Pattern, Port +from masque.builder.tools import AutoTool + + +def make_straight(length, width=2, ptype="wire"): + pat = Pattern() + pat.rect((1, 0), xmin=0, xmax=length, yctr=0, ly=width) + pat.ports["A"] = Port((0, 0), 0, ptype=ptype) + pat.ports["B"] = Port((length, 0), pi, ptype=ptype) + return pat + +def make_bend(R, width=2, ptype="wire", clockwise=True): + pat = Pattern() + # Rectangular approximation of a 90 degree bend. + if clockwise: + pat.rect((1, 0), xmin=0, xmax=R, yctr=0, ly=width) + pat.rect((1, 0), xctr=R, lx=width, ymin=-R, ymax=0) + pat.ports["A"] = Port((0, 0), 0, ptype=ptype) + pat.ports["B"] = Port((R, -R), pi/2, ptype=ptype) + else: + pat.rect((1, 0), xmin=0, xmax=R, yctr=0, ly=width) + pat.rect((1, 0), xctr=R, lx=width, ymin=0, ymax=R) + pat.ports["A"] = Port((0, 0), 0, ptype=ptype) + pat.ports["B"] = Port((R, R), -pi/2, ptype=ptype) + return pat + +@pytest.fixture +def multi_bend_tool(): + lib = Library() + + lib["b1"] = make_bend(2, ptype="wire") + b1_abs = lib.abstract("b1") + lib["b2"] = make_bend(5, ptype="wire") + b2_abs = lib.abstract("b2") + + tool = AutoTool( + straights=[ + AutoTool.Straight(ptype="wire", fn=make_straight, in_port_name="A", out_port_name="B", length_range=(0, 10)), + AutoTool.Straight(ptype="wire", fn=lambda l: make_straight(l, width=4), in_port_name="A", out_port_name="B", length_range=(10, 1e8)) + ], + bends=[ + AutoTool.Bend(b1_abs, "A", "B", clockwise=True, mirror=True), + AutoTool.Bend(b2_abs, "A", "B", clockwise=True, mirror=True) + ], + sbends=[], + transitions={}, + default_out_ptype="wire" + ) + return tool, lib + +def test_autotool_uturn() -> None: + from masque.builder.tools import AutoTool + lib = Library() + + def make_straight(length: float) -> Pattern: + pat = Pattern() + pat.rect(layer='M1', xmin=0, xmax=length, yctr=0, ly=1000) + pat.ports['in'] = Port((0, 0), 0) + pat.ports['out'] = Port((length, 0), pi) + return pat + + bend_pat = Pattern() + bend_pat.polygon(layer='M1', vertices=[(0, -500), (0, 500), (1000, -500)]) + bend_pat.ports['in'] = Port((0, 0), 0) + bend_pat.ports['out'] = Port((500, -500), pi/2) + lib['bend'] = bend_pat + + tool = AutoTool( + straights=[AutoTool.Straight(ptype='wire', fn=make_straight, in_port_name='in', out_port_name='out')], + bends=[AutoTool.Bend(abstract=lib.abstract('bend'), in_port_name='in', out_port_name='out', clockwise=True)], + sbends=[], + transitions={}, + default_out_ptype='wire' + ) + + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), 0) + + p.at('A').uturn(offset=-2000, length=1000) + + # U-turn plan output is transformed into the port extension frame. + assert numpy.allclose(p.pattern.ports['A'].offset, (-1000, 2000)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, pi) + +def test_deferred_render_autotool_double_L(multi_bend_tool) -> None: + tool, lib = multi_bend_tool + rp = Pather(lib, tools=tool, auto_render=False) + rp.ports["A"] = Port((0,0), 0, ptype="wire") + + rp.jog("A", 10, length=20) + + assert_allclose(rp.ports["A"].offset, [-20, -10]) + assert_allclose(rp.ports["A"].rotation, 0) + + rp.render() + assert len(rp.pattern.refs) > 0 + +def test_pather_uturn_fallback_no_heuristic(multi_bend_tool) -> None: + tool, lib = multi_bend_tool + + class BasicTool(AutoTool): + def planU(self, *args, **kwargs): + raise NotImplementedError() + + tool_basic = BasicTool( + straights=tool.straights, + bends=tool.bends, + sbends=tool.sbends, + transitions=tool.transitions, + default_out_ptype=tool.default_out_ptype + ) + + p = Pather(lib, tools=tool_basic) + p.ports["A"] = Port((0,0), 0, ptype="wire") + + p.uturn("A", 10, length=5) + + # Fallback U-turn uses two CCW bends: (7, 2) then (8, 2) in local tool frames, + # yielding a global endpoint at (-5, -10). + assert_allclose(p.ports["A"].offset, [-5, -10]) + assert_allclose(p.ports["A"].rotation, pi) diff --git a/masque/test/test_pather_constraints.py b/masque/test/test_pather_constraints.py new file mode 100644 index 0000000..1e74bd6 --- /dev/null +++ b/masque/test/test_pather_constraints.py @@ -0,0 +1,213 @@ +from typing import Any + +import pytest +import numpy +from numpy import pi + +from masque import Pather, Library, Pattern, Port +from masque.builder.tools import PathTool, Tool +from masque.error import BuildError, PortError, PatternError + + +def test_pather_jog_failed_fallback_is_atomic() -> None: + lib = Library() + tool = PathTool(layer='M1', width=2, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='shorter than required bend'): + p.jog('A', 1.5, length=1.5) + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert p.pattern.ports['A'].rotation == 0 + assert len(p.paths['A']) == 0 + +def test_pather_jog_accepts_sub_width_offset_when_length_is_sufficient() -> None: + lib = Library() + tool = PathTool(layer='M1', width=2, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.jog('A', 1.5, length=5) + + assert numpy.allclose(p.pattern.ports['A'].offset, (-5, -1.5)) + assert p.pattern.ports['A'].rotation == 0 + assert len(p.paths['A']) == 0 + +def test_pather_jog_length_solved_from_single_position_bound() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.jog('A', 2, x=-6) + assert numpy.allclose(p.pattern.ports['A'].offset, (-6, -2)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + + q = Pather(Library(), tools=tool) + q.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + q.jog('A', 2, p=-6) + assert numpy.allclose(q.pattern.ports['A'].offset, (-6, -2)) + +def test_pather_jog_requires_length_or_one_position_bound() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='requires either length'): + p.jog('A', 2) + + with pytest.raises(BuildError, match='exactly one positional bound'): + p.jog('A', 2, x=-6, p=-6) + +def test_pather_trace_to_rejects_conflicting_position_bounds() -> None: + tool = PathTool(layer='M1', width=1, ptype='wire') + + for kwargs in ({'x': -5, 'y': 2}, {'y': 2, 'x': -5}, {'p': -7, 'x': -5}): + p = Pather(Library(), tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + with pytest.raises(BuildError, match='exactly one positional bound'): + p.trace_to('A', None, **kwargs) + + p = Pather(Library(), tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + with pytest.raises(BuildError, match='length cannot be combined'): + p.trace_to('A', None, x=-5, length=3) + +def test_pather_trace_rejects_length_with_bundle_bound() -> None: + p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='length cannot be combined'): + p.trace('A', None, length=5, xmin=-100) + +@pytest.mark.parametrize('kwargs', ({'xmin': -10, 'xmax': -20}, {'xmax': -20, 'xmin': -10})) +def test_pather_trace_rejects_multiple_bundle_bounds(kwargs: dict[str, int]) -> None: + p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.pattern.ports['B'] = Port((0, 5), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='exactly one bundle bound'): + p.trace(['A', 'B'], None, **kwargs) + +def test_pather_jog_rejects_length_with_position_bound() -> None: + p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='length cannot be combined'): + p.jog('A', 2, length=5, x=-999) + +@pytest.mark.parametrize('kwargs', ({'x': -999}, {'xmin': -10})) +def test_pather_uturn_rejects_routing_bounds(kwargs: dict[str, int]) -> None: + p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire')) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='Unsupported routing bounds for uturn'): + p.uturn('A', 4, **kwargs) + +def test_pather_uturn_none_length_defaults_to_zero() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.uturn('A', 4) + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, -4)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, pi) + +def test_pather_two_l_fallback_validation_rejects_out_ptype_sensitive_jog() -> None: + class OutPtypeSensitiveTool(Tool): + def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): + radius = 1 if out_ptype is None else 2 + if ccw is None: + rotation = pi + jog = 0 + elif bool(ccw): + rotation = -pi / 2 + jog = radius + else: + rotation = pi / 2 + jog = -radius + ptype = out_ptype or in_ptype or 'wire' + return Port((length, jog), rotation=rotation, ptype=ptype), {'ccw': ccw, 'length': length} + + p = Pather(Library(), tools=OutPtypeSensitiveTool()) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='fallback via two planL'): + p.jog('A', 5, length=10, out_ptype='wide') + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + assert len(p.paths['A']) == 0 + +def test_pather_two_l_fallback_validation_rejects_out_ptype_sensitive_uturn() -> None: + class OutPtypeSensitiveTool(Tool): + def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): + radius = 1 if out_ptype is None else 2 + if ccw is None: + rotation = pi + jog = 0 + elif bool(ccw): + rotation = -pi / 2 + jog = radius + else: + rotation = pi / 2 + jog = -radius + ptype = out_ptype or in_ptype or 'wire' + return Port((length, jog), rotation=rotation, ptype=ptype), {'ccw': ccw, 'length': length} + + p = Pather(Library(), tools=OutPtypeSensitiveTool()) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='fallback via two planL'): + p.uturn('A', 5, length=10, out_ptype='wide') + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + assert len(p.paths['A']) == 0 + +def test_tool_planL_fallback_accepts_custom_port_names() -> None: + class DummyTool(Tool): + def traceL(self, ccw, length, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: + lib = Library() + pat = Pattern() + pat.ports[port_names[0]] = Port((0, 0), 0, ptype='wire') + pat.ports[port_names[1]] = Port((length, 0), pi, ptype='wire') + lib['top'] = pat + return lib + + out_port, _ = DummyTool().planL(None, 5, port_names=('X', 'Y')) + assert numpy.allclose(out_port.offset, (5, 0)) + assert numpy.isclose(out_port.rotation, pi) + +def test_tool_planS_fallback_accepts_custom_port_names() -> None: + class DummyTool(Tool): + def traceS(self, length, jog, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: + lib = Library() + pat = Pattern() + pat.ports[port_names[0]] = Port((0, 0), 0, ptype='wire') + pat.ports[port_names[1]] = Port((length, jog), pi, ptype='wire') + lib['top'] = pat + return lib + + out_port, _ = DummyTool().planS(5, 2, port_names=('X', 'Y')) + assert numpy.allclose(out_port.offset, (5, 2)) + assert numpy.isclose(out_port.rotation, pi) + +def test_pather_uturn_failed_fallback_is_atomic() -> None: + lib = Library() + tool = PathTool(layer='M1', width=2, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + with pytest.raises(BuildError, match='shorter than required bend'): + p.uturn('A', 1.5, length=0) + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert p.pattern.ports['A'].rotation == 0 + assert len(p.paths['A']) == 0 diff --git a/masque/test/test_pather_core.py b/masque/test/test_pather_core.py new file mode 100644 index 0000000..396534e --- /dev/null +++ b/masque/test/test_pather_core.py @@ -0,0 +1,305 @@ +from typing import Any + +import pytest +import numpy +from numpy import pi +from numpy.testing import assert_allclose, assert_equal + +from masque import Pather, Library, Pattern, Port +from masque.builder.tools import PathTool, Tool +from masque.error import BuildError, PortError, PatternError + + +@pytest.fixture +def pather_setup() -> tuple[Pather, PathTool, Library]: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + p = Pather(lib, tools=tool) + # Port rotation points into the device, so path extension moves in the opposite direction. + p.ports["start"] = Port((0, 0), pi / 2, ptype="wire") + return p, tool, lib + +def test_pather_straight(pather_setup: tuple[Pather, PathTool, Library]) -> None: + p, tool, lib = pather_setup + p.straight("start", 10) + + assert_allclose(p.ports["start"].offset, [0, -10], atol=1e-10) + assert p.ports["start"].rotation is not None + assert_allclose(p.ports["start"].rotation, pi / 2, atol=1e-10) + +def test_pather_bend(pather_setup: tuple[Pather, PathTool, Library]) -> None: + p, tool, lib = pather_setup + p.cw("start", 10) + + assert_allclose(p.ports["start"].offset, [-1, -10], atol=1e-10) + assert p.ports["start"].rotation is not None + assert_allclose(p.ports["start"].rotation, 0, atol=1e-10) + +def test_pather_path_to(pather_setup: tuple[Pather, PathTool, Library]) -> None: + p, tool, lib = pather_setup + p.straight("start", y=-50) + assert_equal(p.ports["start"].offset, [0, -50]) + +def test_pather_mpath(pather_setup: tuple[Pather, PathTool, Library]) -> None: + p, tool, lib = pather_setup + p.ports["A"] = Port((0, 0), pi / 2, ptype="wire") + p.ports["B"] = Port((10, 0), pi / 2, ptype="wire") + + p.straight(["A", "B"], ymin=-20) + assert_equal(p.ports["A"].offset, [0, -20]) + assert_equal(p.ports["B"].offset, [10, -20]) + +def test_pather_at_chaining(pather_setup: tuple[Pather, PathTool, Library]) -> None: + p, tool, lib = pather_setup + p.at("start").straight(10).ccw(10) + assert_allclose(p.ports["start"].offset, [1, -20], atol=1e-10) + assert p.ports["start"].rotation is not None + assert_allclose(p.ports["start"].rotation, pi, atol=1e-10) + +def test_pather_dead_ports() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=1) + p = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool) + p.set_dead() + + p.straight("in", -10) + + assert_allclose(p.ports["in"].offset, [10, 0], atol=1e-10) + + p.straight("in", 20) + assert_allclose(p.ports["in"].offset, [-10, 0], atol=1e-10) + + assert not p.pattern.has_shapes() + +def test_pather_trace_basic() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + + # Routing extends opposite the port's inward-facing rotation. + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').trace(None, 5000) + assert numpy.allclose(p.pattern.ports['A'].offset, (-5000, 0)) + + p.at('A').trace(True, 5000) # CCW bend + assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, -500)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, pi/2) + +def test_pather_trace_to() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').trace_to(None, x=-10000) + assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) + + p.at('A').trace_to(None, p=-20000) + assert numpy.allclose(p.pattern.ports['A'].offset, (-20000, 0)) + +def test_pather_bundle_trace() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + + p.pattern.ports['A'] = Port((0, 0), rotation=0) + p.pattern.ports['B'] = Port((0, 2000), rotation=0) + + p.at(['A', 'B']).straight(xmin=-10000) + assert numpy.isclose(p.pattern.ports['A'].offset[0], -10000) + assert numpy.isclose(p.pattern.ports['B'].offset[0], -10000) + + p.at(['A', 'B']).ccw(xmin=-20000, spacing=2000) + # The lower port is on the inner bend, so `xmin` applies to that route. + assert numpy.isclose(p.pattern.ports['A'].offset[0], -20000) + assert numpy.isclose(p.pattern.ports['B'].offset[0], -22000) + +def test_pather_each_bound() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + + p.pattern.ports['A'] = Port((0, 0), rotation=0) + p.pattern.ports['B'] = Port((-1000, 2000), rotation=0) + + p.at(['A', 'B']).trace(None, each=5000) + assert numpy.allclose(p.pattern.ports['A'].offset, (-5000, 0)) + assert numpy.allclose(p.pattern.ports['B'].offset, (-6000, 2000)) + +def test_selection_management() -> None: + lib = Library() + p = Pather(lib) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + p.pattern.ports['B'] = Port((0, 0), rotation=0) + + pp = p.at('A') + assert pp.ports == ['A'] + + pp.select('B') + assert pp.ports == ['A', 'B'] + + pp.deselect('A') + assert pp.ports == ['B'] + + pp.select(['A']) + assert pp.ports == ['B', 'A'] + + pp.drop() + assert 'A' not in p.pattern.ports + assert 'B' not in p.pattern.ports + assert pp.ports == [] + +def test_mark_fork() -> None: + lib = Library() + p = Pather(lib) + p.pattern.ports['A'] = Port((100, 200), rotation=1) + + pp = p.at('A') + pp.mark('B') + assert 'B' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['B'].offset, (100, 200)) + assert p.pattern.ports['B'].rotation == 1 + assert pp.ports == ['A'] + + pp.fork('C') + assert 'C' in p.pattern.ports + assert pp.ports == ['C'] + +def test_mark_fork_reject_overwrite_and_duplicate_targets() -> None: + lib = Library() + + p_mark = Pather(lib, pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0), + 'C': Port((2, 0), rotation=0), + })) + with pytest.raises(PortError, match='overwrite existing ports'): + p_mark.at('A').mark('C') + assert numpy.allclose(p_mark.pattern.ports['C'].offset, (2, 0)) + + p_fork = Pather(lib, pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0), + 'B': Port((1, 0), rotation=0), + })) + pp = p_fork.at(['A', 'B']) + with pytest.raises(PortError, match='targets would collide'): + pp.fork({'A': 'X', 'B': 'X'}) + assert set(p_fork.pattern.ports) == {'A', 'B'} + assert pp.ports == ['A', 'B'] + +def test_mark_fork_dead_overwrite_and_duplicate_targets() -> None: + lib = Library() + p = Pather(lib, pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0), + 'B': Port((1, 0), rotation=0), + 'C': Port((2, 0), rotation=0), + })) + p.set_dead() + + p.at('A').mark('C') + assert numpy.allclose(p.pattern.ports['C'].offset, (0, 0)) + + pp = p.at(['A', 'B']) + pp.fork({'A': 'X', 'B': 'X'}) + assert numpy.allclose(p.pattern.ports['X'].offset, (1, 0)) + assert pp.ports == ['X'] + +def test_mark_fork_reject_missing_sources() -> None: + lib = Library() + p = Pather(lib, pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0), + 'B': Port((1, 0), rotation=0), + })) + + with pytest.raises(PortError, match='selected ports'): + p.at(['A', 'B']).mark({'Z': 'C'}) + + with pytest.raises(PortError, match='selected ports'): + p.at(['A', 'B']).fork({'Z': 'C'}) + +def test_rename() -> None: + lib = Library() + p = Pather(lib) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').rename('B') + assert 'A' not in p.pattern.ports + assert 'B' in p.pattern.ports + + p.pattern.ports['C'] = Port((0, 0), rotation=0) + pp = p.at(['B', 'C']) + pp.rename({'B': 'D', 'C': 'E'}) + assert 'B' not in p.pattern.ports + assert 'C' not in p.pattern.ports + assert 'D' in p.pattern.ports + assert 'E' in p.pattern.ports + assert set(pp.ports) == {'D', 'E'} + +def test_pather_dead_fallback_preserves_out_ptype() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000, ptype='wire') + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.set_dead() + + p.straight('A', -1000, out_ptype='other') + + assert numpy.allclose(p.pattern.ports['A'].offset, (1000, 0)) + assert p.pattern.ports['A'].ptype == 'other' + assert len(p.paths['A']) == 0 + +def test_pather_dead_place_overwrites_colliding_ports_last_wins() -> None: + lib = Library() + p = Pather(lib, pattern=Pattern(ports={ + 'A': Port((5, 5), rotation=0), + 'keep': Port((9, 9), rotation=0), + })) + p.set_dead() + + other = Pattern() + other.ports['X'] = Port((1, 0), rotation=0) + other.ports['Y'] = Port((2, 0), rotation=pi / 2) + + p.place(other, port_map={'X': 'A', 'Y': 'A'}) + + assert set(p.pattern.ports) == {'A', 'keep'} + assert numpy.allclose(p.pattern.ports['A'].offset, (2, 0)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, pi / 2) + +def test_pather_dead_plug_overwrites_colliding_outputs_last_wins() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000, ptype='wire') + p = Pather(lib, tools=tool, pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0, ptype='wire'), + 'B': Port((99, 99), rotation=0, ptype='wire'), + })) + p.set_dead() + + other = Pattern() + other.ports['in'] = Port((0, 0), rotation=pi, ptype='wire') + other.ports['X'] = Port((10, 0), rotation=0, ptype='wire') + other.ports['Y'] = Port((20, 0), rotation=0, ptype='wire') + + p.plug(other, map_in={'A': 'in'}, map_out={'X': 'B', 'Y': 'B'}) + + assert 'A' not in p.pattern.ports + assert 'B' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['B'].offset, (20, 0)) + assert p.pattern.ports['B'].rotation is not None + assert numpy.isclose(p.pattern.ports['B'].rotation, 0) + +def test_pather_dead_rename_overwrites_colliding_ports_last_wins() -> None: + p = Pather(Library(), pattern=Pattern(ports={ + 'A': Port((0, 0), rotation=0), + 'B': Port((1, 0), rotation=0), + 'C': Port((2, 0), rotation=0), + })) + p.set_dead() + + p.rename_ports({'A': 'C', 'B': 'C'}) + + assert set(p.pattern.ports) == {'C'} + assert numpy.allclose(p.pattern.ports['C'].offset, (1, 0)) diff --git a/masque/test/test_pather_place_plug.py b/masque/test/test_pather_place_plug.py new file mode 100644 index 0000000..0bd6a3c --- /dev/null +++ b/masque/test/test_pather_place_plug.py @@ -0,0 +1,122 @@ +from typing import Any + +import pytest +import numpy +from numpy import pi + +from masque import Pather, Library, Pattern, Port +from masque.builder.tools import PathTool, Tool +from masque.error import BuildError, PortError, PatternError + + +def test_pather_place_treeview_resolves_once() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool) + + tree = {'child': Pattern(ports={'B': Port((1, 0), pi)})} + + p.place(tree) + + assert len(lib) == 1 + assert 'child' in lib + assert 'child' in p.pattern.refs + assert 'B' in p.pattern.ports + +def test_pather_plug_treeview_resolves_once() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + tree = {'child': Pattern(ports={'B': Port((0, 0), pi)})} + + p.plug(tree, {'A': 'B'}) + + assert len(lib) == 1 + assert 'child' in lib + assert 'child' in p.pattern.refs + assert 'A' not in p.pattern.ports + +def test_pather_failed_plug_does_not_add_break_marker() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.annotations = {'k': [1]} + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').trace(None, 5000) + assert [step.opcode for step in p.paths['A']] == ['L'] + + other = Pattern( + annotations={'k': [2]}, + ports={'X': Port((0, 0), pi), 'Y': Port((5, 0), 0)}, + ) + + with pytest.raises(PatternError, match='Annotation keys overlap'): + p.plug(other, {'A': 'X'}, map_out={'Y': 'Z'}, append=True) + + assert [step.opcode for step in p.paths['A']] == ['L'] + assert set(p.pattern.ports) == {'A'} + +def test_pather_place_reused_deleted_name_keeps_break_marker() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').straight(5000) + p.rename_ports({'A': None}) + + other = Pattern(ports={'X': Port((-5000, 0), rotation=0)}) + p.place(other, port_map={'X': 'A'}, append=True) + p.at('A').straight(2000) + + assert [step.opcode for step in p.paths['A']] == ['L', 'P', 'L'] + + p.render() + assert p.pattern.has_shapes() + assert 'A' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['A'].offset, (-7000, 0)) + +def test_pather_plug_reused_deleted_name_keeps_break_marker() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + p.pattern.ports['B'] = Port((0, 0), rotation=0) + + p.at('A').straight(5000) + p.rename_ports({'A': None}) + + other = Pattern( + ports={ + 'X': Port((0, 0), rotation=pi), + 'Y': Port((-5000, 0), rotation=0), + }, + ) + p.plug(other, {'B': 'X'}, map_out={'Y': 'A'}, append=True) + p.at('A').straight(2000) + + assert [step.opcode for step in p.paths['A']] == ['L', 'P', 'L'] + + p.render() + assert p.pattern.has_shapes() + assert 'A' in p.pattern.ports + assert 'B' not in p.pattern.ports + assert numpy.allclose(p.pattern.ports['A'].offset, (-7000, 0)) + +def test_pather_failed_plugged_does_not_add_break_marker() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0) + + p.at('A').straight(5000) + assert [step.opcode for step in p.paths['A']] == ['L'] + + with pytest.raises(PortError, match='Connection destination ports were not found'): + p.plugged({'A': 'missing'}) + + assert [step.opcode for step in p.paths['A']] == ['L'] + assert set(p.paths) == {'A'} diff --git a/masque/test/test_pather_rendering.py b/masque/test/test_pather_rendering.py new file mode 100644 index 0000000..2f7360e --- /dev/null +++ b/masque/test/test_pather_rendering.py @@ -0,0 +1,312 @@ +from typing import TYPE_CHECKING, cast + +import pytest +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from ..builder import Pather +from ..builder.tools import PathTool, Tool +from ..error import BuildError +from ..library import Library +from ..pattern import Pattern +from ..ports import Port + +if TYPE_CHECKING: + from ..shapes import Path + + +@pytest.fixture +def deferred_render_setup() -> tuple[Pather, PathTool, Library]: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + rp = Pather(lib, tools=tool, auto_render=False) + rp.ports["start"] = Port((0, 0), pi / 2, ptype="wire") + return rp, tool, lib + +def test_deferred_render_stores_pending_paths_until_render(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").straight(10).straight(10) + + assert not rp.pattern.has_shapes() + assert len(rp.paths["start"]) == 2 + + rp.render() + assert rp.pattern.has_shapes() + assert len(rp.pattern.shapes[(1, 0)]) == 1 + + # PathTool renders length steps in the port extension direction. + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + assert len(path_shape.vertices) == 3 + assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20]], atol=1e-10) + +def test_deferred_render_bend(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").straight(10).cw(10) + + rp.render() + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + # Clockwise bend adds the bend endpoint after the straight segment vertex. + assert len(path_shape.vertices) == 4 + assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20], [-1, -20]], atol=1e-10) + +def test_deferred_render_jog_uses_native_pathtool_planS(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").jog(4, length=10) + + assert len(rp.paths["start"]) == 1 + assert rp.paths["start"][0].opcode == "S" + + rp.render() + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + # Native PathTool S-bends place the jog width/2 before the route end. + assert_allclose(path_shape.vertices, [[0, 0], [0, -9], [4, -9], [4, -10]], atol=1e-10) + assert_allclose(rp.ports["start"].offset, [4, -10], atol=1e-10) + +def test_deferred_render_mirror_preserves_planned_bend_geometry(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").straight(10).cw(10) + + rp.mirror(0) + rp.render() + + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + assert_allclose(path_shape.vertices, [[0, 0], [0, 10], [0, 20], [-1, 20]], atol=1e-10) + +def test_deferred_render_retool(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool1, lib = deferred_render_setup + tool2 = PathTool(layer=(2, 0), width=4, ptype="wire") + + rp.at("start").straight(10) + rp.retool(tool2, keys=["start"]) + rp.at("start").straight(10) + + rp.render() + assert len(rp.pattern.shapes[(1, 0)]) == 1 + assert len(rp.pattern.shapes[(2, 0)]) == 1 + +def test_portpather_translate_only_affects_future_steps(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + pp = rp.at("start") + pp.straight(10) + pp.translate((5, 0)) + pp.straight(10) + + rp.render() + + shapes = rp.pattern.shapes[(1, 0)] + assert len(shapes) == 2 + assert_allclose(cast("Path", shapes[0]).vertices, [[0, 0], [0, -10]], atol=1e-10) + assert_allclose(cast("Path", shapes[1]).vertices, [[5, -10], [5, -20]], atol=1e-10) + assert_allclose(rp.ports["start"].offset, [5, -20], atol=1e-10) + +def test_deferred_render_dead_ports() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=1) + rp = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool, auto_render=False) + rp.set_dead() + + rp.straight("in", -10) + + assert_allclose(rp.ports["in"].offset, [10, 0], atol=1e-10) + + assert len(rp.paths["in"]) == 0 + + rp.render() + assert not rp.pattern.has_shapes() + +def test_deferred_render_rename_port(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").straight(10) + rp.rename_ports({"start": "new_start"}) + rp.at("new_start").straight(10) + + assert "start" not in rp.paths + assert len(rp.paths["new_start"]) == 2 + + rp.render() + assert rp.pattern.has_shapes() + assert len(rp.pattern.shapes[(1, 0)]) == 1 + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20]], atol=1e-10) + assert "new_start" in rp.ports + assert_allclose(rp.ports["new_start"].offset, [0, -20], atol=1e-10) + +def test_deferred_render_drop_keeps_pending_geometry_without_port(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: + rp, tool, lib = deferred_render_setup + rp.at("start").straight(10).drop() + + assert "start" not in rp.ports + assert len(rp.paths["start"]) == 1 + + rp.render() + assert rp.pattern.has_shapes() + assert "start" not in rp.ports + path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) + assert_allclose(path_shape.vertices, [[0, 0], [0, -10]], atol=1e-10) + +def test_pathtool_traceL_bend_geometry_matches_ports() -> None: + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + + tree = tool.traceL(True, 10) + pat = tree.top_pattern() + path_shape = cast("Path", pat.shapes[(1, 0)][0]) + + assert_allclose(path_shape.vertices, [[0, 0], [10, 0], [10, 1]], atol=1e-10) + assert_allclose(pat.ports["B"].offset, [10, 1], atol=1e-10) + +def test_pathtool_traceS_geometry_matches_ports() -> None: + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + + tree = tool.traceS(10, 4) + pat = tree.top_pattern() + path_shape = cast("Path", pat.shapes[(1, 0)][0]) + + assert_allclose(path_shape.vertices, [[0, 0], [9, 0], [9, 4], [10, 4]], atol=1e-10) + assert_allclose(pat.ports["B"].offset, [10, 4], atol=1e-10) + assert_allclose(pat.ports["B"].rotation, pi, atol=1e-10) + +def test_deferred_render_uturn_fallback() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + rp = Pather(lib, tools=tool, auto_render=False) + rp.pattern.ports['A'] = Port((0, 0), rotation=0) + + rp.at('A').uturn(offset=10000, length=5000) + + assert len(rp.paths['A']) == 2 + assert rp.paths['A'][0].opcode == 'L' + assert rp.paths['A'][1].opcode == 'L' + + rp.render() + assert rp.pattern.ports['A'].rotation is not None + assert numpy.isclose(rp.pattern.ports['A'].rotation, pi) + +def test_pather_render_auto_renames_single_use_tool_children() -> None: + class FullTreeTool(Tool): + def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 + ptype = out_ptype or in_ptype or 'wire' + return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} + + def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 + tree = Library() + top = Pattern(ports={ + port_names[0]: Port((0, 0), 0, ptype='wire'), + port_names[1]: Port((1, 0), pi, ptype='wire'), + }) + child = Pattern(annotations={'batch': [len(batch)]}) + top.ref('_seg') + tree['_top'] = top + tree['_seg'] = child + return tree + + lib = Library() + p = Pather(lib, tools=FullTreeTool(), auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.straight('A', 10) + p.render() + p.straight('A', 10) + p.render() + + assert len(lib) == 2 + assert set(lib.keys()) == set(p.pattern.refs.keys()) + assert len(set(p.pattern.refs.keys())) == 2 + assert all(name.startswith('_seg') for name in lib) + assert p.pattern.referenced_patterns() <= set(lib.keys()) + +def test_tool_render_fallback_preserves_segment_subtrees() -> None: + class TraceTreeTool(Tool): + def traceL(self, ccw, length, *, in_ptype=None, out_ptype=None, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001 + tree = Library() + top = Pattern(ports={ + port_names[0]: Port((0, 0), 0, ptype='wire'), + port_names[1]: Port((length, 0), pi, ptype='wire'), + }) + child = Pattern(annotations={'length': [length]}) + top.ref('_seg') + tree['_top'] = top + tree['_seg'] = child + return tree + + lib = Library() + p = Pather(lib, tools=TraceTreeTool(), auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.straight('A', 10) + p.render() + + assert '_seg' in lib + assert '_seg' in p.pattern.refs + assert p.pattern.referenced_patterns() <= set(lib.keys()) + +def test_pather_render_rejects_missing_single_use_tool_refs() -> None: + class MissingSingleUseTool(Tool): + def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 + ptype = out_ptype or in_ptype or 'wire' + return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} + + def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 + tree = Library() + top = Pattern(ports={ + port_names[0]: Port((0, 0), 0, ptype='wire'), + port_names[1]: Port((1, 0), pi, ptype='wire'), + }) + top.ref('_seg') + tree['_top'] = top + return tree + + lib = Library() + lib['_seg'] = Pattern(annotations={'stale': [1]}) + p = Pather(lib, tools=MissingSingleUseTool(), auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.straight('A', 10) + + with pytest.raises(BuildError, match='missing single-use refs'): + p.render() + + assert list(lib.keys()) == ['_seg'] + assert not p.pattern.refs + +def test_pather_render_allows_missing_non_single_use_tool_refs() -> None: + class SharedRefTool(Tool): + def planL(self, ccw, length, *, in_ptype=None, out_ptype=None, **kwargs): # noqa: ANN001,ANN202 + ptype = out_ptype or in_ptype or 'wire' + return Port((length, 0), rotation=pi, ptype=ptype), {'length': length} + + def render(self, batch, *, port_names=('A', 'B'), **kwargs) -> Library: # noqa: ANN001,ANN202 + tree = Library() + top = Pattern(ports={ + port_names[0]: Port((0, 0), 0, ptype='wire'), + port_names[1]: Port((1, 0), pi, ptype='wire'), + }) + top.ref('shared') + tree['_top'] = top + return tree + + lib = Library() + lib['shared'] = Pattern(annotations={'shared': [1]}) + p = Pather(lib, tools=SharedRefTool(), auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.straight('A', 10) + p.render() + + assert 'shared' in p.pattern.refs + assert p.pattern.referenced_patterns() <= set(lib.keys()) + +def test_deferred_render_rename_to_none_keeps_pending_geometry_without_port() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + rp = Pather(lib, tools=tool, auto_render=False) + rp.pattern.ports['A'] = Port((0, 0), rotation=0) + + rp.at('A').straight(5000) + rp.rename_ports({'A': None}) + + assert 'A' not in rp.pattern.ports + assert len(rp.paths['A']) == 1 + + rp.render() + assert rp.pattern.has_shapes() + assert 'A' not in rp.pattern.ports diff --git a/masque/test/test_pather_trace_into.py b/masque/test/test_pather_trace_into.py new file mode 100644 index 0000000..b5e8aed --- /dev/null +++ b/masque/test/test_pather_trace_into.py @@ -0,0 +1,189 @@ +from typing import Any + +import pytest +import numpy +from numpy import pi +from numpy.testing import assert_equal + +from masque import Pather, Library, Pattern, Port +from masque.builder.tools import PathTool, Tool +from masque.error import BuildError, PortError, PatternError + + +@pytest.fixture +def trace_into_setup() -> tuple[Pather, PathTool, Library]: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + p = Pather(lib, tools=tool, auto_render=True, auto_render_append=False) + return p, tool, lib + +def test_path_into_straight(trace_into_setup: tuple[Pather, PathTool, Library]) -> None: + p, _tool, _lib = trace_into_setup + p.ports["src"] = Port((0, 0), 0, ptype="wire") + p.ports["dst"] = Port((-20, 0), pi, ptype="wire") + + p.trace_into("src", "dst") + + assert "src" not in p.ports + assert "dst" not in p.ports + assert len(p.pattern.refs) == 1 + +def test_path_into_bend(trace_into_setup: tuple[Pather, PathTool, Library]) -> None: + p, _tool, _lib = trace_into_setup + p.ports["src"] = Port((0, 0), 0, ptype="wire") + p.ports["dst"] = Port((-20, -20), 3 * pi / 2, ptype="wire") + + p.trace_into("src", "dst") + + assert "src" not in p.ports + assert "dst" not in p.ports + # `trace_into()` batches internal legs before auto-rendering so the operation + # rolls back cleanly on later failures. + assert len(p.pattern.refs) == 1 + +def test_path_into_sbend(trace_into_setup: tuple[Pather, PathTool, Library]) -> None: + p, _tool, _lib = trace_into_setup + p.ports["src"] = Port((0, 0), 0, ptype="wire") + p.ports["dst"] = Port((-20, -10), pi, ptype="wire") + + p.trace_into("src", "dst") + + assert "src" not in p.ports + assert "dst" not in p.ports + +def test_path_into_thru(trace_into_setup: tuple[Pather, PathTool, Library]) -> None: + p, _tool, _lib = trace_into_setup + p.ports["src"] = Port((0, 0), 0, ptype="wire") + p.ports["dst"] = Port((-20, 0), pi, ptype="wire") + p.ports["other"] = Port((10, 10), 0) + + p.trace_into("src", "dst", thru="other") + + assert "src" in p.ports + assert_equal(p.ports["src"].offset, [10, 10]) + assert "other" not in p.ports + +def test_pather_trace_into() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000) + p = Pather(lib, tools=tool, auto_render=False) + + p.pattern.ports['A'] = Port((0, 0), rotation=0) + p.pattern.ports['B'] = Port((-10000, 0), rotation=pi) + p.at('A').trace_into('B', plug_destination=False) + assert 'B' in p.pattern.ports + assert 'A' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) + + p.pattern.ports['C'] = Port((0, 0), rotation=0) + p.pattern.ports['D'] = Port((-5000, 5000), rotation=pi/2) + p.at('C').trace_into('D', plug_destination=False) + assert 'D' in p.pattern.ports + assert 'C' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['C'].offset, (-5000, 5000)) + + p.pattern.ports['E'] = Port((0, 0), rotation=0) + p.pattern.ports['F'] = Port((-10000, 2000), rotation=pi) + p.at('E').trace_into('F', plug_destination=False) + assert 'F' in p.pattern.ports + assert 'E' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['E'].offset, (-10000, 2000)) + + p.pattern.ports['G'] = Port((0, 0), rotation=0) + p.pattern.ports['H'] = Port((-10000, 2000), rotation=0) + p.at('G').trace_into('H', plug_destination=False) + assert 'H' in p.pattern.ports + assert 'G' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['G'].offset, (-10000, 2000)) + assert p.pattern.ports['G'].rotation is not None + assert numpy.isclose(p.pattern.ports['G'].rotation, pi) + + p.pattern.ports['I'] = Port((0, 0), rotation=pi / 2) + p.pattern.ports['J'] = Port((0, -10000), rotation=3 * pi / 2) + p.at('I').trace_into('J', plug_destination=False) + assert 'J' in p.pattern.ports + assert 'I' in p.pattern.ports + assert numpy.allclose(p.pattern.ports['I'].offset, (0, -10000)) + assert p.pattern.ports['I'].rotation is not None + assert numpy.isclose(p.pattern.ports['I'].rotation, pi / 2) + +def test_pather_trace_into_dead_updates_ports_without_geometry() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1000, ptype='wire') + p = Pather(lib, tools=tool, auto_render=False) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.pattern.ports['B'] = Port((-10000, 0), rotation=pi, ptype='wire') + p.set_dead() + + p.trace_into('A', 'B', plug_destination=False) + + assert set(p.pattern.ports) == {'A', 'B'} + assert numpy.allclose(p.pattern.ports['A'].offset, (-10000, 0)) + assert p.pattern.ports['A'].rotation is not None + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + assert len(p.paths['A']) == 0 + assert not p.pattern.has_shapes() + assert not p.pattern.has_refs() + +def test_pather_trace_into_failure_rolls_back_ports_and_paths() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.pattern.ports['B'] = Port((-5, 5), rotation=pi / 2, ptype='wire') + + with pytest.raises(BuildError, match='does not match path ptype'): + p.trace_into('A', 'B', plug_destination=False, out_ptype='other') + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + assert numpy.allclose(p.pattern.ports['B'].offset, (-5, 5)) + assert numpy.isclose(p.pattern.ports['B'].rotation, pi / 2) + assert len(p.paths['A']) == 0 + +def test_pather_trace_into_rename_failure_rolls_back_ports_and_paths() -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.pattern.ports['B'] = Port((-10, 0), rotation=pi, ptype='wire') + p.pattern.ports['other'] = Port((3, 4), rotation=0, ptype='wire') + + with pytest.raises(PortError, match='overwritten'): + p.trace_into('A', 'B', plug_destination=False, thru='other') + + assert set(p.pattern.ports) == {'A', 'B', 'other'} + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert numpy.allclose(p.pattern.ports['B'].offset, (-10, 0)) + assert numpy.allclose(p.pattern.ports['other'].offset, (3, 4)) + assert len(p.paths['A']) == 0 + +@pytest.mark.parametrize( + ('dst', 'kwargs', 'match'), + ( + (Port((-5, 5), rotation=pi / 2, ptype='wire'), {'x': -99}, r'trace_to\(\) arguments: x'), + (Port((-10, 2), rotation=pi, ptype='wire'), {'length': 1}, r'jog\(\) arguments: length'), + (Port((-10, 2), rotation=0, ptype='wire'), {'length': 1}, r'uturn\(\) arguments: length'), + ), +) +def test_pather_trace_into_rejects_reserved_route_kwargs( + dst: Port, + kwargs: dict[str, Any], + match: str, + ) -> None: + lib = Library() + tool = PathTool(layer='M1', width=1, ptype='wire') + p = Pather(lib, tools=tool) + p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + p.pattern.ports['B'] = dst + + with pytest.raises(BuildError, match=match): + p.trace_into('A', 'B', plug_destination=False, **kwargs) + + assert numpy.allclose(p.pattern.ports['A'].offset, (0, 0)) + assert numpy.isclose(p.pattern.ports['A'].rotation, 0) + assert numpy.allclose(p.pattern.ports['B'].offset, dst.offset) + assert dst.rotation is not None + assert p.pattern.ports['B'].rotation is not None + assert numpy.isclose(p.pattern.ports['B'].rotation, dst.rotation) + assert len(p.paths['A']) == 0 diff --git a/masque/test/test_poly_collection.py b/masque/test/test_poly_collection.py new file mode 100644 index 0000000..70b17d2 --- /dev/null +++ b/masque/test/test_poly_collection.py @@ -0,0 +1,89 @@ +import pytest +from numpy.testing import assert_equal + +from ..error import PatternError +from ..shapes import Circle, Ellipse, Polygon, PolyCollection + + +def test_poly_collection_init() -> None: + verts = [[0, 0], [1, 0], [1, 1], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] + offsets = [0, 4] + pc = PolyCollection(vertex_lists=verts, vertex_offsets=offsets) + assert len(list(pc.polygon_vertices)) == 2 + assert_equal(pc.get_bounds_single(), [[0, 0], [11, 11]]) + +def test_poly_collection_to_polygons() -> None: + verts = [[0, 0], [1, 0], [1, 1], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] + offsets = [0, 4] + pc = PolyCollection(vertex_lists=verts, vertex_offsets=offsets) + polys = pc.to_polygons() + assert len(polys) == 2 + assert_equal(polys[0].vertices, [[0, 0], [1, 0], [1, 1], [0, 1]]) + assert_equal(polys[1].vertices, [[10, 10], [11, 10], [11, 11], [10, 11]]) + +def test_poly_collection_holes() -> None: + # PolyCollection represents separate polygon boundaries, including nested boundaries. + verts = [ + [0, 0], + [10, 0], + [10, 10], + [0, 10], # Poly 1 + [2, 2], + [2, 8], + [8, 8], + [8, 2], # Poly 2 + ] + offsets = [0, 4] + pc = PolyCollection(verts, offsets) + polys = pc.to_polygons() + assert len(polys) == 2 + assert_equal(polys[0].vertices, [[0, 0], [10, 0], [10, 10], [0, 10]]) + assert_equal(polys[1].vertices, [[2, 2], [2, 8], [8, 8], [8, 2]]) + +def test_poly_collection_constituent_empty() -> None: + # Duplicate offsets create an empty constituent slice between valid polygons. + verts = [ + [0, 0], + [1, 0], + [0, 1], # Tri + [10, 10], + [11, 10], + [11, 11], + [10, 11], # Square + ] + offsets = [0, 3, 3] + pc = PolyCollection(verts, offsets) + with pytest.raises(PatternError): + pc.to_polygons() + +def test_poly_collection_valid() -> None: + verts = [[0, 0], [1, 0], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] + offsets = [0, 3] + pc = PolyCollection(verts, offsets) + assert len(pc.to_polygons()) == 2 + shapes = [Circle(radius=20), Circle(radius=10), Polygon([[0, 0], [10, 0], [10, 10]]), Ellipse(radii=(5, 5))] + sorted_shapes = sorted(shapes) + assert len(sorted_shapes) == 4 + assert sorted(sorted_shapes) == sorted_shapes + +def test_poly_collection_normalized_form_reconstruction_is_independent() -> None: + pc = PolyCollection([[0, 0], [1, 0], [0, 1]], [0]) + _intrinsic, _extrinsic, rebuild = pc.normalized_form(1) + + clone = rebuild() + clone.vertex_offsets[:] = [5] + + assert_equal(pc.vertex_offsets, [0]) + assert_equal(clone.vertex_offsets, [5]) + +def test_poly_collection_normalized_form_rebuilds_independent_clones() -> None: + pc = PolyCollection([[0, 0], [1, 0], [0, 1]], [0]) + _intrinsic, _extrinsic, rebuild = pc.normalized_form(1) + + first = rebuild() + second = rebuild() + first.vertex_offsets[:] = [7] + + assert_equal(first.vertex_offsets, [7]) + assert_equal(second.vertex_offsets, [0]) + assert_equal(pc.vertex_offsets, [0]) diff --git a/masque/test/test_renderpather.py b/masque/test/test_renderpather.py deleted file mode 100644 index b518a1f..0000000 --- a/masque/test/test_renderpather.py +++ /dev/null @@ -1,199 +0,0 @@ -import pytest -from typing import cast, TYPE_CHECKING -from numpy.testing import assert_allclose -from numpy import pi - -from ..builder import Pather -from ..builder.tools import PathTool -from ..library import Library -from ..ports import Port - -if TYPE_CHECKING: - from ..shapes import Path - - -@pytest.fixture -def rpather_setup() -> tuple[Pather, PathTool, Library]: - lib = Library() - tool = PathTool(layer=(1, 0), width=2, ptype="wire") - rp = Pather(lib, tools=tool, auto_render=False) - rp.ports["start"] = Port((0, 0), pi / 2, ptype="wire") - return rp, tool, lib - - -def test_renderpather_basic(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - # Plan two segments - rp.at("start").straight(10).straight(10) - - # Before rendering, no shapes in pattern - assert not rp.pattern.has_shapes() - assert len(rp.paths["start"]) == 2 - - # Render - rp.render() - assert rp.pattern.has_shapes() - assert len(rp.pattern.shapes[(1, 0)]) == 1 - - # Path vertices should be (0,0), (0,-10), (0,-20) - # transformed by start port (rot pi/2 -> 270 deg transform) - # wait, PathTool.render for opcode L uses rotation_matrix_2d(port_rot + pi) - # start_port rot pi/2. pi/2 + pi = 3pi/2. - # (10, 0) rotated 3pi/2 -> (0, -10) - # So vertices: (0,0), (0,-10), (0,-20) - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - assert len(path_shape.vertices) == 3 - assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20]], atol=1e-10) - - -def test_renderpather_bend(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - # Plan straight then bend - rp.at("start").straight(10).cw(10) - - rp.render() - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - # Path vertices: - # 1. Start (0,0) - # 2. Straight end: (0, -10) - # 3. Bend end: (-1, -20) - # PathTool.planL(ccw=False, length=10) returns data=[10, -1] - # start_port for 2nd segment is at (0, -10) with rotation pi/2 - # dxy = rot(pi/2 + pi) @ (10, 0) = (0, -10). So vertex at (0, -20). - # and final end_port.offset is (-1, -20). - assert len(path_shape.vertices) == 4 - assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20], [-1, -20]], atol=1e-10) - - -def test_renderpather_jog_uses_native_pathtool_planS(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - rp.at("start").jog(4, length=10) - - assert len(rp.paths["start"]) == 1 - assert rp.paths["start"][0].opcode == "S" - - rp.render() - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - # Native PathTool S-bends place the jog width/2 before the route end. - assert_allclose(path_shape.vertices, [[0, 0], [0, -9], [4, -9], [4, -10]], atol=1e-10) - assert_allclose(rp.ports["start"].offset, [4, -10], atol=1e-10) - - -def test_renderpather_mirror_preserves_planned_bend_geometry(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - rp.at("start").straight(10).cw(10) - - rp.mirror(0) - rp.render() - - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - assert_allclose(path_shape.vertices, [[0, 0], [0, 10], [0, 20], [-1, 20]], atol=1e-10) - - -def test_renderpather_retool(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool1, lib = rpather_setup - tool2 = PathTool(layer=(2, 0), width=4, ptype="wire") - - rp.at("start").straight(10) - rp.retool(tool2, keys=["start"]) - rp.at("start").straight(10) - - rp.render() - # Different tools should cause different batches/shapes - assert len(rp.pattern.shapes[(1, 0)]) == 1 - assert len(rp.pattern.shapes[(2, 0)]) == 1 - - -def test_portpather_translate_only_affects_future_steps(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - pp = rp.at("start") - pp.straight(10) - pp.translate((5, 0)) - pp.straight(10) - - rp.render() - - shapes = rp.pattern.shapes[(1, 0)] - assert len(shapes) == 2 - assert_allclose(cast("Path", shapes[0]).vertices, [[0, 0], [0, -10]], atol=1e-10) - assert_allclose(cast("Path", shapes[1]).vertices, [[5, -10], [5, -20]], atol=1e-10) - assert_allclose(rp.ports["start"].offset, [5, -20], atol=1e-10) - - -def test_renderpather_dead_ports() -> None: - lib = Library() - tool = PathTool(layer=(1, 0), width=1) - rp = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool, auto_render=False) - rp.set_dead() - - # Impossible path - rp.straight("in", -10) - - # port_rot=0, forward is -x. path(-10) means moving -10 in -x direction -> +10 in x. - assert_allclose(rp.ports["in"].offset, [10, 0], atol=1e-10) - - # Verify no render steps were added - assert len(rp.paths["in"]) == 0 - - # Verify no geometry - rp.render() - assert not rp.pattern.has_shapes() - - -def test_renderpather_rename_port(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - rp.at("start").straight(10) - # Rename port while path is planned - rp.rename_ports({"start": "new_start"}) - # Continue path on new name - rp.at("new_start").straight(10) - - assert "start" not in rp.paths - assert len(rp.paths["new_start"]) == 2 - - rp.render() - assert rp.pattern.has_shapes() - assert len(rp.pattern.shapes[(1, 0)]) == 1 - # Total length 20. start_port rot pi/2 -> 270 deg transform. - # Vertices (0,0), (0,-10), (0,-20) - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - assert_allclose(path_shape.vertices, [[0, 0], [0, -10], [0, -20]], atol=1e-10) - assert "new_start" in rp.ports - assert_allclose(rp.ports["new_start"].offset, [0, -20], atol=1e-10) - - -def test_renderpather_drop_keeps_pending_geometry_without_port(rpather_setup: tuple[Pather, PathTool, Library]) -> None: - rp, tool, lib = rpather_setup - rp.at("start").straight(10).drop() - - assert "start" not in rp.ports - assert len(rp.paths["start"]) == 1 - - rp.render() - assert rp.pattern.has_shapes() - assert "start" not in rp.ports - path_shape = cast("Path", rp.pattern.shapes[(1, 0)][0]) - assert_allclose(path_shape.vertices, [[0, 0], [0, -10]], atol=1e-10) - - -def test_pathtool_traceL_bend_geometry_matches_ports() -> None: - tool = PathTool(layer=(1, 0), width=2, ptype="wire") - - tree = tool.traceL(True, 10) - pat = tree.top_pattern() - path_shape = cast("Path", pat.shapes[(1, 0)][0]) - - assert_allclose(path_shape.vertices, [[0, 0], [10, 0], [10, 1]], atol=1e-10) - assert_allclose(pat.ports["B"].offset, [10, 1], atol=1e-10) - - -def test_pathtool_traceS_geometry_matches_ports() -> None: - tool = PathTool(layer=(1, 0), width=2, ptype="wire") - - tree = tool.traceS(10, 4) - pat = tree.top_pattern() - path_shape = cast("Path", pat.shapes[(1, 0)][0]) - - assert_allclose(path_shape.vertices, [[0, 0], [9, 0], [9, 4], [10, 4]], atol=1e-10) - assert_allclose(pat.ports["B"].offset, [10, 4], atol=1e-10) - assert_allclose(pat.ports["B"].rotation, pi, atol=1e-10) diff --git a/masque/test/test_repetition.py b/masque/test/test_repetition.py index 0d0be41..00d2d7a 100644 --- a/masque/test/test_repetition.py +++ b/masque/test/test_repetition.py @@ -7,7 +7,6 @@ from ..error import PatternError def test_grid_displacements() -> None: - # 2x2 grid grid = Grid(a_vector=(10, 0), b_vector=(0, 5), a_count=2, b_count=2) disps = sorted([tuple(d) for d in grid.displacements]) assert disps == [(0.0, 0.0), (0.0, 5.0), (10.0, 0.0), (10.0, 5.0)] @@ -34,7 +33,6 @@ def test_grid_get_bounds() -> None: def test_arbitrary_displacements() -> None: pts = [[0, 0], [10, 20], [-5, 30]] arb = Arbitrary(pts) - # They should be sorted by displacements.setter disps = arb.displacements assert len(disps) == 3 assert any((disps == [0, 0]).all(axis=1)) @@ -47,9 +45,7 @@ def test_arbitrary_transform() -> None: arb.rotate(pi / 2) assert_allclose(arb.displacements, [[0, 10]], atol=1e-10) - arb.mirror(0) # Mirror x across y axis? Wait, mirror(axis=0) in repetition.py is: - # self.displacements[:, 1 - axis] *= -1 - # if axis=0, 1-axis=1, so y *= -1 + arb.mirror(0) assert_allclose(arb.displacements, [[0, -10]], atol=1e-10) diff --git a/masque/test/test_shape_advanced.py b/masque/test/test_shape_advanced.py deleted file mode 100644 index 8e35841..0000000 --- a/masque/test/test_shape_advanced.py +++ /dev/null @@ -1,265 +0,0 @@ -from pathlib import Path -import pytest -import numpy -from numpy.testing import assert_equal, assert_allclose -from numpy import pi - -from ..shapes import Arc, Ellipse, Circle, Polygon, Path as MPath, Text, PolyCollection -from ..error import PatternError - - -# 1. Text shape tests -def test_text_to_polygons() -> None: - pytest.importorskip("freetype") - font_path = "/usr/share/fonts/truetype/dejavu/DejaVuMathTeXGyre.ttf" - if not Path(font_path).exists(): - pytest.skip("Font file not found") - - t = Text("Hi", height=10, font_path=font_path) - polys = t.to_polygons() - assert len(polys) > 0 - assert all(isinstance(p, Polygon) for p in polys) - - # Check that it advances - # Character 'H' and 'i' should have different vertices - # Each character is a set of polygons. We check the mean x of vertices for each character. - char_x_means = [p.vertices[:, 0].mean() for p in polys] - assert len(set(char_x_means)) >= 2 - - -def test_text_bounds_and_normalized_form() -> None: - pytest.importorskip("freetype") - font_path = "/usr/share/fonts/truetype/dejavu/DejaVuMathTeXGyre.ttf" - if not Path(font_path).exists(): - pytest.skip("Font file not found") - - text = Text("Hi", height=10, font_path=font_path) - _intrinsic, extrinsic, ctor = text.normalized_form(5) - normalized = ctor() - - assert extrinsic[1] == 2 - assert normalized.height == 5 - - bounds = text.get_bounds_single() - assert bounds is not None - assert numpy.isfinite(bounds).all() - assert numpy.all(bounds[1] > bounds[0]) - - -def test_text_mirroring_affects_comparison() -> None: - text = Text("A", height=10, font_path="dummy.ttf") - mirrored = Text("A", height=10, font_path="dummy.ttf", mirrored=True) - - assert text != mirrored - assert (text < mirrored) != (mirrored < text) - - -# 2. Manhattanization tests -def test_manhattanize() -> None: - pytest.importorskip("float_raster") - pytest.importorskip("skimage.measure") - # Diamond shape - poly = Polygon([[0, 5], [5, 10], [10, 5], [5, 0]]) - grid = numpy.arange(0, 11, 1) - - manhattan_polys = poly.manhattanize(grid, grid) - assert len(manhattan_polys) >= 1 - for mp in manhattan_polys: - # Check that all edges are axis-aligned - dv = numpy.diff(mp.vertices, axis=0) - # For each segment, either dx or dy must be zero - assert numpy.all((dv[:, 0] == 0) | (dv[:, 1] == 0)) - - -# 3. Comparison and Sorting tests -def test_shape_comparisons() -> None: - c1 = Circle(radius=10) - c2 = Circle(radius=20) - assert c1 < c2 - assert not (c2 < c1) - - p1 = Polygon([[0, 0], [10, 0], [10, 10]]) - p2 = Polygon([[0, 0], [10, 0], [10, 11]]) # Different vertex - assert p1 < p2 - - # Different types - assert c1 < p1 or p1 < c1 - assert (c1 < p1) != (p1 < c1) - - -# 4. Arc/Path Edge Cases -def test_arc_edge_cases() -> None: - # Wrapped arc (> 360 deg) - a = Arc(radii=(10, 10), angles=(0, 3 * pi), width=2) - a.to_polygons(num_vertices=64) - # Should basically be a ring - bounds = a.get_bounds_single() - assert_allclose(bounds, [[-11, -11], [11, 11]], atol=1e-10) - - -def test_rotated_ellipse_bounds_match_polygonized_geometry() -> None: - ellipse = Ellipse(radii=(10, 20), rotation=pi / 4, offset=(100, 200)) - bounds = ellipse.get_bounds_single() - poly_bounds = ellipse.to_polygons(num_vertices=8192)[0].get_bounds_single() - assert_allclose(bounds, poly_bounds, atol=1e-3) - - -def test_rotated_arc_bounds_match_polygonized_geometry() -> None: - arc = Arc(radii=(10, 20), angles=(0, pi), width=2, rotation=pi / 4, offset=(100, 200)) - bounds = arc.get_bounds_single() - poly_bounds = arc.to_polygons(num_vertices=8192)[0].get_bounds_single() - assert_allclose(bounds, poly_bounds, atol=1e-3) - - -def test_rotated_focus_arc_bounds_match_polygonized_geometry() -> None: - arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, rotation=pi / 4, - offset=(100, 200), angle_ref=Arc.AngleRef.FocusPos) - bounds = arc.get_bounds_single() - poly_bounds = arc.to_polygons(num_vertices=8192)[0].get_bounds_single() - assert_allclose(bounds, poly_bounds, atol=1e-3) - - -def test_curve_polygonizers_clamp_large_max_arclen() -> None: - for shape in ( - Circle(radius=10), - Ellipse(radii=(10, 20)), - Arc(radii=(10, 20), angles=(0, 1), width=2), - ): - polys = shape.to_polygons(num_vertices=None, max_arclen=1e9) - assert len(polys) == 1 - assert len(polys[0].vertices) >= 3 - - -def test_arc_polygonization_rejects_nan_implied_arclen() -> None: - arc = Arc(radii=(10, 20), angles=(0, numpy.nan), width=2) - with pytest.raises(PatternError, match='valid max_arclen'): - arc.to_polygons(num_vertices=24) - - -def test_focus_arc_rejects_focus_outside_inner_boundary() -> None: - arc = Arc(radii=(10, 5), angles=(0, 1), width=6, angle_ref=Arc.AngleRef.FocusPos) - with pytest.raises(PatternError, match='inside both arc boundary ellipses'): - arc.to_polygons(num_vertices=24) - - -def test_focus_arc_max_arclen_limits_segments() -> None: - arc = Arc(radii=(10, 6), angles=(-0.25, 1.1), width=1, angle_ref=Arc.AngleRef.FocusNeg) - v = arc.to_polygons(max_arclen=2)[0].vertices - dist = numpy.sqrt(numpy.sum(numpy.diff(v, axis=0, append=v[:1]) ** 2, axis=1)) - assert numpy.all(dist <= 2.000001) - - -def test_ellipse_integer_radii_scale_cleanly() -> None: - ellipse = Ellipse(radii=(10, 20)) - ellipse.scale_by(0.5) - assert_allclose(ellipse.radii, [5, 10]) - - -def test_arc_rejects_zero_radii_up_front() -> None: - with pytest.raises(PatternError, match='Radii must be positive'): - Arc(radii=(0, 5), angles=(0, 1), width=1) - with pytest.raises(PatternError, match='Radii must be positive'): - Arc(radii=(5, 0), angles=(0, 1), width=1) - with pytest.raises(PatternError, match='Radii must be positive'): - Arc(radii=(0, 0), angles=(0, 1), width=1) - - -def test_path_edge_cases() -> None: - # Zero-length segments - p = MPath(vertices=[[0, 0], [0, 0], [10, 0]], width=2) - polys = p.to_polygons() - assert len(polys) == 1 - assert_equal(polys[0].get_bounds_single(), [[0, -1], [10, 1]]) - - -# 5. PolyCollection with holes -def test_poly_collection_holes() -> None: - # Outer square, inner square hole - # PolyCollection doesn't explicitly support holes, but its constituents (Polygons) do? - # wait, Polygon in masque is just a boundary. Holes are usually handled by having multiple - # polygons or using specific winding rules. - # masque.shapes.Polygon doc says "specify an implicitly-closed boundary". - # Pyclipper is used in connectivity.py for holes. - - # Let's test PolyCollection with multiple polygons - verts = [ - [0, 0], - [10, 0], - [10, 10], - [0, 10], # Poly 1 - [2, 2], - [2, 8], - [8, 8], - [8, 2], # Poly 2 - ] - offsets = [0, 4] - pc = PolyCollection(verts, offsets) - polys = pc.to_polygons() - assert len(polys) == 2 - assert_equal(polys[0].vertices, [[0, 0], [10, 0], [10, 10], [0, 10]]) - assert_equal(polys[1].vertices, [[2, 2], [2, 8], [8, 8], [8, 2]]) - - -def test_poly_collection_constituent_empty() -> None: - # One real triangle, one "empty" polygon (0 vertices), one real square - # Note: Polygon requires 3 vertices, so "empty" here might mean just some junk - # that to_polygons should handle. - # Actually PolyCollection doesn't check vertex count per polygon. - verts = [ - [0, 0], - [1, 0], - [0, 1], # Tri - # Empty space - [10, 10], - [11, 10], - [11, 11], - [10, 11], # Square - ] - offsets = [0, 3, 3] # Index 3 is start of "empty", Index 3 is also start of Square? - # No, offsets should be strictly increasing or handle 0-length slices. - # vertex_slices uses zip(offsets, chain(offsets[1:], [len(verts)])) - # if offsets = [0, 3, 3], slices are [0:3], [3:3], [3:7] - offsets = [0, 3, 3] - pc = PolyCollection(verts, offsets) - # Polygon(vertices=[]) will fail because of the setter check. - # Let's see if pc.to_polygons() handles it. - # It calls Polygon(vertices=vv) for each slice. - # slice [3:3] gives empty vv. - with pytest.raises(PatternError): - pc.to_polygons() - - -def test_poly_collection_valid() -> None: - verts = [[0, 0], [1, 0], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] - offsets = [0, 3] - pc = PolyCollection(verts, offsets) - assert len(pc.to_polygons()) == 2 - shapes = [Circle(radius=20), Circle(radius=10), Polygon([[0, 0], [10, 0], [10, 10]]), Ellipse(radii=(5, 5))] - sorted_shapes = sorted(shapes) - assert len(sorted_shapes) == 4 - # Just verify it doesn't crash and is stable - assert sorted(sorted_shapes) == sorted_shapes - - -def test_poly_collection_normalized_form_reconstruction_is_independent() -> None: - pc = PolyCollection([[0, 0], [1, 0], [0, 1]], [0]) - _intrinsic, _extrinsic, rebuild = pc.normalized_form(1) - - clone = rebuild() - clone.vertex_offsets[:] = [5] - - assert_equal(pc.vertex_offsets, [0]) - assert_equal(clone.vertex_offsets, [5]) - - -def test_poly_collection_normalized_form_rebuilds_independent_clones() -> None: - pc = PolyCollection([[0, 0], [1, 0], [0, 1]], [0]) - _intrinsic, _extrinsic, rebuild = pc.normalized_form(1) - - first = rebuild() - second = rebuild() - first.vertex_offsets[:] = [7] - - assert_equal(first.vertex_offsets, [7]) - assert_equal(second.vertex_offsets, [0]) - assert_equal(pc.vertex_offsets, [0]) diff --git a/masque/test/test_shape_ordering.py b/masque/test/test_shape_ordering.py new file mode 100644 index 0000000..fea4efa --- /dev/null +++ b/masque/test/test_shape_ordering.py @@ -0,0 +1,15 @@ +from ..shapes import Circle, Ellipse, Polygon + + +def test_shape_comparisons() -> None: + c1 = Circle(radius=10) + c2 = Circle(radius=20) + assert c1 < c2 + assert not (c2 < c1) + + p1 = Polygon([[0, 0], [10, 0], [10, 10]]) + p2 = Polygon([[0, 0], [10, 0], [10, 11]]) + assert p1 < p2 + + assert c1 < p1 or p1 < c1 + assert (c1 < p1) != (p1 < c1) diff --git a/masque/test/test_shape_transforms.py b/masque/test/test_shape_transforms.py new file mode 100644 index 0000000..2a9092a --- /dev/null +++ b/masque/test/test_shape_transforms.py @@ -0,0 +1,44 @@ +from numpy import pi +from numpy.testing import assert_equal, assert_allclose + +from ..shapes import Arc, Ellipse + + +def test_shape_mirror() -> None: + e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) + e.mirror(0) + assert_equal(e.offset, [10, 20]) + assert_allclose(e.rotation, 3 * pi / 4, atol=1e-10) + + a = Arc(radii=(10, 10), angles=(0, pi / 4), width=2, offset=(10, 20)) + a.mirror(0) + assert_equal(a.offset, [10, 20]) + assert_allclose(a.angles, [0, -pi / 4], atol=1e-10) + + a = Arc(radii=(10, 5), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) + a.mirror(1) + assert a.angle_ref == Arc.AngleRef.FocusNeg + + a = Arc(radii=(5, 10), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) + a.mirror(0) + assert a.angle_ref == Arc.AngleRef.FocusNeg + +def test_shape_flip_across() -> None: + e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) + e.flip_across(axis=0) + assert_equal(e.offset, [10, -20]) + assert_allclose(e.rotation, 3 * pi / 4, atol=1e-10) + + e = Ellipse(radii=(10, 5), offset=(10, 20)) + e.flip_across(y=10) + assert_equal(e.offset, [10, 0]) + +def test_shape_scale() -> None: + e = Ellipse(radii=(10, 5)) + e.scale_by(2) + assert_equal(e.radii, [20, 10]) + + a = Arc(radii=(10, 5), angles=(0, pi), width=2) + a.scale_by(0.5) + assert_equal(a.radii, [5, 2.5]) + assert a.width == 1 diff --git a/masque/test/test_shapes.py b/masque/test/test_shapes.py deleted file mode 100644 index e453f7c..0000000 --- a/masque/test/test_shapes.py +++ /dev/null @@ -1,174 +0,0 @@ -import numpy -from numpy.testing import assert_equal, assert_allclose -from numpy import pi - -from ..shapes import Arc, Ellipse, Circle, Polygon, PolyCollection - - -def test_poly_collection_init() -> None: - # Two squares: [[0,0], [1,0], [1,1], [0,1]] and [[10,10], [11,10], [11,11], [10,11]] - verts = [[0, 0], [1, 0], [1, 1], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] - offsets = [0, 4] - pc = PolyCollection(vertex_lists=verts, vertex_offsets=offsets) - assert len(list(pc.polygon_vertices)) == 2 - assert_equal(pc.get_bounds_single(), [[0, 0], [11, 11]]) - - -def test_poly_collection_to_polygons() -> None: - verts = [[0, 0], [1, 0], [1, 1], [0, 1], [10, 10], [11, 10], [11, 11], [10, 11]] - offsets = [0, 4] - pc = PolyCollection(vertex_lists=verts, vertex_offsets=offsets) - polys = pc.to_polygons() - assert len(polys) == 2 - assert_equal(polys[0].vertices, [[0, 0], [1, 0], [1, 1], [0, 1]]) - assert_equal(polys[1].vertices, [[10, 10], [11, 10], [11, 11], [10, 11]]) - - -def test_circle_init() -> None: - c = Circle(radius=10, offset=(5, 5)) - assert c.radius == 10 - assert_equal(c.offset, [5, 5]) - - -def test_circle_to_polygons() -> None: - c = Circle(radius=10) - polys = c.to_polygons(num_vertices=32) - assert len(polys) == 1 - assert isinstance(polys[0], Polygon) - # A circle with 32 vertices should have vertices distributed around (0,0) - bounds = polys[0].get_bounds_single() - assert_allclose(bounds, [[-10, -10], [10, 10]], atol=1e-10) - - -def test_ellipse_init() -> None: - e = Ellipse(radii=(10, 5), offset=(1, 2), rotation=pi / 4) - assert_equal(e.radii, [10, 5]) - assert_equal(e.offset, [1, 2]) - assert e.rotation == pi / 4 - - -def test_ellipse_to_polygons() -> None: - e = Ellipse(radii=(10, 5)) - polys = e.to_polygons(num_vertices=64) - assert len(polys) == 1 - bounds = polys[0].get_bounds_single() - assert_allclose(bounds, [[-10, -5], [10, 5]], atol=1e-10) - - -def test_arc_init() -> None: - a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2, offset=(0, 0)) - assert_equal(a.radii, [10, 10]) - assert_equal(a.angles, [0, pi / 2]) - assert a.width == 2 - - -def test_arc_to_polygons() -> None: - # Quarter circle arc - a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) - polys = a.to_polygons(num_vertices=32) - assert len(polys) == 1 - # Outer radius 11, inner radius 9 - # Quarter circle from 0 to 90 deg - bounds = polys[0].get_bounds_single() - # Min x should be 0 (inner edge start/stop or center if width is large) - # But wait, the arc is centered at 0,0. - # Outer edge goes from (11, 0) to (0, 11) - # Inner edge goes from (9, 0) to (0, 9) - # So x ranges from 0 to 11, y ranges from 0 to 11. - assert_allclose(bounds, [[0, 0], [11, 11]], atol=1e-10) - - -def test_arc_focus_to_polygons() -> None: - a = Arc(radii=(10, 6), angles=(-0.4, 0.7), width=1, angle_ref=Arc.AngleRef.FocusPos) - polys = a.to_polygons(num_vertices=32) - assert len(polys) == 1 - - focus = numpy.array([8.0, 0.0]) - cuts = a.get_cap_edges() - for angle, cut in zip(a.angles, cuts, strict=True): - direction = numpy.array([numpy.cos(angle), numpy.sin(angle)]) - for point in cut: - delta = point - focus - assert_allclose(direction[0] * delta[1] - direction[1] * delta[0], 0, atol=1e-10) - assert numpy.dot(direction, delta) > 0 - - -def test_arc_circle_focus_matches_center() -> None: - center = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) - focus = Arc(radii=(10, 10), angles=(0, pi / 2), width=2, angle_ref=Arc.AngleRef.FocusPos) - - assert_allclose(focus.to_polygons(num_vertices=32)[0].vertices, - center.to_polygons(num_vertices=32)[0].vertices, - atol=1e-10) - - -def test_shape_mirror() -> None: - e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) - e.mirror(0) # Mirror across x axis (axis 0): in-place relative to offset - assert_equal(e.offset, [10, 20]) - # rotation was pi/4, mirrored(0) -> -pi/4 == 3pi/4 (mod pi) - assert_allclose(e.rotation, 3 * pi / 4, atol=1e-10) - - a = Arc(radii=(10, 10), angles=(0, pi / 4), width=2, offset=(10, 20)) - a.mirror(0) - assert_equal(a.offset, [10, 20]) - # For Arc, mirror(0) negates rotation and angles - assert_allclose(a.angles, [0, -pi / 4], atol=1e-10) - - a = Arc(radii=(10, 5), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) - a.mirror(1) - assert a.angle_ref == Arc.AngleRef.FocusNeg - - a = Arc(radii=(5, 10), angles=(0, pi / 4), width=2, angle_ref=Arc.AngleRef.FocusPos) - a.mirror(0) - assert a.angle_ref == Arc.AngleRef.FocusNeg - - -def test_shape_flip_across() -> None: - e = Ellipse(radii=(10, 5), offset=(10, 20), rotation=pi / 4) - e.flip_across(axis=0) # Mirror across y=0: flips y-offset - assert_equal(e.offset, [10, -20]) - # rotation also flips: -pi/4 == 3pi/4 (mod pi) - assert_allclose(e.rotation, 3 * pi / 4, atol=1e-10) - # Mirror across specific y - e = Ellipse(radii=(10, 5), offset=(10, 20)) - e.flip_across(y=10) # Mirror across y=10 - # y=20 mirrored across y=10 -> y=0 - assert_equal(e.offset, [10, 0]) - - -def test_shape_scale() -> None: - e = Ellipse(radii=(10, 5)) - e.scale_by(2) - assert_equal(e.radii, [20, 10]) - - a = Arc(radii=(10, 5), angles=(0, pi), width=2) - a.scale_by(0.5) - assert_equal(a.radii, [5, 2.5]) - assert a.width == 1 - - -def test_shape_arclen() -> None: - # Test that max_arclen correctly limits segment lengths - - # Ellipse - e = Ellipse(radii=(10, 5)) - # Approximate perimeter is ~48.4 - # With max_arclen=5, should have > 10 segments - polys = e.to_polygons(max_arclen=5) - v = polys[0].vertices - dist = numpy.sqrt(numpy.sum(numpy.diff(v, axis=0, append=v[:1]) ** 2, axis=1)) - assert numpy.all(dist <= 5.000001) - assert len(v) > 10 - - # Arc - a = Arc(radii=(10, 10), angles=(0, pi / 2), width=2) - # Outer perimeter is 11 * pi/2 ~ 17.27 - # Inner perimeter is 9 * pi/2 ~ 14.14 - # With max_arclen=2, should have > 8 segments on outer edge - polys = a.to_polygons(max_arclen=2) - v = polys[0].vertices - # Arc polygons are closed, but contain both inner and outer edges and caps - # Let's just check that all segment lengths are within limit - dist = numpy.sqrt(numpy.sum(numpy.diff(v, axis=0, append=v[:1]) ** 2, axis=1)) - assert numpy.all(dist <= 2.000001) diff --git a/masque/test/test_text.py b/masque/test/test_text.py new file mode 100644 index 0000000..1b6770d --- /dev/null +++ b/masque/test/test_text.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pytest +import numpy + +from ..shapes import Polygon, Text + + +def test_text_to_polygons() -> None: + pytest.importorskip("freetype") + font_path = "/usr/share/fonts/truetype/dejavu/DejaVuMathTeXGyre.ttf" + if not Path(font_path).exists(): + pytest.skip("Font file not found") + + t = Text("Hi", height=10, font_path=font_path) + polys = t.to_polygons() + assert len(polys) > 0 + assert all(isinstance(p, Polygon) for p in polys) + + # Each character produces polygons with distinct horizontal placement. + char_x_means = [p.vertices[:, 0].mean() for p in polys] + assert len(set(char_x_means)) >= 2 + +def test_text_bounds_and_normalized_form() -> None: + pytest.importorskip("freetype") + font_path = "/usr/share/fonts/truetype/dejavu/DejaVuMathTeXGyre.ttf" + if not Path(font_path).exists(): + pytest.skip("Font file not found") + + text = Text("Hi", height=10, font_path=font_path) + _intrinsic, extrinsic, ctor = text.normalized_form(5) + normalized = ctor() + + assert extrinsic[1] == 2 + assert normalized.height == 5 + + bounds = text.get_bounds_single() + assert bounds is not None + assert numpy.isfinite(bounds).all() + assert numpy.all(bounds[1] > bounds[0]) + +def test_text_mirroring_affects_comparison() -> None: + text = Text("A", height=10, font_path="dummy.ttf") + mirrored = Text("A", height=10, font_path="dummy.ttf", mirrored=True) + + assert text != mirrored + assert (text < mirrored) != (mirrored < text) diff --git a/masque/test/test_utils.py b/masque/test/test_utils.py index ddab9cd..d142896 100644 --- a/masque/test/test_utils.py +++ b/masque/test/test_utils.py @@ -33,19 +33,13 @@ def test_remove_colinear_vertices() -> None: def test_remove_colinear_vertices_exhaustive() -> None: - # U-turn v = [[0, 0], [10, 0], [0, 0]] v_clean = remove_colinear_vertices(v, closed_path=False, preserve_uturns=True) - # Open path should keep ends. [10,0] is between [0,0] and [0,0]? - # They are colinear, but it's a 180 degree turn. - # We preserve 180 degree turns if preserve_uturns is True. assert len(v_clean) == 3 v_collapsed = remove_colinear_vertices(v, closed_path=False, preserve_uturns=False) - # If not preserving u-turns, it should collapse to just the endpoints assert len(v_collapsed) == 2 - # 180 degree U-turn in closed path v = [[0, 0], [10, 0], [5, 0]] v_clean = remove_colinear_vertices(v, closed_path=True, preserve_uturns=False) assert len(v_clean) == 2 diff --git a/masque/test/test_visualize.py b/masque/test/test_visualize.py index 4dab435..a20286c 100644 --- a/masque/test/test_visualize.py +++ b/masque/test/test_visualize.py @@ -43,7 +43,6 @@ def test_visualize_noninteractive(tmp_path) -> None: def test_visualize_empty() -> None: """ Test visualizing an empty pattern. """ pat = Pattern() - # Should not raise pat.visualize(overdraw=True) @pytest.mark.skipif(not HAS_MATPLOTLIB, reason="matplotlib not installed") @@ -51,5 +50,4 @@ def test_visualize_no_refs() -> None: """ Test visualizing a pattern with only local shapes (no library needed). """ pat = Pattern() pat.polygon('L1', [[0, 0], [1, 0], [0, 1]]) - # Should not raise even if library is None pat.visualize(overdraw=True) From 844e0401749707aff027246b857cabea14d715c2 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 20/36] [gdsii_arrow] add gdsii_arrow --- masque/file/gdsii_arrow.py | 453 +++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..e56a48e --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,453 @@ +# ruff: noqa: ARG001, F401 +""" +GDSII file format readers and writers using the `TODO` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_equal +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path, PolyCollection +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def _read_to_arrow( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> pyarrow.Array: + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + return arrow_arr + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_tups = [tuple(pair) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + rf = libarr['cells'].values.field('refs') + refs = dict( + offsets = rf.offsets.to_numpy(), + targets = rf.values.field('target').to_numpy(), + xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), + angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + scale = rf.values.field('mag').fill_null(1).to_numpy(), + rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), + rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + prop_off = rf.values.field('properties').offsets.to_numpy(), + prop_key = rf.values.field('properties').values.field('key').to_numpy(), + prop_val = rf.values.field('properties').values.field('value').to_pylist(), + ) + + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), + ) + + elements = dict( + boundaries = get_geom(libarr, 'boundaries'), + paths = get_geom(libarr, 'paths'), + boxes = get_geom(libarr, 'boxes'), + nodes = get_geom(libarr, 'nodes'), + texts = texts, + refs = refs, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').fill_null(0).to_numpy(), + path_type = paths.values.field('path_type').fill_null(0).to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').fill_null(0).to_numpy(), + paths.values.field('extension_end').fill_null(0).to_numpy(), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + raw_mode = raw_mode, + ) + + mlib = Library() + for cc in range(len(libarr['cells'])): + name = cell_names[cell_ids[cc]] + pat = Pattern() + _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def _grefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + targets = elem['targets'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_invert_y = elem['invert_y'][elem_slc][:elem_count] + elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] + elem_scale = elem['scale'][elem_slc][:elem_count] + elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] + elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] + elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] + rep_valid = elem['rep_valid'][elem_slc][:elem_count] + + + for ee in range(elem_count): + target = cell_names[targets[ee]] + offset = xy[ee] + mirr = elem_invert_y[ee] + rot = elem_angle_rad[ee] + mag = elem_scale[ee] + + rep: None | Grid = None + if rep_valid[ee]: + a_vector = elem_rep_xy0[ee] + b_vector = elem_rep_xy1[ee] + a_count, b_count = elem_rep_counts[ee] + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + pat.refs[target].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_strings = elem['string'][elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + offset = xy[ee] + string = elem_strings[ee] + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + mlabel = Label(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_widths = elem['width'][elem_slc][:elem_count] + elem_path_types = elem['path_type'][elem_slc][:elem_count] + elem_extensions = elem['extensions'][elem_slc][:elem_count] + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem_widths[ee] + cap_int = elem_path_types[ee] + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem_extensions[ee] + else: + cap_extensions = None + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + width=width, cap=cap,cap_extensions=cap_extensions) + pat.shapes[layer].append(path) + + +def _boundaries_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + xy_counts = xy_offs[1:] - xy_offs[:-1] + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + prop_counts = prop_offs[1:] - prop_offs[:-1] + elem_layer_inds = layer_inds[elem_slc][:elem_count] + + order = numpy.argsort(elem_layer_inds, stable=True) + unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) + + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): + ee_inds = order[ff:ff + nn] + layer = layer_tups[layer_ind] + propless_mask = prop_counts[ee_inds] == 0 + + poly_count_on_layer = propless_mask.sum() + if poly_count_on_layer == 1: + propless_mask[:] = 0 # Never make a 1-element collection + elif poly_count_on_layer > 1: + propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point + vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) + vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) + + for ii, ee in enumerate(ee_inds[propless_mask]): + vo = vertex_offsets[ii] + vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] + + polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) + pat.shapes[layer].append(polys) + + # Handle single polygons + for ee in ee_inds[~propless_mask]: + layer = layer_tups[elem_layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + + annotations: None | dict[str, list[int | float | str]] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} + + poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) + + +#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: +# return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From 1dbf195251f5bc6b62064a6273ac44e185cfa5df Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:20:50 -0700 Subject: [PATCH 21/36] add some missing deps --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af8802c..ad03fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ [dependency-groups] dev = [ - "pytest", + "masque[arrow]", "masque[oasis]", "masque[dxf]", "masque[svg]", @@ -52,6 +52,7 @@ dev = [ "masque[text]", "masque[manhattanize]", "masque[manhattanize_slow]", + "masque[boolean]", "matplotlib>=3.10.8", "pytest>=9.0.2", "ruff>=0.15.5", @@ -66,6 +67,7 @@ build-backend = "hatchling.build" path = "masque/__init__.py" [project.optional-dependencies] +arrow = ["pyarrow", "cffi"] oasis = ["fatamorgana~=0.11"] dxf = ["ezdxf~=1.4"] svg = ["svgwrite"] @@ -121,4 +123,3 @@ mypy_path = "stubs" python_version = "3.11" strict = false check_untyped_defs = true - From 74878063186f6725c77c7b62f83769d79edf6dec Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:21:59 -0700 Subject: [PATCH 22/36] [gdsii_arrow] misc correctness work --- masque/file/gdsii_arrow.py | 173 +++++++++++++++++++++++------- masque/test/test_gdsii_arrow.py | 180 ++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 37 deletions(-) create mode 100644 masque/test/test_gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index e56a48e..546d575 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -25,12 +25,16 @@ Notes: """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable +from importlib.machinery import EXTENSION_SUFFIXES +import importlib.util import io import mmap import logging +import os import pathlib import gzip import string +import sys from pprint import pformat import numpy @@ -49,9 +53,10 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +clib: Any | None = None + path_cap_map = { 0: Path.Cap.Flush, @@ -65,16 +70,123 @@ def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) +def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]: + layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16) + dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16) + return numpy.stack((layer, dtype), axis=-1) + + +def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]: + a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64) + b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64) + return numpy.stack((a_count, b_count), axis=-1) + + +def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]: + xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32) + yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32) + return numpy.stack((xx, yy), axis=-1) + + +def _local_library_filename() -> str: + if sys.platform.startswith('linux'): + return 'libklamath_rs_ext.so' + if sys.platform == 'darwin': + return 'libklamath_rs_ext.dylib' + if sys.platform == 'win32': + return 'klamath_rs_ext.dll' + raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}') + + +def _installed_library_candidates() -> list[pathlib.Path]: + candidates: list[pathlib.Path] = [] + + try: + spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext') + except ModuleNotFoundError: + spec = None + if spec is not None and spec.origin is not None: + candidates.append(pathlib.Path(spec.origin)) + + try: + pkg_spec = importlib.util.find_spec('klamath_rs_ext') + except ModuleNotFoundError: + pkg_spec = None + if pkg_spec is not None and pkg_spec.submodule_search_locations is not None: + for location in pkg_spec.submodule_search_locations: + pkg_dir = pathlib.Path(location) + for suffix in EXTENSION_SUFFIXES: + candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}'))) + + return candidates + + +def _repo_library_candidates() -> list[pathlib.Path]: + repo_root = pathlib.Path(__file__).resolve().parents[2] + library_name = _local_library_filename() + return [ + repo_root / 'klamath-rs' / 'target' / 'release' / library_name, + repo_root / 'klamath-rs' / 'target' / 'debug' / library_name, + ] + + +def find_klamath_rs_library() -> pathlib.Path | None: + env_path = os.environ.get('KLAMATH_RS_EXT_LIB') + if env_path: + candidate = pathlib.Path(env_path).expanduser() + if candidate.exists(): + return candidate.resolve() + + seen: set[pathlib.Path] = set() + for candidate in _installed_library_candidates() + _repo_library_candidates(): + resolved = candidate.expanduser() + if resolved in seen: + continue + seen.add(resolved) + if resolved.exists(): + return resolved.resolve() + return None + + +def is_available() -> bool: + return find_klamath_rs_library() is not None + + +def _get_clib() -> Any: + global clib + if clib is None: + lib_path = find_klamath_rs_library() + if lib_path is None: + raise ImportError( + 'Could not locate klamath_rs_ext shared library. ' + 'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` ' + 'or set KLAMATH_RS_EXT_LIB to the built library path.' + ) + clib = ffi.dlopen(str(lib_path)) + return clib + + +def _read_annotations( + prop_offs: NDArray[numpy.integer[Any]], + prop_key: NDArray[numpy.integer[Any]], + prop_val: list[str], + ee: int, + ) -> annotations_t: + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii >= prop_ff: + return None + return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + def _read_to_arrow( filename: str | pathlib.Path, *args, **kwargs, ) -> pyarrow.Array: - path = pathlib.Path(filename) - path.resolve() + path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - clib.read_path(str(path).encode(), ptr_array, ptr_schema) + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) @@ -132,7 +244,7 @@ def read_arrow( """ library_info = _read_header(libarr) - layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) layer_tups = [tuple(pair) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() @@ -155,14 +267,14 @@ def read_arrow( refs = dict( offsets = rf.offsets.to_numpy(), targets = rf.values.field('target').to_numpy(), - xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), scale = rf.values.field('mag').fill_null(1).to_numpy(), rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), - rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), + rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), + rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), prop_off = rf.values.field('properties').offsets.to_numpy(), prop_key = rf.values.field('properties').values.field('key').to_numpy(), prop_val = rf.values.field('properties').values.field('value').to_pylist(), @@ -172,7 +284,7 @@ def read_arrow( texts = dict( offsets = txt.offsets.to_numpy(), layer_inds = txt.values.field('layer').to_numpy(), - xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()), string = txt.values.field('string').to_pylist(), prop_off = txt.values.field('properties').offsets.to_numpy(), prop_key = txt.values.field('properties').values.field('key').to_numpy(), @@ -222,9 +334,9 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: Read the file header and create the library_info dict. """ library_info = dict( - name = libarr['lib_name'], - meters_per_unit = libarr['meters_per_db_unit'], - logical_units_per_unit = libarr['user_units_per_db_unit'], + name = libarr['lib_name'].as_py(), + meters_per_unit = libarr['meters_per_db_unit'].as_py(), + logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(), ) return library_info @@ -245,6 +357,8 @@ def _grefs_to_mrefs( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_targets = targets[elem_slc][:elem_count] + elem_xy = xy[elem_slc][:elem_count] elem_invert_y = elem['invert_y'][elem_slc][:elem_count] elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] elem_scale = elem['scale'][elem_slc][:elem_count] @@ -255,8 +369,8 @@ def _grefs_to_mrefs( for ee in range(elem_count): - target = cell_names[targets[ee]] - offset = xy[ee] + target = cell_names[elem_targets[ee]] + offset = elem_xy[ee] mirr = elem_invert_y[ee] rot = elem_angle_rad[ee] mag = elem_scale[ee] @@ -268,11 +382,7 @@ def _grefs_to_mrefs( a_count, b_count = elem_rep_counts[ee] rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -293,19 +403,16 @@ def _texts_to_labels( elem_count = elem_off[cc + 1] - elem_off[cc] elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] - offset = xy[ee] + offset = elem_xy[ee] string = elem_strings[ee] - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -345,11 +452,7 @@ def _gpaths_to_mpaths( else: cap_extensions = None - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) @@ -406,11 +509,7 @@ def _boundaries_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - annotations: None | dict[str, list[int | float | str]] = None - prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] - if prop_ii < prop_ff: - annotations = {str(prop_key[off]): prop_val[off] for off in range(prop_ii, prop_ff)} - + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py new file mode 100644 index 0000000..26713e9 --- /dev/null +++ b/masque/test/test_gdsii_arrow.py @@ -0,0 +1,180 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..shapes import Path as MPath +from ..file import gdsii, gdsii_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: + if annotations is None: + return None + return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) + + +def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(values, dtype=float)).astype(int) + if arr.ndim == 1: + return tuple(arr.tolist()) + return tuple(tuple(row.tolist()) for row in arr) + + +def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: + if isinstance(shape, MPath): + cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) + return [( + 'path', + layer, + _coord_key(shape.vertices), + _coord_key(shape.offset), + int(round(float(shape.width))), + shape.cap.name, + cap_extensions, + _annotations_key(shape.annotations), + )] + + keys = [] + for poly in shape.to_polygons(): + keys.append(( + 'polygon', + layer, + _coord_key(poly.vertices), + _coord_key(poly.offset), + _annotations_key(poly.annotations), + )) + return keys + + +def _ref_key(target: str, ref: object) -> tuple[object, ...]: + repetition = None + if ref.repetition is not None: + repetition = ( + _coord_key(ref.repetition.a_vector), + int(ref.repetition.a_count), + _coord_key(ref.repetition.b_vector), + int(ref.repetition.b_count), + ) + return ( + target, + _coord_key(ref.offset), + round(float(ref.rotation), 8), + round(float(ref.scale), 8), + bool(ref.mirrored), + repetition, + _annotations_key(ref.annotations), + ) + + +def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: + return ( + layer, + label.string, + _coord_key(label.offset), + _annotations_key(label.annotations), + ) + + +def _pattern_summary(pattern: Pattern) -> dict[str, object]: + shape_keys: list[tuple[object, ...]] = [] + for layer, shapes in pattern.shapes.items(): + for shape in shapes: + shape_keys.extend(_shape_key(shape, layer)) + + ref_keys = [ + _ref_key(target, ref) + for target, refs in pattern.refs.items() + for ref in refs + ] + + label_keys = [ + _label_key(layer, label) + for layer, labels in pattern.labels.items() + for label in labels + ] + + return { + 'shapes': sorted(shape_keys), + 'refs': sorted(ref_keys), + 'labels': sorted(label_keys), + } + + +def _library_summary(lib: Library) -> dict[str, dict[str, object]]: + return {name: _pattern_summary(pattern) for name, pattern in lib.items()} + + +def _make_arrow_test_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) + lib['leaf'] = leaf + + child = Pattern() + child.path( + (2, 0), + vertices=[[0, 0], [15, 5], [30, 5]], + width=6, + cap=MPath.Cap.SquareCustom, + cap_extensions=(2, 4), + annotations={'2': ['child-path']}, + ) + child.label((11, 0), string='CHILD', offset=(7, 8), annotations={'11': ['child-label']}) + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25, annotations={'12': ['child-ref']}) + lib['child'] = child + + sibling = Pattern() + sibling.polygon((3, 0), vertices=[[0, 0], [5, 0], [5, 6], [0, 6]]) + sibling.label((12, 0), string='SIB', offset=(1, 2), annotations={'13': ['sib-label']}) + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + annotations={'14': ['sib-ref']}, + ) + lib['sibling'] = sibling + + top = Pattern() + top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) + lib['top'] = top + + return lib + + +def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + +def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_smoke.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert 'TOP' in lib + assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 From e2ebbbe2a5601af7757e63cfc92aa6ae95b553fa Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 13:22:27 -0700 Subject: [PATCH 23/36] [gdsii] add some profiling helpers --- examples/generate_gds_perf.py | 5 + examples/profile_gdsii_readers.py | 74 ++++ masque/file/gdsii_perf.py | 633 ++++++++++++++++++++++++++++++ masque/test/test_gdsii_perf.py | 24 ++ 4 files changed, 736 insertions(+) create mode 100644 examples/generate_gds_perf.py create mode 100644 examples/profile_gdsii_readers.py create mode 100644 masque/file/gdsii_perf.py create mode 100644 masque/test/test_gdsii_perf.py diff --git a/examples/generate_gds_perf.py b/examples/generate_gds_perf.py new file mode 100644 index 0000000..6bdd999 --- /dev/null +++ b/examples/generate_gds_perf.py @@ -0,0 +1,5 @@ +from masque.file.gdsii_perf import main + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py new file mode 100644 index 0000000..c77c589 --- /dev/null +++ b/examples/profile_gdsii_readers.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import importlib +import json +import time +from pathlib import Path + +from masque import LibraryError + + +READERS = { + 'gdsii': ('masque.file.gdsii', 'readfile'), + 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), + } + + +def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: + assert hasattr(lib, '__len__') + assert hasattr(lib, 'tops') + tops = lib.tops() # type: ignore[no-any-return, attr-defined] + try: + unique_top = lib.top() # type: ignore[no-any-return, attr-defined] + except LibraryError: + unique_top = None + + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'library_name': info['name'], + 'cell_count': len(lib), # type: ignore[arg-type] + 'topcells': tops, + 'topcell': unique_top, + } + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') + parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--path', type=Path, required=True) + parser.add_argument('--warmup', type=int, default=1) + parser.add_argument('--repeat', type=int, default=1) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + module_name, attr_name = READERS[args.reader] + readfile = getattr(importlib.import_module(module_name), attr_name) + path = args.path.expanduser().resolve() + + for _ in range(args.warmup): + readfile(path) + + runs = [] + for _ in range(args.repeat): + start = time.perf_counter() + lib, info = readfile(path) + elapsed_s = time.perf_counter() - start + runs.append(_summarize(path, elapsed_s, info, lib)) + + print(json.dumps({ + 'reader': args.reader, + 'warmup': args.warmup, + 'repeat': args.repeat, + 'runs': runs, + }, indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/file/gdsii_perf.py b/masque/file/gdsii_perf.py new file mode 100644 index 0000000..38d1a7d --- /dev/null +++ b/masque/file/gdsii_perf.py @@ -0,0 +1,633 @@ +""" +Synthetic GDS fixture generation for reader/writer performance testing. + +The presets here are intentionally hierarchical and deterministic. They aim to +approximate a pair of real-world layout families discussed during GDS reader and +writer work: + +* `many_cells`: tens of thousands of cells, moderate reference count, very heavy + box usage after flattening, and moderate polygon density. +* `many_instances`: a much smaller cell library with very high reference count, + similar box density, and far fewer polygons. + +Fixtures are written by streaming structures through `klamath` directly so large +benchmark files can be produced without first materializing an equally large +`masque.Library` in Python. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +import argparse +import json +import math + +import numpy +import klamath +from klamath import elements + + +EMPTY_PROPERTIES: dict[int, bytes] = {} +METERS_PER_DB_UNIT = 1e-9 +USER_UNITS_PER_DB_UNIT = 1e-3 +TOTAL_LAYERS = 200 + + +@dataclass(frozen=True) +class FixturePreset: + name: str + total_layers: int + box_layers: int + heavy_box_layers: int + polygon_layers: int + box_cells: int + poly_cells: int + box_wrappers: int + poly_wrappers: int + box_clusters: int + poly_clusters: int + box_cluster_refs: int + poly_cluster_refs: int + top_direct_box_refs: int + top_direct_poly_refs: int + heavy_boxes_per_cell: int + regular_boxes_per_cell: int + polygons_per_cell: int + path_stride: int + text_stride: int + box_cluster_array: tuple[int, int] + top_box_array: tuple[int, int] + poly_cluster_array: tuple[int, int] + top_poly_array: tuple[int, int] + rare_annotation_stride: int + + +PRESETS: dict[str, FixturePreset] = { + 'many_cells': FixturePreset( + name='many_cells', + total_layers=TOTAL_LAYERS, + box_layers=20, + heavy_box_layers=3, + polygon_layers=20, + box_cells=17_000, + poly_cells=6_000, + box_wrappers=18_000, + poly_wrappers=6_000, + box_clusters=2_000, + poly_clusters=999, + box_cluster_refs=24, + poly_cluster_refs=16, + top_direct_box_refs=21_000, + top_direct_poly_refs=7_000, + heavy_boxes_per_cell=6, + regular_boxes_per_cell=2, + polygons_per_cell=50, + path_stride=2, + text_stride=3, + box_cluster_array=(24, 16), + top_box_array=(8, 8), + poly_cluster_array=(4, 2), + top_poly_array=(3, 2), + rare_annotation_stride=1_250, + ), + 'many_instances': FixturePreset( + name='many_instances', + total_layers=TOTAL_LAYERS, + box_layers=25, + heavy_box_layers=3, + polygon_layers=10, + box_cells=2_500, + poly_cells=500, + box_wrappers=1_000, + poly_wrappers=500, + box_clusters=1_000, + poly_clusters=499, + box_cluster_refs=1_200, + poly_cluster_refs=400, + top_direct_box_refs=102_001, + top_direct_poly_refs=0, + heavy_boxes_per_cell=40, + regular_boxes_per_cell=16, + polygons_per_cell=60, + path_stride=1, + text_stride=2, + box_cluster_array=(1, 1), + top_box_array=(1, 1), + poly_cluster_array=(1, 1), + top_poly_array=(1, 1), + rare_annotation_stride=250, + ), + } + + +@dataclass(frozen=True) +class FixtureManifest: + preset: str + scale: float + gds_path: str + library_name: str + cells: int + refs: int + layers: int + box_layers: int + heavy_box_layers: list[list[int]] + polygon_layers: list[list[int]] + hierarchical_boxes_per_heavy_layer: int + hierarchical_boxes_per_regular_layer: int + hierarchical_polygons_total: int + hierarchical_paths_total: int + hierarchical_texts_total: int + flattened_box_placements: int + flattened_poly_placements: int + estimated_flat_boxes_per_heavy_layer: int + estimated_flat_polygons_per_active_polygon_layer: int + + +def _scaled_count(value: int, scale: float, minimum: int = 0) -> int: + if value == 0: + return 0 + scaled = int(math.ceil(value * scale)) + return max(minimum, scaled) + + +def _scaled_preset(preset: FixturePreset, scale: float) -> FixturePreset: + if scale <= 0: + raise ValueError(f'scale must be positive, got {scale!r}') + + return FixturePreset( + name=preset.name, + total_layers=preset.total_layers, + box_layers=min(preset.box_layers, preset.total_layers), + heavy_box_layers=min(preset.heavy_box_layers, preset.box_layers), + polygon_layers=min(preset.polygon_layers, preset.total_layers), + box_cells=_scaled_count(preset.box_cells, scale, minimum=1), + poly_cells=_scaled_count(preset.poly_cells, scale, minimum=1), + box_wrappers=_scaled_count(preset.box_wrappers, scale), + poly_wrappers=_scaled_count(preset.poly_wrappers, scale), + box_clusters=_scaled_count(preset.box_clusters, scale, minimum=1), + poly_clusters=_scaled_count(preset.poly_clusters, scale, minimum=1), + box_cluster_refs=_scaled_count(preset.box_cluster_refs, scale, minimum=1), + poly_cluster_refs=_scaled_count(preset.poly_cluster_refs, scale, minimum=1), + top_direct_box_refs=_scaled_count(preset.top_direct_box_refs, scale), + top_direct_poly_refs=_scaled_count(preset.top_direct_poly_refs, scale), + heavy_boxes_per_cell=max(1, preset.heavy_boxes_per_cell), + regular_boxes_per_cell=max(1, preset.regular_boxes_per_cell), + polygons_per_cell=max(1, preset.polygons_per_cell), + path_stride=max(1, preset.path_stride), + text_stride=max(1, preset.text_stride), + box_cluster_array=preset.box_cluster_array, + top_box_array=preset.top_box_array, + poly_cluster_array=preset.poly_cluster_array, + top_poly_array=preset.top_poly_array, + rare_annotation_stride=max(1, _scaled_count(preset.rare_annotation_stride, scale, minimum=1)), + ) + + +def _rect_xy(xmin: int, ymin: int, xmax: int, ymax: int) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + return numpy.array( + [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]], + dtype=numpy.int32, + ) + + +def _poly_xy(points: list[tuple[int, int]]) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + closed = points + [points[0]] + return numpy.array(closed, dtype=numpy.int32) + + +def _sref( + target: str, + xy: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([xy], dtype=numpy.int32), + colrow=None, + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _aref( + target: str, + origin: tuple[int, int], + counts: tuple[int, int], + step: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + cols, rows = counts + dx, dy = step + xy = numpy.array( + [ + origin, + (origin[0] + cols * dx, origin[1]), + (origin[0], origin[1] + rows * dy), + ], + dtype=numpy.int32, + ) + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=xy, + colrow=(cols, rows), + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _annotation(index: int) -> dict[int, bytes]: + return {1: f'perf-{index}'.encode('ASCII')} + + +def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 17) * 600 + ybase = (index // 17) * 180 + + for layer in range(cfg.heavy_box_layers): + for box_idx in range(cfg.heavy_boxes_per_cell): + x0 = xbase + box_idx * 22 + y0 = ybase + layer * 40 + width = 10 + ((index + box_idx + layer) % 7) * 6 + height = 10 + ((index * 3 + box_idx + layer) % 5) * 8 + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 and box_idx == 0 and layer == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=properties, + )) + + for layer in range(cfg.heavy_box_layers, cfg.box_layers): + for box_idx in range(cfg.regular_boxes_per_cell): + x0 = xbase + box_idx * 38 + y0 = ybase + (layer - cfg.heavy_box_layers) * 28 + 400 + width = 18 + ((index + layer + box_idx) % 9) * 4 + height = 12 + ((index + 2 * layer + box_idx) % 6) * 5 + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=EMPTY_PROPERTIES, + )) + + return cell_elements + + +def _make_poly_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 19) * 900 + ybase = (index // 19) * 260 + + for poly_idx in range(cfg.polygons_per_cell): + layer = poly_idx % cfg.polygon_layers + dx = xbase + (poly_idx % 5) * 120 + dy = ybase + (poly_idx // 5) * 80 + size = 18 + ((index + poly_idx + layer) % 11) * 7 + points = [ + (dx, dy), + (dx + size, dy + size // 5), + (dx + size + size // 3, dy + size), + (dx + size // 2, dy + size + size // 2), + (dx - size // 4, dy + size // 2), + ] + properties = _annotation(index) if poly_idx == 0 and index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_poly_xy(points), + properties=properties, + )) + + if index % cfg.path_stride == 0: + layer = index % cfg.polygon_layers + cell_elements.append(elements.Path( + layer=(layer, 1), + path_type=2, + width=12 + (index % 5) * 4, + extension=(0, 0), + xy=numpy.array( + [ + [xbase, ybase + 900], + [xbase + 240, ybase + 930], + [xbase + 420, ybase + 960], + ], + dtype=numpy.int32, + ), + properties=EMPTY_PROPERTIES, + )) + + if index % cfg.text_stride == 0: + layer = index % cfg.polygon_layers + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Text( + layer=(layer, 2), + presentation=0, + path_type=0, + width=0, + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([[xbase + 64, ybase + 1536]], dtype=numpy.int32), + string=f'T{index:05d}'.encode('ASCII'), + properties=properties, + )) + + return cell_elements + + +def _write_struct(stream: Any, name: str, cell_elements: list[elements.Element]) -> None: + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=cell_elements) + + +def _box_name(index: int) -> str: + return f'box_{index:05d}' + + +def _poly_name(index: int) -> str: + return f'poly_{index:05d}' + + +def _box_wrapper_name(index: int) -> str: + return f'box_wrap_{index:05d}' + + +def _poly_wrapper_name(index: int) -> str: + return f'poly_wrap_{index:05d}' + + +def _box_cluster_name(index: int) -> str: + return f'box_cluster_{index:05d}' + + +def _poly_cluster_name(index: int) -> str: + return f'poly_cluster_{index:05d}' + + +def _write_box_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_cells): + _write_struct(stream, _box_name(idx), _make_box_cell(_box_name(idx), idx, cfg)) + + +def _write_poly_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.poly_cells): + _write_struct(stream, _poly_name(idx), _make_poly_cell(_poly_name(idx), idx, cfg)) + + +def _write_wrappers(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_wrappers): + target = _box_name(idx % cfg.box_cells) + origin = ((idx % 97) * 2_000, (idx // 97) * 2_000) + _write_struct(stream, _box_wrapper_name(idx), [_sref(target, origin)]) + + for idx in range(cfg.poly_wrappers): + target = _poly_name(idx % cfg.poly_cells) + origin = ((idx % 61) * 3_200, (idx // 61) * 3_200) + _write_struct(stream, _poly_wrapper_name(idx), [_sref(target, origin)]) + + +def _write_box_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + for idx in range(cfg.box_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.box_cluster_refs): + target = _box_name((idx * cfg.box_cluster_refs + ref_idx) % cfg.box_cells) + origin = ( + (ref_idx % 6) * 48_000, + (ref_idx // 6) * 48_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.box_cluster_array, (720, 900))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _box_cluster_name(idx), cell_elements) + + +def _write_poly_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + for idx in range(cfg.poly_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.poly_cluster_refs): + target = _poly_name((idx * cfg.poly_cluster_refs + ref_idx) % cfg.poly_cells) + origin = ( + (ref_idx % 10) * 96_000, + (ref_idx // 10) * 96_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.poly_cluster_array, (12_000, 8_500))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _poly_cluster_name(idx), cell_elements) + + +def _top_box_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.box_wrappers): + refs.append(_sref( + _box_wrapper_name(idx), + ((idx % 240) * 240_000, (idx // 240) * 240_000), + )) + + for idx in range(cfg.box_clusters): + refs.append(_sref( + _box_cluster_name(idx), + ((idx % 100) * 800_000, (idx // 100) * 800_000 + 14_000_000), + )) + + for idx in range(cfg.top_direct_box_refs): + target = _box_name(idx % cfg.box_cells) + origin = ( + (idx % 150) * 160_000, + (idx // 150) * 160_000 + 26_000_000, + ) + if cfg.top_box_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_box_array, (1_100, 1_350))) + + return refs + + +def _top_poly_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.poly_wrappers): + refs.append(_sref( + _poly_wrapper_name(idx), + ((idx % 180) * 360_000, (idx // 180) * 360_000 + 44_000_000), + )) + + for idx in range(cfg.poly_clusters): + refs.append(_sref( + _poly_cluster_name(idx), + ((idx % 70) * 1_100_000, (idx // 70) * 1_100_000 + 58_000_000), + )) + + for idx in range(cfg.top_direct_poly_refs): + target = _poly_name(idx % cfg.poly_cells) + origin = ( + (idx % 110) * 420_000, + (idx // 110) * 420_000 + 72_000_000, + ) + if cfg.top_poly_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_poly_array, (16_000, 14_000))) + + return refs + + +def _write_top(stream: Any, cfg: FixturePreset) -> None: + cell_elements: list[elements.Element] = [] + cell_elements.extend(_top_box_refs(cfg)) + cell_elements.extend(_top_poly_refs(cfg)) + _write_struct(stream, 'TOP', cell_elements) + + +def _poly_paths_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.path_stride + 1 + + +def _poly_texts_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.text_stride + 1 + + +def _ref_instances_per_box_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] + return array_refs * array_mult + (cfg.box_cluster_refs - array_refs) + + +def _ref_instances_per_poly_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] + return array_refs * array_mult + (cfg.poly_cluster_refs - array_refs) + + +def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> FixtureManifest: + base = PRESETS[preset] + cfg = _scaled_preset(base, scale) + + flattened_box_placements = ( + cfg.box_wrappers + + cfg.box_clusters * _ref_instances_per_box_cluster(cfg) + + cfg.top_direct_box_refs * cfg.top_box_array[0] * cfg.top_box_array[1] + ) + flattened_poly_placements = ( + cfg.poly_wrappers + + cfg.poly_clusters * _ref_instances_per_poly_cluster(cfg) + + cfg.top_direct_poly_refs * cfg.top_poly_array[0] * cfg.top_poly_array[1] + ) + polygon_layers = max(1, cfg.polygon_layers) + polys_per_layer = (cfg.poly_cells * cfg.polygons_per_cell) // polygon_layers + + return FixtureManifest( + preset=cfg.name, + scale=scale, + gds_path=str(Path(path)), + library_name=f'masque-perf-{cfg.name}', + cells=cfg.box_cells + cfg.poly_cells + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + 1, + refs=( + cfg.box_wrappers + + cfg.poly_wrappers + + cfg.box_clusters * cfg.box_cluster_refs + + cfg.poly_clusters * cfg.poly_cluster_refs + + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + + cfg.top_direct_box_refs + cfg.top_direct_poly_refs + ), + layers=cfg.total_layers, + box_layers=cfg.box_layers, + heavy_box_layers=[[layer, 0] for layer in range(cfg.heavy_box_layers)], + polygon_layers=[[layer, 0] for layer in range(cfg.polygon_layers)], + hierarchical_boxes_per_heavy_layer=cfg.box_cells * cfg.heavy_boxes_per_cell, + hierarchical_boxes_per_regular_layer=cfg.box_cells * cfg.regular_boxes_per_cell, + hierarchical_polygons_total=cfg.poly_cells * cfg.polygons_per_cell, + hierarchical_paths_total=_poly_paths_total(cfg), + hierarchical_texts_total=_poly_texts_total(cfg), + flattened_box_placements=flattened_box_placements, + flattened_poly_placements=flattened_poly_placements, + estimated_flat_boxes_per_heavy_layer=flattened_box_placements * cfg.heavy_boxes_per_cell, + estimated_flat_polygons_per_active_polygon_layer=flattened_poly_placements * polys_per_layer // cfg.poly_cells if cfg.poly_cells else 0, + ) + + +def write_fixture( + path: str | Path, + *, + preset: str, + scale: float = 1.0, + write_manifest: bool = True, + ) -> FixtureManifest: + if preset not in PRESETS: + known = ', '.join(sorted(PRESETS)) + raise KeyError(f'unknown preset {preset!r}; expected one of: {known}') + + manifest = fixture_manifest(path, preset, scale) + cfg = _scaled_preset(PRESETS[preset], scale) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + + with output.open('wb') as stream: + header = klamath.library.FileHeader( + name=manifest.library_name.encode('ASCII'), + user_units_per_db_unit=USER_UNITS_PER_DB_UNIT, + meters_per_db_unit=METERS_PER_DB_UNIT, + ) + header.write(stream) + _write_box_cells(stream, cfg) + _write_poly_cells(stream, cfg) + _write_wrappers(stream, cfg) + _write_box_clusters(stream, cfg) + _write_poly_clusters(stream, cfg) + _write_top(stream, cfg) + klamath.records.ENDLIB.write(stream, None) + + if write_manifest: + manifest_path = output.with_suffix(output.suffix + '.json') + manifest_path.write_text(json.dumps(asdict(manifest), indent=2, sort_keys=True) + '\n') + + return manifest + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Generate synthetic GDS fixtures for GDS reader/writer performance work.') + parser.add_argument( + 'preset', + nargs='?', + default='many_cells', + choices=sorted(PRESETS), + help='Fixture family to generate.', + ) + parser.add_argument( + 'output', + nargs='?', + help='Output .gds path. Defaults to build/gds_perf/.gds', + ) + parser.add_argument( + '--scale', + type=float, + default=1.0, + help='Scale the preset counts down or up while keeping the same shape mix. Default: 1.0', + ) + parser.add_argument( + '--no-manifest', + action='store_true', + help='Do not write the sidecar JSON manifest.', + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + output = Path(args.output) if args.output is not None else Path('build/gds_perf') / f'{args.preset}.gds' + manifest = write_fixture(output, preset=args.preset, scale=args.scale, write_manifest=not args.no_manifest) + print(json.dumps(asdict(manifest), indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/test/test_gdsii_perf.py b/masque/test/test_gdsii_perf.py new file mode 100644 index 0000000..a595fe8 --- /dev/null +++ b/masque/test/test_gdsii_perf.py @@ -0,0 +1,24 @@ +from dataclasses import asdict +import json +from pathlib import Path + +from ..file import gdsii +from ..file.gdsii_perf import fixture_manifest, write_fixture + + +def test_gdsii_perf_fixture_smoke(tmp_path: Path) -> None: + output = tmp_path / 'many_cells.gds' + manifest = write_fixture(output, preset='many_cells', scale=0.002) + expected = fixture_manifest(output, preset='many_cells', scale=0.002) + + assert output.exists() + assert manifest == expected + + sidecar = json.loads(output.with_suffix('.gds.json').read_text()) + assert sidecar == asdict(manifest) + + read_lib, info = gdsii.readfile(output) + assert info['name'] == manifest.library_name + assert len(read_lib) == manifest.cells + assert 'TOP' in read_lib + assert len(read_lib['TOP'].refs) > 0 From c68e76a40e72e4a9e71da5bfd450c168e29b14e1 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:46:53 -0700 Subject: [PATCH 24/36] [Label / Ref / Grid] add raw constructors --- masque/label.py | 16 ++++++++++++++++ masque/ref.py | 20 ++++++++++++++++++++ masque/repetition.py | 16 ++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/masque/label.py b/masque/label.py index 6a2a61f..98a65df 100644 --- a/masque/label.py +++ b/masque/label.py @@ -53,6 +53,22 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + string: str, + *, + offset: NDArray[numpy.float64], + repetition: Repetition | None = None, + annotations: annotations_t | None = None, + ) -> Self: + new = cls.__new__(cls) + new._string = string + new._offset = offset + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> Self: return type(self)( string=self.string, diff --git a/masque/ref.py b/masque/ref.py index 268d8d4..1d71432 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -86,6 +86,26 @@ class Ref( self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + *, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + scale: float, + repetition: Repetition | None, + annotations: annotations_t | None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._rotation = rotation % (2 * pi) + new._scale = scale + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations if annotations is not None else {} + return new + def __copy__(self) -> 'Ref': new = Ref( offset=self.offset.copy(), diff --git a/masque/repetition.py b/masque/repetition.py index 1e12fcd..9e8af26 100644 --- a/masque/repetition.py +++ b/masque/repetition.py @@ -113,6 +113,22 @@ class Grid(Repetition): self.a_count = a_count self.b_count = b_count + @classmethod + def _from_raw( + cls: type[GG], + *, + a_vector: NDArray[numpy.float64], + a_count: int, + b_vector: NDArray[numpy.float64], + b_count: int, + ) -> GG: + new = cls.__new__(cls) + new._a_vector = a_vector + new._b_vector = b_vector + new._a_count = int(a_count) + new._b_count = int(b_count) + return new + @classmethod def aligned( cls: type[GG], From 51cab706da5c27885cb5071e93c83e3913327569 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:47:41 -0700 Subject: [PATCH 25/36] [gdsii_arrow] further improvements to speed --- examples/profile_gdsii_readers.py | 71 ++++++++++++--- masque/file/gdsii_arrow.py | 144 ++++++++++++++++++++---------- masque/test/test_gdsii_arrow.py | 81 +++++++++++++++++ 3 files changed, 235 insertions(+), 61 deletions(-) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index c77c589..129a391 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -5,17 +5,18 @@ import importlib import json import time from pathlib import Path +from typing import Any from masque import LibraryError -READERS = { - 'gdsii': ('masque.file.gdsii', 'readfile'), - 'gdsii_arrow': ('masque.file.gdsii_arrow', 'readfile'), +READERS: dict[str, tuple[str, tuple[str, ...]]] = { + 'gdsii': ('masque.file.gdsii', ('readfile',)), + 'gdsii_arrow': ('masque.file.gdsii_arrow', ('readfile', 'arrow_import', 'arrow_convert')), } -def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: +def _summarize_library(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: assert hasattr(lib, '__len__') assert hasattr(lib, 'tops') tops = lib.tops() # type: ignore[no-any-return, attr-defined] @@ -34,12 +35,50 @@ def _summarize(path: Path, elapsed_s: float, info: dict[str, object], lib: objec } +def _summarize_arrow_import(path: Path, elapsed_s: float, arrow_arr: Any) -> dict[str, object]: + libarr = arrow_arr[0] + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': len(arrow_arr), + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + + +def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: + start = time.perf_counter() + + if stage == 'readfile': + lib, info = module.readfile(path) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + if stage == 'arrow_import': + arrow_arr = module._read_to_arrow(path) + elapsed_s = time.perf_counter() - start + return _summarize_arrow_import(path, elapsed_s, arrow_arr) + + if stage == 'arrow_convert': + arrow_arr = module._read_to_arrow(path) + libarr = arrow_arr[0] + start = time.perf_counter() + lib, info = module.read_arrow(libarr) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + raise ValueError(f'Unsupported stage {stage!r}') + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--stage', default='readfile') parser.add_argument('--path', type=Path, required=True) parser.add_argument('--warmup', type=int, default=1) parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--output-json', type=Path) return parser @@ -47,26 +86,32 @@ def main(argv: list[str] | None = None) -> int: parser = build_arg_parser() args = parser.parse_args(argv) - module_name, attr_name = READERS[args.reader] - readfile = getattr(importlib.import_module(module_name), attr_name) + module_name, stages = READERS[args.reader] + if args.stage not in stages: + parser.error(f'reader {args.reader!r} only supports stages: {", ".join(stages)}') + + module = importlib.import_module(module_name) path = args.path.expanduser().resolve() for _ in range(args.warmup): - readfile(path) + _profile_stage(module, args.stage, path) runs = [] for _ in range(args.repeat): - start = time.perf_counter() - lib, info = readfile(path) - elapsed_s = time.perf_counter() - start - runs.append(_summarize(path, elapsed_s, info, lib)) + runs.append(_profile_stage(module, args.stage, path)) - print(json.dumps({ + payload = { 'reader': args.reader, + 'stage': args.stage, 'warmup': args.warmup, 'repeat': args.repeat, 'runs': runs, - }, indent=2, sort_keys=True)) + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(rendered + '\n') + print(rendered) return 0 diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 546d575..f6a7739 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -57,6 +57,8 @@ ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchem clib: Any | None = None +ZERO_OFFSET = numpy.zeros(2) + path_cap_map = { 0: Path.Cap.Flush, @@ -263,6 +265,29 @@ def read_arrow( ) return elem + def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('boundary_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = batches.values.field('vertices').offsets.to_numpy() // 2, + poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(), + poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), + ) + + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: + boundaries = libarr['cells'].values.field('boundary_props') + return dict( + offsets = boundaries.offsets.to_numpy(), + layer_inds = boundaries.values.field('layer').to_numpy(), + vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2, + prop_off = boundaries.values.field('properties').offsets.to_numpy(), + prop_key = boundaries.values.field('properties').values.field('key').to_numpy(), + prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), + ) + rf = libarr['cells'].values.field('refs') refs = dict( offsets = rf.offsets.to_numpy(), @@ -292,10 +317,9 @@ def read_arrow( ) elements = dict( - boundaries = get_geom(libarr, 'boundaries'), + boundary_batches = get_boundary_batches(libarr), + boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), - boxes = get_geom(libarr, 'boxes'), - nodes = get_geom(libarr, 'nodes'), texts = texts, refs = refs, ) @@ -320,7 +344,8 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() - _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) + _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) _grefs_to_mrefs(pat, global_args, elements['refs'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) @@ -366,6 +391,7 @@ def _grefs_to_mrefs( elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] rep_valid = elem['rep_valid'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): @@ -380,10 +406,16 @@ def _grefs_to_mrefs( a_vector = elem_rep_xy0[ee] b_vector = elem_rep_xy1[ee] a_count, b_count = elem_rep_counts[ee] - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + if raw_mode: + rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + else: + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + if raw_mode: + ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + else: + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) pat.refs[target].append(ref) @@ -406,6 +438,7 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] + raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -413,7 +446,10 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - mlabel = Label(string=string, offset=offset, annotations=annotations) + if raw_mode: + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) + else: + mlabel = Label(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -439,7 +475,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[elem_layer_inds[ee]] @@ -453,65 +488,78 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) -def _boundaries_to_polygons( +def _boundary_batches_to_polygons( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: elem_off = elem['offsets'] # which elements belong to each cell - xy_val = elem['xy_arr'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + poly_off = elem['poly_off'] + poly_offsets = elem['poly_offsets'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) # +1 to capture ending location for last elem + elem_vert_off = vert_off[elem_slc] + elem_poly_off = poly_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] + vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + + if vertex_offsets.size == 1: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(poly) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + pat.shapes[layer].append(polys) + + +def _boundary_props_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] layer_inds = elem['layer_inds'] layer_tups = global_args['layer_tups'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element - xy_counts = xy_offs[1:] - xy_offs[:-1] - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - prop_counts = prop_offs[1:] - prop_offs[:-1] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + elem_vert_off = vert_off[elem_slc] + prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - order = numpy.argsort(elem_layer_inds, stable=True) - unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True) - - zeros = numpy.zeros((elem_count, 2)) raw_mode = global_args['raw_mode'] - for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True): - ee_inds = order[ff:ff + nn] - layer = layer_tups[layer_ind] - propless_mask = prop_counts[ee_inds] == 0 - - poly_count_on_layer = propless_mask.sum() - if poly_count_on_layer == 1: - propless_mask[:] = 0 # Never make a 1-element collection - elif poly_count_on_layer > 1: - propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point - vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64) - vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts])) - - for ii, ee in enumerate(ee_inds[propless_mask]): - vo = vertex_offsets[ii] - vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] - - polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee]) - pat.shapes[layer].append(polys) - - # Handle single polygons - for ee in ee_inds[~propless_mask]: - layer = layer_tups[elem_layer_inds[ee]] - vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point - - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) - pat.shapes[layer].append(poly) + for ee in range(elem_count): + layer = layer_tups[elem_layer_inds[ee]] + vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + pat.shapes[layer].append(poly) #def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 26713e9..b005cd7 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -5,6 +5,7 @@ import pytest pytest.importorskip('pyarrow') +from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -119,7 +120,10 @@ def _make_arrow_test_library() -> Library: leaf = Pattern() leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((2, 0), vertices=[[40, 0], [50, 0], [50, 10], [40, 10]]) leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.polygon((1, 0), vertices=[[80, 0], [90, 0], [90, 10], [80, 10]]) + leaf.polygon((2, 0), vertices=[[60, 0], [70, 0], [70, 10], [60, 10]], annotations={'18': ['leaf-poly-2']}) leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) lib['leaf'] = leaf @@ -178,3 +182,80 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert len(lib) == manifest.cells assert 'TOP' in lib assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 + + +def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + + leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + + boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() + boundary_props = cells.field('boundary_props')[leaf_index].as_py() + + assert len(boundary_batches) == 2 + assert len(boundary_props) == 2 + + batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} + assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] + assert len(batch_by_layer[(1, 0)]['vertices']) == 16 + assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] + assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + + props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} + assert sorted(props_by_layer) == [(1, 0), (2, 0)] + assert props_by_layer[(1, 0)]['properties'][0]['value'] == 'leaf-poly' + assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' + + +def test_raw_ref_grid_label_constructors_match_public() -> None: + raw_grid = Grid._from_raw( + a_vector=numpy.array([20, 0]), + a_count=3, + b_vector=numpy.array([0, 30]), + b_count=2, + ) + public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) + assert raw_grid == public_grid + + raw_ref = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=raw_grid, + annotations={'12': ['child-ref']}, + ) + public_ref = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=public_grid, + annotations={'12': ['child-ref']}, + ) + assert raw_ref == public_ref + assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + + raw_label = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations={'10': ['leaf-label']}, + ) + public_label = Label( + 'LEAF', + offset=(3, 4), + annotations={'10': ['leaf-label']}, + ) + assert raw_label == public_label + assert numpy.array_equal(raw_label.get_bounds_single(), public_label.get_bounds_single()) From 07e569520c121afe689d7dc98b43cbcb438f1baa Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 15:56:42 -0700 Subject: [PATCH 26/36] enable annotations=None by default --- masque/label.py | 2 +- masque/ref.py | 2 +- masque/test/test_gdsii_arrow.py | 34 ++++++++++++++++++++++++++++++++- masque/utils/comparisons.py | 10 ++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/masque/label.py b/masque/label.py index 98a65df..d220fee 100644 --- a/masque/label.py +++ b/masque/label.py @@ -66,7 +66,7 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl new._string = string new._offset = offset new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> Self: diff --git a/masque/ref.py b/masque/ref.py index 1d71432..0cc911f 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -103,7 +103,7 @@ class Ref( new._scale = scale new._mirrored = mirrored new._repetition = repetition - new._annotations = annotations if annotations is not None else {} + new._annotations = annotations return new def __copy__(self) -> 'Ref': diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index b005cd7..cec48c8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -19,7 +19,7 @@ if not gdsii_arrow.is_available(): def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: - if annotations is None: + if not annotations: return None return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) @@ -228,6 +228,25 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_ref_empty = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + public_ref_empty = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + assert raw_ref_empty.annotations is None + assert raw_ref_empty == public_ref_empty + raw_ref = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, @@ -247,6 +266,19 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: assert raw_ref == public_ref assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + raw_label_empty = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations=None, + ) + public_label_empty = Label( + 'LEAF', + offset=(3, 4), + annotations=None, + ) + assert raw_label_empty.annotations is None + assert raw_label_empty == public_label_empty + raw_label = Label._from_raw( 'LEAF', offset=numpy.array([3, 4]), diff --git a/masque/utils/comparisons.py b/masque/utils/comparisons.py index ffb7206..bb2dfee 100644 --- a/masque/utils/comparisons.py +++ b/masque/utils/comparisons.py @@ -9,7 +9,15 @@ def annotation2key(aaa: int | float | str) -> tuple[bool, Any]: return (isinstance(aaa, str), aaa) +def _normalized_annotations(annotations: annotations_t) -> annotations_t: + if not annotations: + return None + return annotations + + def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is not None elif bb is None: # noqa: RET505 @@ -36,6 +44,8 @@ def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: def annotations_eq(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is None elif bb is None: # noqa: RET505 From bdb9b038924aaea3a4d4e5f11fff811dc875e27f Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:17 -0700 Subject: [PATCH 27/36] [RectCollection] add a RectCollection shape --- masque/__init__.py | 1 + masque/file/gdsii.py | 16 +- masque/shapes/__init__.py | 1 + masque/shapes/rect_collection.py | 256 ++++++++++++++++++++++++++++ masque/test/test_rect_collection.py | 70 ++++++++ 5 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 masque/shapes/rect_collection.py create mode 100644 masque/test/test_rect_collection.py diff --git a/masque/__init__.py b/masque/__init__.py index 8e13bb9..b6cc5e9 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -42,6 +42,7 @@ from .error import ( from .shapes import ( Shape as Shape, Polygon as Polygon, + RectCollection as RectCollection, Path as Path, Circle as Circle, Arc as Arc, diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 116fa07..323dabd 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -37,7 +37,7 @@ from klamath import records from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path +from ..shapes import Polygon, Path, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -466,6 +466,20 @@ def _shapes_to_elements( properties=properties, ) elements.append(path) + elif isinstance(shape, RectCollection): + for rect in shape.rects: + xy_closed = numpy.empty((5, 2), dtype=numpy.int32) + xy_closed[0] = rint_cast((rect[0], rect[1])) + xy_closed[1] = rint_cast((rect[0], rect[3])) + xy_closed[2] = rint_cast((rect[2], rect[3])) + xy_closed[3] = rint_cast((rect[2], rect[1])) + xy_closed[4] = xy_closed[0] + boundary = klamath.elements.Boundary( + layer=(layer, data_type), + xy=xy_closed, + properties=properties, + ) + elements.append(boundary) elif isinstance(shape, Polygon): polygon = shape xy_closed = numpy.empty((polygon.vertices.shape[0] + 1, 2), dtype=numpy.int32) diff --git a/masque/shapes/__init__.py b/masque/shapes/__init__.py index fd66c59..ac3a14b 100644 --- a/masque/shapes/__init__.py +++ b/masque/shapes/__init__.py @@ -11,6 +11,7 @@ from .shape import ( from .polygon import Polygon as Polygon from .poly_collection import PolyCollection as PolyCollection +from .rect_collection import RectCollection as RectCollection from .circle import Circle as Circle from .ellipse import Ellipse as Ellipse from .arc import Arc as Arc diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py new file mode 100644 index 0000000..96becfd --- /dev/null +++ b/masque/shapes/rect_collection.py @@ -0,0 +1,256 @@ +from typing import Any, cast, Self +from collections.abc import Iterator +import copy +import functools + +import numpy +from numpy import pi +from numpy.typing import NDArray, ArrayLike + +from . import Shape, normalized_shape_tuple +from .polygon import Polygon +from ..error import PatternError +from ..repetition import Repetition +from ..utils import annotations_lt, annotations_eq, rep2key, annotations_t + + +def _normalize_rects(rects: ArrayLike) -> NDArray[numpy.float64]: + arr = numpy.asarray(rects, dtype=float) + if arr.ndim != 2 or arr.shape[1] != 4: + raise PatternError('Rectangles must be an Nx4 array of [xmin, ymin, xmax, ymax]') + if numpy.any(arr[:, 0] > arr[:, 2]) or numpy.any(arr[:, 1] > arr[:, 3]): + raise PatternError('Rectangles must satisfy xmin <= xmax and ymin <= ymax') + if arr.shape[0] <= 1: + return arr + order = numpy.lexsort((arr[:, 3], arr[:, 2], arr[:, 1], arr[:, 0])) + return arr[order] + + +def _renormalize_rects_in_place(rects: NDArray[numpy.float64]) -> None: + x0 = numpy.minimum(rects[:, 0], rects[:, 2]) + x1 = numpy.maximum(rects[:, 0], rects[:, 2]) + y0 = numpy.minimum(rects[:, 1], rects[:, 3]) + y1 = numpy.maximum(rects[:, 1], rects[:, 3]) + rects[:, 0] = x0 + rects[:, 1] = y0 + rects[:, 2] = x1 + rects[:, 3] = y1 + + +@functools.total_ordering +class RectCollection(Shape): + """ + A collection of axis-aligned rectangles, stored as an Nx4 array of + `[xmin, ymin, xmax, ymax]` rows. + """ + __slots__ = ( + '_rects', + '_repetition', '_annotations', + ) + + _rects: NDArray[numpy.float64] + + @property + def rects(self) -> NDArray[numpy.float64]: + return self._rects + + @rects.setter + def rects(self, val: ArrayLike) -> None: + self._rects = _normalize_rects(val) + + @property + def offset(self) -> NDArray[numpy.float64]: + return numpy.zeros(2) + + @offset.setter + def offset(self, val: ArrayLike) -> None: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + + def set_offset(self, val: ArrayLike) -> Self: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + return self + + def translate(self, offset: ArrayLike) -> Self: + delta = numpy.asarray(offset, dtype=float).reshape(2) + self._rects[:, [0, 2]] += delta[0] + self._rects[:, [1, 3]] += delta[1] + return self + + def __init__( + self, + rects: ArrayLike, + *, + offset: ArrayLike = (0.0, 0.0), + rotation: float = 0.0, + repetition: Repetition | None = None, + annotations: annotations_t = None, + raw: bool = False, + ) -> None: + if raw: + assert isinstance(rects, numpy.ndarray) + self._rects = rects + self._repetition = repetition + self._annotations = annotations + else: + self.rects = rects + self.repetition = repetition + self.annotations = annotations + if rotation: + self.rotate(rotation) + if numpy.any(offset): + self.translate(offset) + + @classmethod + def _from_raw( + cls, + *, + rects: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._rects = rects + new._repetition = repetition + new._annotations = annotations + return new + + @property + def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: + for rect in self._rects: + xmin, ymin, xmax, ymax = rect + yield numpy.array([ + [xmin, ymin], + [xmin, ymax], + [xmax, ymax], + [xmax, ymin], + ], dtype=float) + + def __deepcopy__(self, memo: dict | None = None) -> Self: + memo = {} if memo is None else memo + new = copy.copy(self) + new._rects = self._rects.copy() + new._repetition = copy.deepcopy(self._repetition, memo) + new._annotations = copy.deepcopy(self._annotations) + return new + + def _sorted_rects(self) -> NDArray[numpy.float64]: + if self._rects.shape[0] <= 1: + return self._rects + order = numpy.lexsort((self._rects[:, 3], self._rects[:, 2], self._rects[:, 1], self._rects[:, 0])) + return self._rects[order] + + def __eq__(self, other: Any) -> bool: + return ( + type(self) is type(other) + and numpy.array_equal(self._sorted_rects(), other._sorted_rects()) + and self.repetition == other.repetition + and annotations_eq(self.annotations, other.annotations) + ) + + def __lt__(self, other: Shape) -> bool: + if type(self) is not type(other): + if repr(type(self)) != repr(type(other)): + return repr(type(self)) < repr(type(other)) + return id(type(self)) < id(type(other)) + + other = cast('RectCollection', other) + self_rects = self._sorted_rects() + other_rects = other._sorted_rects() + if not numpy.array_equal(self_rects, other_rects): + min_len = min(self_rects.shape[0], other_rects.shape[0]) + eq_mask = self_rects[:min_len] != other_rects[:min_len] + eq_lt = self_rects[:min_len] < other_rects[:min_len] + eq_lt_masked = eq_lt[eq_mask] + if eq_lt_masked.size > 0: + return bool(eq_lt_masked.flat[0]) + return self_rects.shape[0] < other_rects.shape[0] + if self.repetition != other.repetition: + return rep2key(self.repetition) < rep2key(other.repetition) + return annotations_lt(self.annotations, other.annotations) + + def to_polygons( + self, + num_vertices: int | None = None, # unused # noqa: ARG002 + max_arclen: float | None = None, # unused # noqa: ARG002 + ) -> list[Polygon]: + return [ + Polygon( + vertices=vertices, + repetition=copy.deepcopy(self.repetition), + annotations=copy.deepcopy(self.annotations), + ) + for vertices in self.polygon_vertices + ] + + def get_bounds_single(self) -> NDArray[numpy.float64] | None: + if self._rects.size == 0: + return None + mins = self._rects[:, :2].min(axis=0) + maxs = self._rects[:, 2:].max(axis=0) + return numpy.vstack((mins, maxs)) + + def rotate(self, theta: float) -> Self: + quarter_turns = int(numpy.rint(theta / (pi / 2))) + if not numpy.isclose(theta, quarter_turns * (pi / 2)): + raise PatternError('RectCollection only supports Manhattan rotations') + turns = quarter_turns % 4 + if turns == 0 or self._rects.size == 0: + return self + + corners = numpy.stack(( + self._rects[:, [0, 1]], + self._rects[:, [0, 3]], + self._rects[:, [2, 3]], + self._rects[:, [2, 1]], + ), axis=1) + flat = corners.reshape(-1, 2) + if turns == 1: + rotated = numpy.column_stack((-flat[:, 1], flat[:, 0])) + elif turns == 2: + rotated = -flat + else: + rotated = numpy.column_stack((flat[:, 1], -flat[:, 0])) + corners = rotated.reshape(corners.shape) + self._rects[:, 0] = corners[:, :, 0].min(axis=1) + self._rects[:, 1] = corners[:, :, 1].min(axis=1) + self._rects[:, 2] = corners[:, :, 0].max(axis=1) + self._rects[:, 3] = corners[:, :, 1].max(axis=1) + return self + + def mirror(self, axis: int = 0) -> Self: + if axis not in (0, 1): + raise PatternError('Axis must be 0 or 1') + if axis == 0: + self._rects[:, [1, 3]] *= -1 + else: + self._rects[:, [0, 2]] *= -1 + _renormalize_rects_in_place(self._rects) + return self + + def scale_by(self, c: float) -> Self: + self._rects *= c + _renormalize_rects_in_place(self._rects) + return self + + def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + rects = self._sorted_rects() + centers = 0.5 * (rects[:, :2] + rects[:, 2:]) + offset = centers.mean(axis=0) + zeroed = rects.copy() + zeroed[:, [0, 2]] -= offset[0] + zeroed[:, [1, 3]] -= offset[1] + normed = zeroed / norm_value + return ( + (type(self), normed.data.tobytes()), + (offset, 1.0, 0.0, False), + lambda: RectCollection(rects=normed * norm_value), + ) + + def __repr__(self) -> str: + if self._rects.size == 0: + return '' + centers = 0.5 * (self._rects[:, :2] + self._rects[:, 2:]) + centroid = centers.mean(axis=0) + return f'' diff --git a/masque/test/test_rect_collection.py b/masque/test/test_rect_collection.py new file mode 100644 index 0000000..449f4fa --- /dev/null +++ b/masque/test/test_rect_collection.py @@ -0,0 +1,70 @@ +import copy + +import numpy +import pytest +from numpy.testing import assert_allclose, assert_equal + +from ..error import PatternError +from ..shapes import Polygon, RectCollection + + +def test_rect_collection_init_and_to_polygons() -> None: + rects = RectCollection([[10, 10, 12, 12], [0, 0, 5, 5]]) + assert_equal(rects.rects, [[0, 0, 5, 5], [10, 10, 12, 12]]) + + polys = rects.to_polygons() + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_equal(polys[0].vertices, [[0, 0], [0, 5], [5, 5], [5, 0]]) + + +def test_rect_collection_rejects_invalid_rects() -> None: + with pytest.raises(PatternError): + RectCollection([[0, 0, 1]]) + with pytest.raises(PatternError): + RectCollection([[5, 0, 1, 2]]) + with pytest.raises(PatternError): + RectCollection([[0, 5, 1, 2]]) + + +def test_rect_collection_raw_constructor_matches_public() -> None: + raw = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'1': ['rects']}, + ) + public = RectCollection( + [[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'1': ['rects']}, + ) + assert raw == public + assert_equal(raw.get_bounds_single(), [[0, 0], [12, 12]]) + + +def test_rect_collection_manhattan_transforms() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + + mirrored = copy.deepcopy(rects).mirror(1) + assert_equal(mirrored.rects, [[-2, 0, 0, 4], [-12, 20, -10, 22]]) + + scaled = copy.deepcopy(rects).scale_by(-2) + assert_equal(scaled.rects, [[-4, -8, 0, 0], [-24, -44, -20, -40]]) + + rotated = copy.deepcopy(rects).rotate(numpy.pi / 2) + assert_equal(rotated.rects, [[-4, 0, 0, 2], [-22, 10, -20, 12]]) + + +def test_rect_collection_non_manhattan_rotation_raises() -> None: + rects = RectCollection([[0, 0, 2, 4]]) + with pytest.raises(PatternError, match='Manhattan rotations'): + rects.rotate(numpy.pi / 4) + + +def test_rect_collection_normalized_form_rebuild_is_independent() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + _intrinsic, extrinsic, rebuild = rects.normalized_form(2) + + clone = rebuild() + clone.rects[:] = [[1, 1, 2, 2], [3, 3, 4, 4]] + + assert_allclose(extrinsic[0], [6, 11.5]) + assert_equal(rects.rects, [[0, 0, 2, 4], [10, 20, 12, 22]]) From d30695f7f62a9b02de78965f8a22980291a8d419 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:39 -0700 Subject: [PATCH 28/36] [Polygon / PolyCollection] add raw constructors --- masque/shapes/poly_collection.py | 16 ++++++++++++++++ masque/shapes/polygon.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 6c23da7..0b5543f 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -119,6 +119,22 @@ class PolyCollection(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertex_lists: NDArray[numpy.float64], + vertex_offsets: NDArray[numpy.integer[Any]], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertex_lists = vertex_lists + new._vertex_offsets = vertex_offsets + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index a743de2..42c2f78 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -131,6 +131,20 @@ class Polygon(Shape): if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Polygon': memo = {} if memo is None else memo new = copy.copy(self) From b7d7b1ac19fc7b5caa7bc60ac95e95b48eb6061b Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:52 -0700 Subject: [PATCH 29/36] [gdsii_arrow] more performance work --- masque/file/gdsii_arrow.py | 356 ++++++++++++++++++---- masque/shapes/poly_collection.py | 13 +- masque/test/test_file_format_roundtrip.py | 29 +- masque/test/test_gdsii_arrow.py | 264 ++++++++++++++-- 4 files changed, 562 insertions(+), 100 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index f6a7739..c5e7681 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -45,7 +45,7 @@ from pyarrow.cffi import ffi from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, PolyCollection +from ..shapes import Polygon, Path, PolyCollection, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -276,6 +276,15 @@ def read_arrow( poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), ) + def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('rect_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_off = batches.values.field('rects').offsets.to_numpy() // 4, + ) + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: boundaries = libarr['cells'].values.field('boundary_props') return dict( @@ -288,22 +297,46 @@ def read_arrow( prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), ) - rf = libarr['cells'].values.field('refs') - refs = dict( - offsets = rf.offsets.to_numpy(), - targets = rf.values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), - invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), - scale = rf.values.field('mag').fill_null(1).to_numpy(), - rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), - rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), - rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), - prop_off = rf.values.field('properties').offsets.to_numpy(), - prop_key = rf.values.field('properties').values.field('key').to_numpy(), - prop_val = rf.values.field('properties').values.field('value').to_pylist(), - ) + def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + prop_off = values.field('properties').offsets.to_numpy(), + prop_key = values.field('properties').values.field('key').to_numpy(), + prop_val = values.field('properties').values.field('value').to_pylist(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem txt = libarr['cells'].values.field('texts') texts = dict( @@ -317,11 +350,15 @@ def read_arrow( ) elements = dict( + srefs = get_refs(libarr, 'srefs', has_repetition=False), + arefs = get_refs(libarr, 'arefs', has_repetition=True), + sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False), + aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True), + rect_batches = get_rect_batches(libarr), boundary_batches = get_boundary_batches(libarr), boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), texts = texts, - refs = refs, ) paths = libarr['cells'].values.field('paths') @@ -344,10 +381,14 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() + _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) - _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _srefs_to_mrefs(pat, global_args, elements['srefs'], cc) + _arefs_to_mrefs(pat, global_args, elements['arefs'], cc) + _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc) + _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) mlib[name] = pat @@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: return library_info -def _grefs_to_mrefs( +def _srefs_to_mrefs( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: cell_names = global_args['cell_names'] - elem_off = elem['offsets'] # which elements belong to each cell - xy = elem['xy'] - prop_key = elem['prop_key'] - prop_val = elem['prop_val'] - targets = elem['targets'] - + elem_off = elem['offsets'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - elem_targets = targets[elem_slc][:elem_count] - elem_xy = xy[elem_slc][:elem_count] - elem_invert_y = elem['invert_y'][elem_slc][:elem_count] - elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] - elem_scale = elem['scale'][elem_slc][:elem_count] - elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] - elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] - elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] - rep_valid = elem['rep_valid'][elem_slc][:elem_count] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] raw_mode = global_args['raw_mode'] + _append_plain_refs_sorted( + pat=pat, + cell_names=cell_names, + elem_targets=elem_targets, + elem_xy=elem_xy, + elem_invert_y=elem_invert_y, + elem_angle_rad=elem_angle_rad, + elem_scale=elem_scale, + raw_mode=raw_mode, + ) + + +def _append_plain_refs_sorted( + *, + pat: Pattern, + cell_names: list[str], + elem_targets: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.integer[Any]], + elem_invert_y: NDArray[numpy.bool_ | numpy.bool], + elem_angle_rad: NDArray[numpy.floating[Any]], + elem_scale: NDArray[numpy.floating[Any]], + raw_mode: bool, + ) -> None: + elem_count = len(elem_targets) + if elem_count == 0: + return + + make_ref = Ref._from_raw if raw_mode else Ref + + target_start = 0 + while target_start < elem_count: + target_id = elem_targets[target_start] + target_stop = target_start + 1 + while target_stop < elem_count and elem_targets[target_stop] == target_id: + target_stop += 1 + + append_refs = pat.refs[cell_names[target_id]].extend + append_refs( + make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=None, + ) + for ee in range(target_start, target_stop) + ) + + target_start = target_stop + + +def _arefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + elem_xy0 = elem['xy0'][start:stop] + elem_xy1 = elem['xy1'][start:stop] + elem_counts = elem['counts'][start:stop] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + if len(elem_targets) == 0: + return + + target = None + append_ref: Callable[[Ref], Any] | None = None + for ee in range(len(elem_targets)): + if target != elem_targets[ee]: + target = elem_targets[ee] + append_ref = pat.refs[cell_names[target]].append + assert append_ref is not None + a_count, b_count = elem_counts[ee] + append_ref(make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=None, + )) + + +def _sref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): - target = cell_names[elem_targets[ee]] - offset = elem_xy[ee] - mirr = elem_invert_y[ee] - rot = elem_angle_rad[ee] - mag = elem_scale[ee] - - rep: None | Grid = None - if rep_valid[ee]: - a_vector = elem_rep_xy0[ee] - b_vector = elem_rep_xy1[ee] - a_count, b_count = elem_rep_counts[ee] - if raw_mode: - rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - else: - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - else: - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - pat.refs[target].append(ref) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) + + +def _aref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] + elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] + elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + for ee in range(elem_count): + a_count, b_count = elem_counts[ee] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) def _texts_to_labels( @@ -520,16 +712,53 @@ def _boundary_batches_to_polygons( for bb in range(batch_count): layer = layer_tups[elem_layer_inds[bb]] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] - vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=None) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(poly) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(polys) +def _rect_batches_to_rectcollections( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + rect_arr = elem['rect_arr'] + rect_off = elem['rect_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) + elem_rect_off = rect_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] + if raw_mode: + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) + else: + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + pat.shapes[layer].append(rect_collection) + + def _boundary_props_to_polygons( pat: Pattern, global_args: dict[str, Any], @@ -558,7 +787,10 @@ def _boundary_props_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) pat.shapes[layer].append(poly) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 0b5543f..f90c29a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -34,7 +34,7 @@ class PolyCollection(Shape): _vertex_lists: NDArray[numpy.float64] """ 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """ - _vertex_offsets: NDArray[numpy.intp] + _vertex_offsets: NDArray[numpy.integer[Any]] """ 1D NDArray specifying the starting offset for each polygon """ @property @@ -45,7 +45,7 @@ class PolyCollection(Shape): return self._vertex_lists @property - def vertex_offsets(self) -> NDArray[numpy.intp]: + def vertex_offsets(self) -> NDArray[numpy.integer[Any]]: """ Starting offset (in `vertex_lists`) for each polygon """ @@ -63,7 +63,7 @@ class PolyCollection(Shape): chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]), strict=True, ): - yield slice(ii, ff) + yield slice(int(ii), int(ff)) @property def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: @@ -105,6 +105,7 @@ class PolyCollection(Shape): if raw: assert isinstance(vertex_lists, numpy.ndarray) assert isinstance(vertex_offsets, numpy.ndarray) + assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) self._vertex_lists = vertex_lists self._vertex_offsets = vertex_offsets self._repetition = repetition @@ -148,7 +149,7 @@ class PolyCollection(Shape): return ( type(self) is type(other) and numpy.array_equal(self._vertex_lists, other._vertex_lists) - and numpy.array_equal(self._vertex_offsets, other._vertex_offsets) + and numpy.array_equal(self.vertex_offsets, other.vertex_offsets) and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -231,11 +232,11 @@ class PolyCollection(Shape): # TODO: normalize mirroring? - return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()), + return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()), (offset, scale / norm_value, rotation, False), lambda: PolyCollection( vertex_lists=rotated_vertices * norm_value, - vertex_offsets=self._vertex_offsets.copy(), + vertex_offsets=self.vertex_offsets.copy(), ), ) diff --git a/masque/test/test_file_format_roundtrip.py b/masque/test/test_file_format_roundtrip.py index 6bd079d..36c4ce4 100644 --- a/masque/test/test_file_format_roundtrip.py +++ b/masque/test/test_file_format_roundtrip.py @@ -5,7 +5,7 @@ from numpy.testing import assert_allclose from ..pattern import Pattern from ..library import Library -from ..shapes import Path as MPath, Circle, Polygon +from ..shapes import Path as MPath, Circle, Polygon, RectCollection from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: @@ -109,3 +109,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: assert poly.repetition is not None assert isinstance(poly.repetition, Grid) assert poly.repetition.a_count == 5 + + +def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None: + from ..file import gdsii + + lib = Library() + pat = Pattern() + pat.shapes[(5, 0)].append( + RectCollection( + rects=[[0, 0, 10, 5], [20, -5, 30, 10]], + annotations={'1': ['rects']}, + ) + ) + lib['rects'] = pat + + gds_file = tmp_path / 'rect_collection.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + read_lib, _ = gdsii.readfile(gds_file) + polys = read_lib['rects'].shapes[(5, 0)] + + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]]) + assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]]) + assert polys[0].annotations == {'1': ['rects']} + assert polys[1].annotations == {'1': ['rects']} diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index cec48c8..2ef4d83 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -9,7 +9,7 @@ from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid -from ..shapes import Path as MPath +from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection from ..file import gdsii, gdsii_arrow from ..file.gdsii_perf import write_fixture @@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: return tuple(tuple(row.tolist()) for row in arr) +def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int) + rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])] + rev = arr[::-1] + rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0])) + return min(rows) + + def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: if isinstance(shape, MPath): cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) @@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...] keys.append(( 'polygon', layer, - _coord_key(poly.vertices), + _canonical_polygon_key(poly.vertices), _coord_key(poly.offset), _annotations_key(poly.annotations), )) return keys -def _ref_key(target: str, ref: object) -> tuple[object, ...]: - repetition = None - if ref.repetition is not None: - repetition = ( - _coord_key(ref.repetition.a_vector), - int(ref.repetition.a_count), - _coord_key(ref.repetition.b_vector), - int(ref.repetition.b_count), - ) - return ( - target, - _coord_key(ref.offset), - round(float(ref.rotation), 8), - round(float(ref.scale), 8), - bool(ref.mirrored), - repetition, - _annotations_key(ref.annotations), - ) +def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]: + keys = [] + for transform in ref.as_transforms(): + keys.append(( + target, + _coord_key(transform[:2]), + round(float(transform[2]), 8), + round(float(transform[4]), 8), + bool(int(round(float(transform[3])))), + _annotations_key(ref.annotations), + )) + return keys def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: @@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]: for shape in shapes: shape_keys.extend(_shape_key(shape, layer)) - ref_keys = [ - _ref_key(target, ref) - for target, refs in pattern.refs.items() - for ref in refs - ] + ref_keys: list[tuple[object, ...]] = [] + for target, refs in pattern.refs.items(): + for ref in refs: + ref_keys.extend(_ref_keys(target, ref)) label_keys = [ _label_key(layer, label) @@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library: ) lib['sibling'] = sibling + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(20, 0)) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2)) + fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2)) + fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']}) + fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2), + annotations={'20': ['fanout-aref']}) + lib['fanout'] = fanout + top = Pattern() top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.ref('fanout', offset=(250, -75)) top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) lib['top'] = top @@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 +def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, _ = gdsii.readfile(gds_file) + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + assert _library_summary(arrow_lib) == _library_summary(canonical_lib) + + decoded_ref = arrow_lib['top'].refs['leaf'][0] + assert decoded_ref.repetition is None + + +def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'plain_srefs.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + fanout = arrow_lib['fanout'] + + plain_leaf_refs = [ + ref + for ref in fanout.refs['leaf'] + if ref.annotations is None and ref.repetition is None + ] + assert len(plain_leaf_refs) == 2 + assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs) + + +def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref_schema.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + srefs = cells.field('srefs')[top_index].as_py() + arefs = cells.field('arefs')[top_index].as_py() + + assert len(srefs) == 1 + assert len(arefs) == 0 + assert cell_names[srefs[0]['target']] == 'leaf' + + def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_batches.gds' @@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + rect_batches = cells.field('rect_batches')[leaf_index].as_py() boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() boundary_props = cells.field('boundary_props')[leaf_index].as_py() - assert len(boundary_batches) == 2 + assert len(rect_batches) == 2 + assert len(boundary_batches) == 0 assert len(boundary_props) == 2 - batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} - assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] - assert len(batch_by_layer[(1, 0)]['vertices']) == 16 - assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] - assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches} + assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10] + assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10] props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} assert sorted(props_by_layer) == [(1, 0), (2, 0)] @@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' +def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None: + lib = Library() + top = Pattern() + top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]])) + top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]]) + top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]]) + lib['top'] = top + + gds_file = tmp_path / 'arrow_rect_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + rect_batches = cells.field('rect_batches')[top_index].as_py() + boundary_batches = cells.field('boundary_batches')[top_index].as_py() + + assert len(rect_batches) == 1 + assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0) + assert rect_batches[0]['rects'] == [ + 0, 0, 10, 10, + 20, 0, 30, 10, + 40, 0, 50, 10, + 60, 0, 70, 10, + ] + + assert len(boundary_batches) == 1 + assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0) + assert boundary_batches[0]['vertex_offsets'] == [0, 3] + + +def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_ref_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout') + + srefs = cells.field('srefs')[fanout_index].as_py() + arefs = cells.field('arefs')[fanout_index].as_py() + sref_props = cells.field('sref_props')[fanout_index].as_py() + aref_props = cells.field('aref_props')[fanout_index].as_py() + + sref_target_ids = [entry['target'] for entry in srefs] + sref_targets = [cell_names[target] for target in sref_target_ids] + assert sorted(sref_targets) == ['child', 'leaf', 'leaf'] + assert sref_target_ids == sorted(sref_target_ids) + sref_by_target = {} + for entry in srefs: + sref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in sref_by_target['child']] == [True] + assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1]) + assert len(sref_by_target['leaf']) == 2 + + aref_target_ids = [entry['target'] for entry in arefs] + aref_targets = [cell_names[target] for target in aref_target_ids] + assert sorted(aref_targets) == ['child', 'leaf', 'leaf'] + assert aref_target_ids == sorted(aref_target_ids) + aref_by_target = {} + for entry in arefs: + aref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in aref_by_target['child']] == [True] + assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2]) + assert len(aref_by_target['leaf']) == 2 + + assert len(sref_props) == 1 + assert cell_names[sref_props[0]['target']] == 'leaf' + assert sref_props[0]['properties'][0]['value'] == 'fanout-sref' + + assert len(aref_props) == 1 + assert cell_names[aref_props[0]['target']] == 'child' + assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), @@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_poly = Polygon._from_raw( + vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]), + annotations={'1': ['poly']}, + ) + public_poly = Polygon( + vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], + annotations={'1': ['poly']}, + ) + assert raw_poly == public_poly + + raw_poly_collection = PolyCollection._from_raw( + vertex_lists=numpy.array([ + [0.0, 0.0], [2.0, 0.0], [2.0, 2.0], + [10.0, 10.0], [12.0, 10.0], [12.0, 12.0], + ]), + vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32), + annotations={'2': ['pc']}, + ) + public_poly_collection = PolyCollection( + vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]], + vertex_offsets=[0, 3], + annotations={'2': ['pc']}, + ) + assert raw_poly_collection == public_poly_collection + assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)] + + raw_rect_collection = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'3': ['rects']}, + ) + public_rect_collection = RectCollection( + rects=[[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'3': ['rects']}, + ) + assert raw_rect_collection == public_rect_collection + raw_ref_empty = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2, From 1d7f82386e933a3347aa86a5ee32437a015bc691 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:55:30 -0700 Subject: [PATCH 30/36] [shapes] move to per-shape purpose-built _from_raw constructors --- masque/file/gdsii.py | 44 ++++++++----- masque/file/gdsii_arrow.py | 27 ++++++-- masque/shapes/arc.py | 53 ++++++++------- masque/shapes/circle.py | 32 +++++---- masque/shapes/ellipse.py | 38 ++++++----- masque/shapes/path.py | 47 ++++++++------ masque/shapes/poly_collection.py | 18 ++---- masque/shapes/polygon.py | 13 +--- masque/shapes/rect_collection.py | 13 +--- masque/shapes/text.py | 49 ++++++++------ masque/test/test_raw_constructors.py | 97 ++++++++++++++++++++++++++++ 11 files changed, 290 insertions(+), 141 deletions(-) create mode 100644 masque/test/test_raw_constructors.py diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 323dabd..1d8c3d1 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -323,26 +323,40 @@ def _gpath_to_mpath(gpath: klamath.library.Path, raw_mode: bool) -> tuple[layer_ else: raise PatternError(f'Unrecognized path type: {gpath.path_type}') - mpath = Path( - vertices=gpath.xy.astype(float), - width=gpath.width, - cap=cap, - offset=numpy.zeros(2), - annotations=_properties_to_annotations(gpath.properties), - raw=raw_mode, - ) + vertices = gpath.xy.astype(float) + annotations = _properties_to_annotations(gpath.properties) + cap_extensions = None if cap == Path.Cap.SquareCustom: - mpath.cap_extensions = gpath.extension + cap_extensions = numpy.asarray(gpath.extension, dtype=float) + + if raw_mode: + mpath = Path._from_raw( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + mpath = Path( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + offset=numpy.zeros(2), + annotations=annotations, + ) return gpath.layer, mpath def _boundary_to_polygon(boundary: klamath.library.Boundary, raw_mode: bool) -> tuple[layer_t, Polygon]: - return boundary.layer, Polygon( - vertices=boundary.xy[:-1].astype(float), - offset=numpy.zeros(2), - annotations=_properties_to_annotations(boundary.properties), - raw=raw_mode, - ) + vertices = boundary.xy[:-1].astype(float) + annotations = _properties_to_annotations(boundary.properties) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=numpy.zeros(2), annotations=annotations) + return boundary.layer, poly def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.Reference]: diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index c5e7681..58493c1 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -680,8 +680,23 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode, - width=width, cap=cap,cap_extensions=cap_extensions) + if raw_mode: + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + path = Path( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + offset=ZERO_OFFSET, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -718,13 +733,13 @@ def _boundary_batches_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=None) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(poly) else: if raw_mode: polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(polys) @@ -755,7 +770,7 @@ def _rect_batches_to_rectcollections( if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) pat.shapes[layer].append(rect_collection) @@ -790,7 +805,7 @@ def _boundary_props_to_polygons( if raw_mode: poly = Polygon._from_raw(vertices=vertices, annotations=annotations) else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index fe258c4..aaceeb5 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -197,29 +197,38 @@ class Arc(PositionableImpl, Shape): repetition: Repetition | None = None, annotations: annotations_t = None, angle_ref: ArcAngleRef | str = ArcAngleRef.Center, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(angles, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._angles = angles - self._width = width - self._offset = offset - self._rotation = rotation - self._angle_ref = ArcAngleRef(angle_ref) - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.angles = angles - self.width = width - self.offset = offset - self.rotation = rotation - self.angle_ref = angle_ref - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.angles = angles + self.width = width + self.offset = offset + self.rotation = rotation + self.angle_ref = angle_ref + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + angles: NDArray[numpy.float64], + width: float, + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Arc': + new = cls.__new__(cls) + new._radii = radii + new._angles = angles + new._width = width + new._offset = offset + new._rotation = rotation % (2 * pi) + new._angle_ref = ArcAngleRef(angle_ref) + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Arc': memo = {} if memo is None else memo diff --git a/masque/shapes/circle.py b/masque/shapes/circle.py index 19f3f2b..d7591db 100644 --- a/masque/shapes/circle.py +++ b/masque/shapes/circle.py @@ -50,19 +50,27 @@ class Circle(PositionableImpl, Shape): offset: ArrayLike = (0.0, 0.0), repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._radius = radius - self._offset = offset - self._repetition = repetition - self._annotations = annotations - else: - self.radius = radius - self.offset = offset - self.repetition = repetition - self.annotations = annotations + self.radius = radius + self.offset = offset + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radius: float, + offset: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Circle': + new = cls.__new__(cls) + new._radius = radius + new._offset = offset + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Circle': memo = {} if memo is None else memo diff --git a/masque/shapes/ellipse.py b/masque/shapes/ellipse.py index 38799da..52a3297 100644 --- a/masque/shapes/ellipse.py +++ b/masque/shapes/ellipse.py @@ -95,22 +95,30 @@ class Ellipse(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._radii = radii + new._offset = offset + new._rotation = rotation % pi + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo diff --git a/masque/shapes/path.py b/masque/shapes/path.py index 7038309..a1e04af 100644 --- a/masque/shapes/path.py +++ b/masque/shapes/path.py @@ -201,34 +201,43 @@ class Path(Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: self._cap_extensions = None # Since .cap setter might access it - if raw: - assert isinstance(vertices, numpy.ndarray) - assert isinstance(cap_extensions, numpy.ndarray) or cap_extensions is None - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - self._width = width - self._cap = cap - self._cap_extensions = cap_extensions + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations + self._cap = cap + if cap == PathCap.SquareCustom and cap_extensions is None: + self._cap_extensions = numpy.zeros(2) else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations - self._cap = cap - if cap == PathCap.SquareCustom and cap_extensions is None: - self._cap_extensions = numpy.zeros(2) - else: - self.cap_extensions = cap_extensions - self.width = width + self.cap_extensions = cap_extensions + self.width = width if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + width: float, + cap: PathCap, + cap_extensions: NDArray[numpy.float64] | None = None, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._width = width + new._cap = cap + new._cap_extensions = cap_extensions + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Path': memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index f90c29a..f1c840a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -100,21 +100,11 @@ class PolyCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertex_lists, numpy.ndarray) - assert isinstance(vertex_offsets, numpy.ndarray) - assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) - self._vertex_lists = vertex_lists - self._vertex_offsets = vertex_offsets - self._repetition = repetition - self._annotations = annotations - else: - self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) - self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) - self.repetition = repetition - self.annotations = annotations + self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) + self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index 42c2f78..06e5c2b 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -115,17 +115,10 @@ class Polygon(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertices, numpy.ndarray) - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py index 96becfd..eaf028f 100644 --- a/masque/shapes/rect_collection.py +++ b/masque/shapes/rect_collection.py @@ -86,17 +86,10 @@ class RectCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(rects, numpy.ndarray) - self._rects = rects - self._repetition = repetition - self._annotations = annotations - else: - self.rects = rects - self.repetition = repetition - self.annotations = annotations + self.rects = rects + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): diff --git a/masque/shapes/text.py b/masque/shapes/text.py index 5047dc4..c078879 100644 --- a/masque/shapes/text.py +++ b/masque/shapes/text.py @@ -73,27 +73,40 @@ class Text(PositionableImpl, RotatableImpl, Shape): mirrored: bool = False, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._offset = offset - self._string = string - self._height = height - self._rotation = rotation - self._mirrored = mirrored - self._repetition = repetition - self._annotations = annotations - else: - self.offset = offset - self.string = string - self.height = height - self.rotation = rotation - self.mirrored = mirrored - self.repetition = repetition - self.annotations = annotations + self.offset = offset + self.string = string + self.height = height + self.rotation = rotation + self.mirrored = mirrored + self.repetition = repetition + self.annotations = annotations self.font_path = font_path + @classmethod + def _from_raw( + cls, + *, + string: str, + height: float, + font_path: str, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._string = string + new._height = height + new._rotation = rotation % (2 * pi) + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations + new.font_path = font_path + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/test/test_raw_constructors.py b/masque/test/test_raw_constructors.py new file mode 100644 index 0000000..2f86ba0 --- /dev/null +++ b/masque/test/test_raw_constructors.py @@ -0,0 +1,97 @@ +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from ..shapes import Arc, Circle, Ellipse, Path, Text + + +def test_circle_raw_constructor_matches_public() -> None: + raw = Circle._from_raw( + radius=5.0, + offset=numpy.array([1.0, 2.0]), + annotations={'1': ['circle']}, + ) + public = Circle( + radius=5.0, + offset=(1.0, 2.0), + annotations={'1': ['circle']}, + ) + assert raw == public + + +def test_ellipse_raw_constructor_matches_public() -> None: + raw = Ellipse._from_raw( + radii=numpy.array([3.0, 5.0]), + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + public = Ellipse( + radii=(3.0, 5.0), + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + assert raw == public + + +def test_arc_raw_constructor_matches_public() -> None: + raw = Arc._from_raw( + radii=numpy.array([10.0, 6.0]), + angles=numpy.array([0.0, pi / 2]), + width=2.0, + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + public = Arc( + radii=(10.0, 6.0), + angles=(0.0, pi / 2), + width=2.0, + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + assert raw == public + + +def test_path_raw_constructor_matches_public() -> None: + raw = Path._from_raw( + vertices=numpy.array([[0.0, 0.0], [10.0, 0.0], [10.0, 5.0]]), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=numpy.array([1.0, 3.0]), + annotations={'4': ['path']}, + ) + public = Path( + vertices=((0.0, 0.0), (10.0, 0.0), (10.0, 5.0)), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=(1.0, 3.0), + annotations={'4': ['path']}, + ) + assert raw == public + assert raw.cap_extensions is not None + assert_allclose(raw.cap_extensions, [1.0, 3.0]) + + +def test_text_raw_constructor_matches_public() -> None: + raw = Text._from_raw( + string='RAW', + height=12.0, + font_path='font.otf', + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + public = Text( + string='RAW', + height=12.0, + font_path='font.otf', + offset=(1.0, 2.0), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + assert raw == public From d6b6e39dbe3bd18d529af718b6d45d4628c6b696 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:31 -0700 Subject: [PATCH 31/36] [arrow] add lazy arrow reader --- examples/profile_gdsii_readers.py | 12 + masque/file/gdsii_arrow.py | 95 ++- masque/file/gdsii_lazy_arrow.py | 960 +++++++++++++++++++++++++++ masque/test/test_gdsii_arrow.py | 12 + masque/test/test_gdsii_lazy_arrow.py | 174 +++++ 5 files changed, 1237 insertions(+), 16 deletions(-) create mode 100644 masque/file/gdsii_lazy_arrow.py create mode 100644 masque/test/test_gdsii_lazy_arrow.py diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index 129a391..fb7c99e 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -56,6 +56,18 @@ def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: return _summarize_library(path, elapsed_s, info, lib) if stage == 'arrow_import': + if hasattr(module, 'readfile_arrow'): + libarr, _info = module.readfile_arrow(path) + elapsed_s = time.perf_counter() - start + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': 1, + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + arrow_arr = module._read_to_arrow(path) elapsed_s = time.perf_counter() - start return _summarize_arrow_import(path, elapsed_s, arrow_arr) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 58493c1..b97005b 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -53,7 +53,20 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') +ffi.cdef( + """ + void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); + void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + void read_cells_bytes( + uint8_t* data, + size_t size, + uint64_t* ranges, + size_t range_count, + struct ArrowArray* array, + struct ArrowSchema* schema + ); + """ +) clib: Any | None = None @@ -189,12 +202,36 @@ def _read_to_arrow( ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + +def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + return pyarrow.Array._import_from_c(iptr_array, iptr_schema) - return arrow_arr + +def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) + + +def _read_selected_cells_to_arrow( + buffer: bytes | mmap.mmap | memoryview, + ranges: NDArray[numpy.uint64], + ) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) + cranges = ffi.from_buffer('uint64_t[]', flat_ranges) + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + return _import_arrow_array(ptr_array, ptr_schema) def readfile( @@ -211,6 +248,9 @@ def readfile( filename: Filename to save to. *args: passed to `read()` **kwargs: passed to `read()` + + For callers that can consume Arrow directly, prefer `readfile_arrow()` + to skip Python `Pattern` construction entirely. """ arrow_arr = _read_to_arrow(filename) assert len(arrow_arr) == 1 @@ -220,6 +260,28 @@ def readfile( return results +def readfile_arrow( + filename: str | pathlib.Path, + ) -> tuple[pyarrow.StructScalar, dict[str, Any]]: + """ + Read a GDSII file into the native Arrow representation without converting + it into `masque.Library` / `Pattern` objects. + + This is the lowest-overhead public read path exposed by this module. + + Args: + filename: Filename to read. + + Returns: + - Arrow struct scalar for the library payload + - dict of GDSII library info + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + libarr = arrow_arr[0] + return libarr, _read_header(libarr) + + def read_arrow( libarr: pyarrow.Array, raw_mode: bool = True, @@ -247,7 +309,7 @@ def read_arrow( library_info = _read_header(libarr) layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) - layer_tups = [tuple(pair) for pair in layer_names_np] + layer_tups = [(int(pair[0]), int(pair[1])) for pair in layer_names_np] cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() @@ -379,7 +441,7 @@ def read_arrow( mlib = Library() for cc in range(len(libarr['cells'])): - name = cell_names[cell_ids[cc]] + name = cell_names[int(cell_ids[cc])] pat = Pattern() _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) @@ -459,7 +521,7 @@ def _append_plain_refs_sorted( target_start = 0 while target_start < elem_count: - target_id = elem_targets[target_start] + target_id = int(elem_targets[target_start]) target_stop = target_start + 1 while target_stop < elem_count and elem_targets[target_stop] == target_id: target_stop += 1 @@ -513,9 +575,10 @@ def _arefs_to_mrefs( target = None append_ref: Callable[[Ref], Any] | None = None for ee in range(len(elem_targets)): - if target != elem_targets[ee]: - target = elem_targets[ee] - append_ref = pat.refs[cell_names[target]].append + target_id = int(elem_targets[ee]) + if target != target_id: + target = target_id + append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] append_ref(make_ref( @@ -564,7 +627,7 @@ def _sref_props_to_mrefs( repetition=None, annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _aref_props_to_mrefs( @@ -608,7 +671,7 @@ def _aref_props_to_mrefs( repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) - pat.refs[cell_names[elem_targets[ee]]].append(ref) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) def _texts_to_labels( @@ -633,7 +696,7 @@ def _texts_to_labels( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] offset = elem_xy[ee] string = elem_strings[ee] @@ -669,7 +732,7 @@ def _gpaths_to_mpaths( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] cap_int = elem_path_types[ee] @@ -725,7 +788,7 @@ def _boundary_batches_to_polygons( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] @@ -765,7 +828,7 @@ def _rect_batches_to_rectcollections( raw_mode = global_args['raw_mode'] for bb in range(batch_count): - layer = layer_tups[elem_layer_inds[bb]] + layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] if raw_mode: rect_collection = RectCollection._from_raw(rects=rects, annotations=None) @@ -799,7 +862,7 @@ def _boundary_props_to_polygons( raw_mode = global_args['raw_mode'] for ee in range(elem_count): - layer = layer_tups[elem_layer_inds[ee]] + layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) if raw_mode: diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py new file mode 100644 index 0000000..9a03960 --- /dev/null +++ b/masque/file/gdsii_lazy_arrow.py @@ -0,0 +1,960 @@ +""" +Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. + +This module is intentionally separate from `gdsii_arrow` so the eager read path +keeps its current behavior and performance profile. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping, Sequence +import copy +import gzip +import logging +import mmap +import pathlib + +import numpy +from numpy.typing import NDArray +import pyarrow +import klamath + +from . import gdsii, gdsii_arrow +from .utils import is_gzipped, tmpfile +from ..error import LibraryError +from ..library import ILibrary, ILibraryView, Library, LibraryView, dangling_mode_t +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _StructRange: + start: int + end: int + + +@dataclass +class _SourceBuffer: + path: pathlib.Path + data: bytes | mmap.mmap + handle: IO[bytes] | None = None + + def raw_slice(self, start: int, end: int) -> bytes: + return self.data[start:end] + + +@dataclass +class _ScanRefs: + offsets: NDArray[numpy.integer[Any]] + targets: NDArray[numpy.integer[Any]] + xy: NDArray[numpy.int32] + xy0: NDArray[numpy.int32] + xy1: NDArray[numpy.int32] + counts: NDArray[numpy.int64] + invert_y: NDArray[numpy.bool_ | numpy.bool] + angle_rad: NDArray[numpy.floating[Any]] + scale: NDArray[numpy.floating[Any]] + + +@dataclass(frozen=True) +class _CellScan: + cell_id: int + struct_range: _StructRange + ref_start: int + ref_stop: int + children: set[str] + + +@dataclass +class _ScanPayload: + libarr: pyarrow.StructScalar + library_info: dict[str, Any] + cell_names: list[str] + cell_order: list[str] + cells: dict[str, _CellScan] + refs: _ScanRefs + + +@dataclass +class _SourceLayer: + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + layer_index: int + source_name: str + + +def is_available() -> bool: + return gdsii_arrow.is_available() + + +def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: + return gdsii_arrow._read_header(libarr) + + +def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: + if is_gzipped(path): + with gzip.open(path, mode='rb') as stream: + data = stream.read() + return _SourceBuffer(path=path, data=data) + + handle = path.open(mode='rb', buffering=0) + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + return _SourceBuffer(path=path, data=mapped, handle=handle) + + +def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: + library_info = _read_header(libarr) + cell_names = libarr['cell_names'].as_py() + + cells = libarr['cells'] + cell_values = cells.values + cell_ids = cell_values.field('id').to_numpy() + struct_starts = cell_values.field('struct_start_offset').to_numpy() + struct_ends = cell_values.field('struct_end_offset').to_numpy() + + refs = cell_values.field('refs') + ref_values = refs.values + ref_offsets = refs.offsets.to_numpy() + targets = ref_values.field('target').to_numpy() + xy = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) + xy0 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) + xy1 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) + counts = gdsii_arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) + invert_y = ref_values.field('invert_y').to_numpy(zero_copy_only=False) + angle_rad = ref_values.field('angle_rad').to_numpy() + scale = ref_values.field('scale').to_numpy() + + ref_payload = _ScanRefs( + offsets=ref_offsets, + targets=targets, + xy=xy, + xy0=xy0, + xy1=xy1, + counts=counts, + invert_y=invert_y, + angle_rad=angle_rad, + scale=scale, + ) + + cell_order = [cell_names[int(cell_id)] for cell_id in cell_ids] + cell_scan: dict[str, _CellScan] = {} + for cc, name in enumerate(cell_order): + ref_start = int(ref_offsets[cc]) + ref_stop = int(ref_offsets[cc + 1]) + children = { + cell_names[int(target)] + for target in targets[ref_start:ref_stop] + } + cell_scan[name] = _CellScan( + cell_id=int(cell_ids[cc]), + struct_range=_StructRange(int(struct_starts[cc]), int(struct_ends[cc])), + ref_start=ref_start, + ref_stop=ref_stop, + children=children, + ) + + return _ScanPayload( + libarr=libarr, + library_info=library_info, + cell_names=cell_names, + cell_order=cell_order, + cells=cell_scan, + refs=ref_payload, + ) + + +def _pattern_children(pat: Pattern) -> set[str]: + return {child for child, refs in pat.refs.items() if child is not None and refs} + + +def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: + if not pat.refs: + return pat + pat.refs = map_targets(pat.refs, remap) + return pat + + +def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: + if isinstance(source, ILibraryView): + return source + return LibraryView(source) + + +def _source_order(source: ILibraryView) -> list[str]: + if isinstance(source, ArrowLibrary): + return list(source.source_order()) + return list(source.keys()) + + +def _make_ref_rows( + xy: NDArray[numpy.integer[Any]], + angle_rad: NDArray[numpy.floating[Any]], + invert_y: NDArray[numpy.bool_ | numpy.bool], + scale: NDArray[numpy.floating[Any]], + ) -> NDArray[numpy.float64]: + rows = numpy.empty((len(xy), 5), dtype=float) + rows[:, :2] = xy + rows[:, 2] = angle_rad + rows[:, 3] = invert_y.astype(float) + rows[:, 4] = scale + return rows + + +def _expand_aref_row( + xy: NDArray[numpy.integer[Any]], + xy0: NDArray[numpy.integer[Any]], + xy1: NDArray[numpy.integer[Any]], + counts: NDArray[numpy.integer[Any]], + angle_rad: float, + invert_y: bool, + scale: float, + ) -> NDArray[numpy.float64]: + a_count = int(counts[0]) + b_count = int(counts[1]) + aa, bb = numpy.meshgrid(numpy.arange(a_count), numpy.arange(b_count), indexing='ij') + displacements = aa.reshape(-1, 1) * xy0[None, :] + bb.reshape(-1, 1) * xy1[None, :] + rows = numpy.empty((displacements.shape[0], 5), dtype=float) + rows[:, :2] = xy + displacements + rows[:, 2] = angle_rad + rows[:, 3] = float(invert_y) + rows[:, 4] = scale + return rows + + +class ArrowLibrary(ILibraryView): + """ + Read-only library backed by the native lazy Arrow scan schema. + + Materializing a cell via `__getitem__` caches a real `Pattern` for that cell. + Cached cells are treated as edited for future writes from this module. + """ + + path: pathlib.Path + library_info: dict[str, Any] + + def __init__( + self, + *, + path: pathlib.Path, + payload: _ScanPayload, + source: _SourceBuffer, + ) -> None: + self.path = path + self.library_info = payload.library_info + self._payload = payload + self._source = source + self._cache: dict[str, Pattern] = {} + + @classmethod + def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: + path = pathlib.Path(filename).expanduser().resolve() + source = _open_source_buffer(path) + scan_arr = gdsii_arrow._scan_buffer_to_arrow(source.data) + assert len(scan_arr) == 1 + payload = _extract_scan_payload(scan_arr[0]) + return cls(path=path, payload=payload, source=source) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._payload.cell_order) + + def __len__(self) -> int: + return len(self._payload.cell_order) + + def __contains__(self, key: object) -> bool: + return key in self._payload.cells + + def source_order(self) -> tuple[str, ...]: + return tuple(self._payload.cell_order) + + def raw_struct_bytes(self, name: str) -> bytes: + struct_range = self._payload.cells[name].struct_range + return self._source.raw_slice(struct_range.start, struct_range.end) + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = self._materialize_patterns(names, persist=persist) + return LibraryView(mats) + + def _materialize_patterns( + self, + names: Sequence[str], + *, + persist: bool, + ) -> dict[str, Pattern]: + ordered_names = list(dict.fromkeys(names)) + missing = [name for name in ordered_names if name not in self._payload.cells] + if missing: + raise KeyError(missing[0]) + + materialized: dict[str, Pattern] = {} + uncached = [name for name in ordered_names if name not in self._cache] + if uncached: + ranges = numpy.asarray( + [ + [ + self._payload.cells[name].struct_range.start, + self._payload.cells[name].struct_range.end, + ] + for name in uncached + ], + dtype=numpy.uint64, + ) + arrow_arr = gdsii_arrow._read_selected_cells_to_arrow(self._source.data, ranges) + assert len(arrow_arr) == 1 + selected_lib, _info = gdsii_arrow.read_arrow(arrow_arr[0]) + for name in uncached: + pat = selected_lib[name] + materialized[name] = pat + if persist: + self._cache[name] = pat + + for name in ordered_names: + if name in self._cache: + materialized[name] = self._cache[name] + return materialized + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + return self._materialize_patterns((name,), persist=persist)[name] + + def _raw_children(self, name: str) -> set[str]: + return set(self._payload.cells[name].children) + + def _collect_raw_transforms(self, cell: _CellScan, target_id: int) -> list[NDArray[numpy.float64]]: + refs = self._payload.refs + start = cell.ref_start + stop = cell.ref_stop + if stop <= start: + return [] + + targets = refs.targets[start:stop] + mask = targets == target_id + if not mask.any(): + return [] + + rows: list[NDArray[numpy.float64]] = [] + counts = refs.counts[start:stop] + unit_mask = mask & (counts[:, 0] == 1) & (counts[:, 1] == 1) + if unit_mask.any(): + rows.append(_make_ref_rows( + refs.xy[start:stop][unit_mask], + refs.angle_rad[start:stop][unit_mask], + refs.invert_y[start:stop][unit_mask], + refs.scale[start:stop][unit_mask], + )) + + aref_indices = numpy.nonzero(mask & ~unit_mask)[0] + for idx in aref_indices: + abs_idx = start + int(idx) + rows.append(_expand_aref_row( + xy=refs.xy[abs_idx], + xy0=refs.xy0[abs_idx], + xy1=refs.xy1[abs_idx], + counts=refs.counts[abs_idx], + angle_rad=float(refs.angle_rad[abs_idx]), + invert_y=bool(refs.invert_y[abs_idx]), + scale=float(refs.scale[abs_idx]), + )) + return rows + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._payload.cell_order: + if name in self._cache: + graph[name] = _pattern_children(self._cache[name]) + else: + graph[name] = self._raw_children(name) + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + graph = self.child_graph(dangling='ignore') + names = set(graph) + not_toplevel: set[str] = set() + for children in graph.values(): + not_toplevel |= children + return list(names - not_toplevel) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + target_id = self._payload.cells.get(name) + for parent in parent_graph.get(name, set()): + if parent in self._cache: + for ref in self._cache[parent].refs.get(name, []): + instances[parent].append(ref.as_transforms()) + continue + + if target_id is None or parent not in self._payload.cells: + continue + rows = self._collect_raw_transforms(self._payload.cells[parent], target_id.cell_id) + if rows: + instances[parent].extend(rows) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + full_path = (parent,) + path + result[full_path] = instances + return result + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + view = _coerce_library_view(source) + source_order = _source_order(view) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + + for name in source_order: + visible = name + if visible in self._entries or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._entries or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return rename_map + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + remap = lambda target: None if target is None else self._effective_target(layer, target) + pat = _remap_pattern_targets(source_pat, remap) + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = _pattern_children(entry) + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + +def readfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + lib = ArrowLibrary.from_file(filename) + return lib, lib.library_info + + +def load_libraryfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + return readfile(filename) + + +def _get_write_info( + library: Mapping[str, Pattern] | ILibraryView, + *, + meters_per_unit: float | None, + logical_units_per_unit: float | None, + library_name: str | None, + ) -> tuple[float, float, str]: + if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: + return meters_per_unit, logical_units_per_unit, library_name + + infos: list[dict[str, Any]] = [] + if isinstance(library, ArrowLibrary): + infos.append(library.library_info) + elif isinstance(library, OverlayLibrary): + for layer in library._layers: + if isinstance(layer.library, ArrowLibrary): + infos.append(layer.library.library_info) + + if infos: + unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} + if len(unit_pairs) > 1: + raise LibraryError('Merged lazy GDS sources must have identical units before writing') + info = infos[0] + meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit + logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit + name = info['name'] if library_name is None else library_name + return meters, logical, name + + if meters_per_unit is None or logical_units_per_unit is None or library_name is None: + raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') + return meters_per_unit, logical_units_per_unit, library_name + + +def _can_copy_arrow_cell(library: ArrowLibrary, name: str) -> bool: + return name not in library._cache + + +def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: + layer = library._layers[entry.layer_index] + if not isinstance(layer.library, ArrowLibrary): + return False + if name != entry.source_name: + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(library._effective_target(layer, child) == child for child in children) + + +def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: + elements: list[klamath.elements.Element] = [] + elements += gdsii._shapes_to_elements(pat.shapes) + elements += gdsii._labels_to_texts(pat.labels) + elements += gdsii._mrefs_to_grefs(pat.refs) + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) + + +def write( + library: Mapping[str, Pattern] | ILibraryView, + stream: IO[bytes], + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + meters_per_unit, logical_units_per_unit, library_name = _get_write_info( + library, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + + header = klamath.library.FileHeader( + name=library_name.encode('ASCII'), + user_units_per_db_unit=logical_units_per_unit, + meters_per_db_unit=meters_per_unit, + ) + header.write(stream) + + if isinstance(library, ArrowLibrary): + for name in library.source_order(): + if _can_copy_arrow_cell(library, name): + stream.write(library.raw_struct_bytes(name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + if isinstance(library, OverlayLibrary): + for name in library.source_order(): + entry = library._entries[name] + if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + layer = library._layers[entry.layer_index] + assert isinstance(layer.library, ArrowLibrary) + stream.write(layer.library.raw_struct_bytes(entry.source_name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) + + +def writefile( + library: Mapping[str, Pattern] | ILibraryView, + filename: str | pathlib.Path, + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + path = pathlib.Path(filename) + + with tmpfile(path) as base_stream: + streams: tuple[Any, ...] = (base_stream,) + if path.suffix == '.gz': + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + streams = (stream,) + streams + else: + stream = base_stream + + try: + write( + library, + stream, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + finally: + for ss in streams: + ss.close() diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 2ef4d83..f3f4f6a 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_native.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == manifest.library_name + assert libarr['lib_name'].as_py() == manifest.library_name + assert len(libarr['cells']) == manifest.cells + assert 0 < len(libarr['layers']) <= manifest.layers + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py new file mode 100644 index 0000000..61a99af --- /dev/null +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -0,0 +1,174 @@ +from pathlib import Path + +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..file import gdsii, gdsii_lazy_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_lazy_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _make_small_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 5], [0, 5]]) + lib['leaf'] = leaf + + mid = Pattern() + mid.ref('leaf', offset=(10, 20)) + mid.ref('leaf', offset=(40, 0), repetition=Grid(a_vector=(12, 0), a_count=2, b_vector=(0, 9), b_count=2)) + lib['mid'] = mid + + top = Pattern() + top.ref('mid', offset=(100, 200)) + lib['top'] = top + return lib + + +def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_lazy.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert lib.top() == 'TOP' + assert 'TOP' in lib.child_graph(dangling='ignore') + + +def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: + gds_file = tmp_path / 'refs.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-refs') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + local = lib.find_refs_local('leaf') + assert set(local) == {'mid'} + assert sum(arr.shape[0] for arr in local['mid']) == 5 + + global_refs = lib.find_refs_global('leaf') + assert {path for path in global_refs} == {('top', 'mid', 'leaf')} + assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 + + +def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: + base_a = Library() + leaf_a = Pattern() + leaf_a.polygon((1, 0), vertices=[[0, 0], [8, 0], [8, 8], [0, 8]]) + base_a['leaf'] = leaf_a + top_a = Pattern() + top_a.ref('leaf', offset=(0, 0)) + base_a['top_a'] = top_a + + base_b = Library() + leaf_b = Pattern() + leaf_b.polygon((2, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + base_b['leaf'] = leaf_b + top_b = Pattern() + top_b.ref('leaf', offset=(20, 30)) + base_b['top_b'] = top_b + + gds_a = tmp_path / 'a.gds' + gds_b = tmp_path / 'b.gds' + gdsii.writefile(base_a, gds_a, meters_per_unit=1e-9, library_name='overlay') + gdsii.writefile(base_b, gds_b, meters_per_unit=1e-9, library_name='overlay') + + lib_a, _ = gdsii_lazy_arrow.readfile(gds_a) + lib_b, _ = gdsii_lazy_arrow.readfile(gds_b) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib_a) + rename_map = overlay.add_source(lib_b, rename_theirs=lambda lib, name: lib.get_name(name)) + renamed_leaf = rename_map['leaf'] + + assert rename_map == {'leaf': renamed_leaf} + assert renamed_leaf != 'leaf' + assert len(lib_a._cache) == 0 + assert len(lib_b._cache) == 0 + + overlay.move_references('leaf', renamed_leaf) + + out_file = tmp_path / 'overlay_out.gds' + gdsii_lazy_arrow.writefile(overlay, out_file) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf', renamed_leaf, 'top_a', 'top_b'} + assert 'top_b' in roundtrip + assert list(roundtrip['top_b'].refs.keys()) == [renamed_leaf] + + +def test_gdsii_writer_accepts_overlay_library(tmp_path: Path) -> None: + gds_file = tmp_path / 'overlay_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-src') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib) + overlay.rename('leaf', 'leaf_copy', move_references=True) + + out_file = tmp_path / 'overlay_via_eager_writer.gds' + gdsii.writefile( + overlay, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf_copy', 'mid', 'top'} + assert list(roundtrip['mid'].refs.keys()) == ['leaf_copy'] + + +def test_svg_writer_uses_detached_materialized_copy(tmp_path: Path) -> None: + pytest.importorskip('svgwrite') + from ..file import svg + from ..shapes import Path as MPath + + gds_file = tmp_path / 'svg_source.gds' + src = _make_small_library() + src['top'].path((3, 0), vertices=[[0, 0], [0, 20]], width=4) + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='svg-src') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + top_pat = lib['top'] + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) + + svg_path = tmp_path / 'lazy.svg' + svg.writefile(lib, 'top', str(svg_path)) + + assert svg_path.exists() + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) From d73a20cfca840e00fca9f9a813301916ab5bc00a Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 20:50:45 -0700 Subject: [PATCH 32/36] [svg] avoid mutating the original library --- masque/file/svg.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/masque/file/svg.py b/masque/file/svg.py index 621bcdb..772aa39 100644 --- a/masque/file/svg.py +++ b/masque/file/svg.py @@ -45,6 +45,10 @@ def _make_svg_ids(names: Mapping[str, Pattern]) -> dict[str, str]: return svg_ids +def _detached_library(library: Mapping[str, Pattern]) -> dict[str, Pattern]: + return {name: pat.deepcopy() for name, pat in library.items()} + + def writefile( library: Mapping[str, Pattern], top: str, @@ -53,13 +57,12 @@ def writefile( annotate_ports: bool = False, ) -> None: """ - Write a Pattern to an SVG file, by first calling .polygonize() on it + Write a Pattern to an SVG file, by first calling .polygonize() on a detached + materialized copy to change the shapes into polygons, and then writing patterns as SVG groups (, inside ), polygons as paths (), and refs as elements. - Note that this function modifies the Pattern. - If `custom_attributes` is `True`, a non-standard `pattern_layer` attribute is written to the relevant elements. @@ -71,19 +74,21 @@ def writefile( prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. custom_attributes: Whether to write non-standard `pattern_layer` attribute to the SVG elements. annotate_ports: If True, draw an arrow for each port (similar to `Pattern.visualize(..., ports=True)`). """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize pattern pattern.polygonize() - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) @@ -96,10 +101,10 @@ def writefile( # Create file svg = svgwrite.Drawing(filename, profile='full', viewBox=viewbox_string, debug=(not custom_attributes)) - svg_ids = _make_svg_ids(library) + svg_ids = _make_svg_ids(detached) # Now create a group for each pattern and add in any Boundary and Use elements - for name, pat in library.items(): + for name, pat in detached.items(): svg_group = svg.g(id=svg_ids[name], fill='blue', stroke='red') for layer, shapes in pat.shapes.items(): @@ -158,21 +163,21 @@ def writefile_inverted( box and drawing the polygons with reverse vertex order inside it, all within one `` element. - Note that this function modifies the Pattern. - If you want pattern polygonized with non-default arguments, just call `pattern.polygonize()` prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize and flatten pattern - pattern.polygonize().flatten(library) + pattern.polygonize().flatten(detached) - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) From 4eff8203ce3d9c138818d6b81f668bfa3d321285 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 20 Apr 2026 21:34:10 -0700 Subject: [PATCH 33/36] [gdsii_arrow] remove non-raw-mode arrow option; fix gzip wrapper --- masque/file/gdsii_arrow.py | 115 ++++++++------------------- masque/test/test_gdsii_arrow.py | 39 +++++++++ masque/test/test_gdsii_lazy_arrow.py | 85 ++++++++++++++++++++ 3 files changed, 158 insertions(+), 81 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index b97005b..a38dee5 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -27,7 +27,6 @@ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable from importlib.machinery import EXTENSION_SUFFIXES import importlib.util -import io import mmap import logging import os @@ -35,11 +34,11 @@ import pathlib import gzip import string import sys +import tempfile from pprint import pformat import numpy from numpy.typing import ArrayLike, NDArray -from numpy.testing import assert_equal import pyarrow from pyarrow.cffi import ffi @@ -70,8 +69,6 @@ ffi.cdef( clib: Any | None = None -ZERO_OFFSET = numpy.zeros(2) - path_cap_map = { 0: Path.Cap.Flush, @@ -195,13 +192,22 @@ def _read_annotations( def _read_to_arrow( filename: str | pathlib.Path, - *args, - **kwargs, ) -> pyarrow.Array: path = pathlib.Path(filename).expanduser().resolve() ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + if is_gzipped(path): + with gzip.open(path, mode='rb') as src: + data = src.read() + with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream: + tmp_stream.write(data) + tmp_name = tmp_stream.name + try: + _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + finally: + pathlib.Path(tmp_name).unlink(missing_ok=True) + else: + _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) return _import_arrow_array(ptr_array, ptr_schema) @@ -236,18 +242,14 @@ def _read_selected_cells_to_arrow( def readfile( filename: str | pathlib.Path, - *args, - **kwargs, ) -> tuple[Library, dict[str, Any]]: """ - Wrapper for `read()` that takes a filename or path instead of a stream. + Read a GDSII file from a path into `masque.Library` / `Pattern` objects. Will automatically decompress gzipped files. Args: - filename: Filename to save to. - *args: passed to `read()` - **kwargs: passed to `read()` + filename: Filename to read. For callers that can consume Arrow directly, prefer `readfile_arrow()` to skip Python `Pattern` construction entirely. @@ -284,7 +286,6 @@ def readfile_arrow( def read_arrow( libarr: pyarrow.Array, - raw_mode: bool = True, ) -> tuple[Library, dict[str, Any]]: """ # TODO check GDSII file for cycles! @@ -299,8 +300,7 @@ def read_arrow( per database unit Args: - stream: Stream to read from. - raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + libarr: Arrow library payload as returned by `readfile_arrow()`. Returns: - dict of pattern_name:Patterns generated from GDSII structures @@ -436,7 +436,6 @@ def read_arrow( global_args = dict( cell_names = cell_names, layer_tups = layer_tups, - raw_mode = raw_mode, ) mlib = Library() @@ -488,7 +487,6 @@ def _srefs_to_mrefs( elem_invert_y = elem['invert_y'][start:stop] elem_angle_rad = elem['angle_rad'][start:stop] elem_scale = elem['scale'][start:stop] - raw_mode = global_args['raw_mode'] _append_plain_refs_sorted( pat=pat, @@ -498,7 +496,6 @@ def _srefs_to_mrefs( elem_invert_y=elem_invert_y, elem_angle_rad=elem_angle_rad, elem_scale=elem_scale, - raw_mode=raw_mode, ) @@ -511,14 +508,11 @@ def _append_plain_refs_sorted( elem_invert_y: NDArray[numpy.bool_ | numpy.bool], elem_angle_rad: NDArray[numpy.floating[Any]], elem_scale: NDArray[numpy.floating[Any]], - raw_mode: bool, ) -> None: elem_count = len(elem_targets) if elem_count == 0: return - make_ref = Ref._from_raw if raw_mode else Ref - target_start = 0 while target_start < elem_count: target_id = int(elem_targets[target_start]) @@ -528,7 +522,7 @@ def _append_plain_refs_sorted( append_refs = pat.refs[cell_names[target_id]].extend append_refs( - make_ref( + Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -564,10 +558,6 @@ def _arefs_to_mrefs( elem_xy0 = elem['xy0'][start:stop] elem_xy1 = elem['xy1'][start:stop] elem_counts = elem['counts'][start:stop] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid if len(elem_targets) == 0: return @@ -581,12 +571,12 @@ def _arefs_to_mrefs( append_ref = pat.refs[cell_names[target_id]].append assert append_ref is not None a_count, b_count = elem_counts[ee] - append_ref(make_ref( + append_ref(Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=None, )) @@ -613,13 +603,10 @@ def _sref_props_to_mrefs( elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], @@ -655,20 +642,16 @@ def _aref_props_to_mrefs( elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] - raw_mode = global_args['raw_mode'] - - make_ref = Ref._from_raw if raw_mode else Ref - make_grid = Grid._from_raw if raw_mode else Grid for ee in range(elem_count): a_count, b_count = elem_counts[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - ref = make_ref( + ref = Ref._from_raw( offset=elem_xy[ee], mirrored=elem_invert_y[ee], rotation=elem_angle_rad[ee], scale=elem_scale[ee], - repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), annotations=annotations, ) pat.refs[cell_names[int(elem_targets[ee])]].append(ref) @@ -693,7 +676,6 @@ def _texts_to_labels( elem_xy = xy[elem_slc][:elem_count] elem_layer_inds = layer_inds[elem_slc][:elem_count] elem_strings = elem['string'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] @@ -701,10 +683,7 @@ def _texts_to_labels( string = elem_strings[ee] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) - else: - mlabel = Label(string=string, offset=offset, annotations=annotations) + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) pat.labels[layer].append(mlabel) @@ -730,7 +709,6 @@ def _gpaths_to_mpaths( elem_path_types = elem['path_type'][elem_slc][:elem_count] elem_extensions = elem['extensions'][elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] @@ -743,23 +721,13 @@ def _gpaths_to_mpaths( cap_extensions = None annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - path = Path._from_raw( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - annotations=annotations, - ) - else: - path = Path( - vertices=vertices, - width=width, - cap=cap, - cap_extensions=cap_extensions, - offset=ZERO_OFFSET, - annotations=annotations, - ) + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) pat.shapes[layer].append(path) @@ -786,23 +754,16 @@ def _boundary_batches_to_polygons( elem_poly_off = poly_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=None) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None) + poly = Polygon._from_raw(vertices=vertices, annotations=None) pat.shapes[layer].append(poly) else: - if raw_mode: - polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) - else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None) + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) pat.shapes[layer].append(polys) @@ -826,14 +787,10 @@ def _rect_batches_to_rectcollections( elem_rect_off = rect_off[elem_slc] elem_layer_inds = layer_inds[elem_slc][:batch_count] - raw_mode = global_args['raw_mode'] for bb in range(batch_count): layer = layer_tups[int(elem_layer_inds[bb])] rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] - if raw_mode: - rect_collection = RectCollection._from_raw(rects=rects, annotations=None) - else: - rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None) + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) pat.shapes[layer].append(rect_collection) @@ -860,15 +817,11 @@ def _boundary_props_to_polygons( prop_offs = elem['prop_off'][elem_slc] elem_layer_inds = layer_inds[elem_slc][:elem_count] - raw_mode = global_args['raw_mode'] for ee in range(elem_count): layer = layer_tups[int(elem_layer_inds[ee])] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - poly = Polygon._from_raw(vertices=vertices, annotations=annotations) - else: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations) + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) pat.shapes[layer].append(poly) diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index f3f4f6a..660818c 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -187,6 +187,18 @@ def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: assert _library_summary(canonical_lib) == _library_summary(arrow_lib) +def test_gdsii_arrow_matches_gdsii_readfile_for_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_native.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -199,6 +211,33 @@ def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> No assert 0 < len(libarr['layers']) <= manifest.layers +def test_gdsii_arrow_readfile_arrow_reads_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'native_payload.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == 'masque-klamath' + assert libarr['lib_name'].as_py() == 'masque-klamath' + assert len(libarr['cells']) == len(lib) + assert len(libarr['layers']) > 0 + + +def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'removed_raw_mode.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, _ = gdsii_arrow.readfile_arrow(gds_file) + + with pytest.raises(TypeError): + gdsii_arrow.readfile(gds_file, raw_mode=False) + + with pytest.raises(TypeError): + gdsii_arrow.read_arrow(libarr, raw_mode=False) + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 61a99af..b30b058 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -34,6 +34,78 @@ def _make_small_library() -> Library: return lib +def _make_complex_ref_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]]) + lib['leaf'] = leaf + + child = Pattern() + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25) + lib['child'] = child + + sibling = Pattern() + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + ) + lib['sibling'] = sibling + + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref( + 'child', + offset=(40, 0), + mirrored=True, + rotation=numpy.pi / 4, + scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2), + ) + lib['fanout'] = fanout + + top = Pattern() + top.ref('child', offset=(500, 600)) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi) + top.ref('fanout', offset=(250, -75)) + lib['top'] = top + + return lib + + +def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: + arr = numpy.asarray(values, dtype=float) + arr = numpy.atleast_2d(arr) + rows = [ + ( + round(float(row[0]), 8), + round(float(row[1]), 8), + round(float(row[2]), 8), + bool(int(round(float(row[3])))), + round(float(row[4]), 8), + ) + for row in arr + ] + return tuple(sorted(rows)) + + +def _local_refs_key(refs: dict[str, list[numpy.ndarray]]) -> dict[str, tuple[tuple[object, ...], ...]]: + return { + parent: _transform_rows_key(numpy.concatenate(transforms)) + for parent, transforms in refs.items() + } + + +def _global_refs_key(refs: dict[tuple[str, ...], numpy.ndarray]) -> dict[tuple[str, ...], tuple[tuple[object, ...], ...]]: + return { + path: _transform_rows_key(transforms) + for path, transforms in refs.items() + } + + def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_lazy.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -62,6 +134,19 @@ def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 +def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None: + gds_file = tmp_path / 'complex_refs.gds' + src = _make_complex_ref_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-complex-refs') + + eager, _ = gdsii.readfile(gds_file) + lazy, _ = gdsii_lazy_arrow.readfile(gds_file) + + for name in ('leaf', 'child'): + assert _local_refs_key(lazy.find_refs_local(name)) == _local_refs_key(eager.find_refs_local(name)) + assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() From 8b765413f68f2507819233c6f17431e87b4c2666 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 21 Apr 2026 21:50:18 -0700 Subject: [PATCH 34/36] [arrow] improve test coverage and error handling --- masque/file/gdsii_arrow.py | 35 ++++++++++--- masque/test/test_gdsii_arrow.py | 59 +++++++++++++++++++++- masque/test/test_gdsii_lazy_arrow.py | 75 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index a38dee5..95229ea 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -37,6 +37,7 @@ import sys import tempfile from pprint import pformat +from klamath.basic import KlamathError import numpy from numpy.typing import ArrayLike, NDArray import pyarrow @@ -54,9 +55,10 @@ logger = logging.getLogger(__name__) ffi.cdef( """ - void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema); - void scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); - void read_cells_bytes( + const char* last_error_message(void); + int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); + int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + int read_cells_bytes( uint8_t* data, size_t size, uint64_t* ranges, @@ -203,11 +205,11 @@ def _read_to_arrow( tmp_stream.write(data) tmp_name = tmp_stream.name try: - _get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path') finally: pathlib.Path(tmp_name).unlink(missing_ok=True) else: - _get_clib().read_path(str(path).encode(), ptr_array, ptr_schema) + _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path') return _import_arrow_array(ptr_array, ptr_schema) @@ -217,12 +219,24 @@ def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: return pyarrow.Array._import_from_c(iptr_array, iptr_schema) +def _call_native(status: int, action: str) -> None: + if status == 0: + return + + err_ptr = _get_clib().last_error_message() + if err_ptr == ffi.NULL: + raise KlamathError(f'{action} failed') + + message = ffi.string(err_ptr).decode(errors='replace') + raise KlamathError(message) + + def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: ptr_array = ffi.new('struct ArrowArray[]', 1) ptr_schema = ffi.new('struct ArrowSchema[]', 1) buf_view = memoryview(buffer) cbuf = ffi.from_buffer('uint8_t[]', buf_view) - _get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema) + _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes') return _import_arrow_array(ptr_array, ptr_schema) @@ -236,7 +250,10 @@ def _read_selected_cells_to_arrow( cbuf = ffi.from_buffer('uint8_t[]', buf_view) flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) cranges = ffi.from_buffer('uint64_t[]', flat_ranges) - _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema) + _call_native( + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema), + 'read_cells_bytes', + ) return _import_arrow_array(ptr_array, ptr_schema) @@ -713,7 +730,9 @@ def _gpaths_to_mpaths( layer = layer_tups[int(elem_layer_inds[ee])] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] width = elem_widths[ee] - cap_int = elem_path_types[ee] + cap_int = int(elem_path_types[ee]) + if cap_int not in path_cap_map: + raise PatternError(f'Unrecognized path type: {cap_int}') cap = path_cap_map[cap_int] if cap_int == 4: cap_extensions = elem_extensions[ee] diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 660818c..b2962f8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -1,11 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') -from .. import Ref, Label +from .. import Ref, Label, PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -175,6 +179,26 @@ def _make_arrow_test_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_roundtrip.gds' @@ -238,6 +262,28 @@ def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: gdsii_arrow.read_arrow(libarr, raw_mode=False) +def test_gdsii_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_arrow + try: + gdsii_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: gds_file = tmp_path / 'many_cells_smoke.gds' manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) @@ -435,6 +481,17 @@ def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' +def test_gdsii_arrow_invalid_path_type_matches_gdsii(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii_arrow.readfile(gds_file) + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index b30b058..eb62721 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -1,10 +1,15 @@ from pathlib import Path +import subprocess +import sys +import textwrap +import klamath import numpy import pytest pytest.importorskip('pyarrow') +from .. import PatternError from ..library import Library from ..pattern import Pattern from ..repetition import Grid @@ -76,6 +81,26 @@ def _make_complex_ref_library() -> Library: return lib +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: arr = numpy.asarray(values, dtype=float) arr = numpy.atleast_2d(arr) @@ -147,6 +172,38 @@ def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) +def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_lazy_arrow + try: + gdsii_lazy_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_lazy_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + +def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + lib['top'] + + def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() @@ -165,6 +222,24 @@ def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> Non assert out_file.read_bytes() == gds_file.read_bytes() +def test_gdsii_lazy_arrow_gzipped_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds.gz' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through-gz') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds.gz' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: base_a = Library() leaf_a = Pattern() From 3ea67a9a6feb708867ebed1eaf9437c1d548e49e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 21 Apr 2026 23:17:31 -0700 Subject: [PATCH 35/36] [wip] Rework load_libraryfile and LazyLibrary using overlays --- examples/tutorial/library.py | 42 +- masque/file/gdsii.py | 115 +----- masque/file/gdsii_lazy.py | 373 ++++++++++++++++++ masque/file/gdsii_lazy_arrow.py | 515 ++----------------------- masque/file/gdsii_lazy_core.py | 665 ++++++++++++++++++++++++++++++++ masque/library.py | 9 + masque/test/test_gdsii_lazy.py | 101 +++++ 7 files changed, 1204 insertions(+), 616 deletions(-) create mode 100644 masque/file/gdsii_lazy.py create mode 100644 masque/file/gdsii_lazy_core.py create mode 100644 masque/test/test_gdsii_lazy.py diff --git a/examples/tutorial/library.py b/examples/tutorial/library.py index 1b9a1da..f6bcc4e 100644 --- a/examples/tutorial/library.py +++ b/examples/tutorial/library.py @@ -1,5 +1,5 @@ """ -Tutorial: using `LazyLibrary` and `Pather.interface()`. +Tutorial: using a source-backed lazy GDS library and `Pather.interface()`. This example assumes you have already read `devices.py` and generated the `circuit.gds` file it writes. The goal here is not the photonic-crystal geometry @@ -10,32 +10,28 @@ from typing import Any from pprint import pformat -from masque import Pather, LazyLibrary -from masque.file.gdsii import writefile, load_libraryfile +from masque import Pather +from masque.file.gdsii import writefile +from masque.file.gdsii_lazy import OverlayLibrary, readfile import basic_shapes import devices -from devices import data_to_ports from basic_shapes import GDS_OPTS def main() -> None: - # A `LazyLibrary` delays work until a pattern is actually needed. - # That applies both to GDS cells we load from disk and to python callables - # that generate patterns on demand. - lib = LazyLibrary() + # `OverlayLibrary` lets us mix source-backed GDS cells with python-generated + # patterns behind the same library interface. + lib = OverlayLibrary() # # Load some devices from a GDS file # - # Scan circuit.gds and prepare to lazy-load its contents - gds_lib, _properties = load_libraryfile('circuit.gds', postprocess=data_to_ports) - - # Add those cells into our lazy library. - # Nothing is read yet; we are only registering how to fetch and postprocess - # each pattern when it is first requested. - lib.add(gds_lib) + # Scan circuit.gds and prepare to lazy-load its contents. Port labels are + # imported on first materialization, but the raw source remains untouched. + gds_lib, _properties = readfile('circuit.gds') + lib.add_source(gds_lib.with_ports_from_data(layers=[(3, 0)], max_depth=1)) print('Patterns loaded from GDS into library:\n' + pformat(list(lib.keys()))) @@ -43,20 +39,18 @@ def main() -> None: # Add some new devices to the library, this time from python code rather than GDS # - lib['triangle'] = lambda: basic_shapes.triangle(devices.RADIUS) + lib['triangle'] = basic_shapes.triangle(devices.RADIUS) opts: dict[str, Any] = dict( lattice_constant = devices.LATTICE_CONSTANT, hole = 'triangle', ) - # Triangle-based variants. These lambdas are only recipes for building the - # patterns; they do not execute until someone asks for the cell. - lib['tri_wg10'] = lambda: devices.waveguide(length=10, mirror_periods=5, **opts) - lib['tri_wg05'] = lambda: devices.waveguide(length=5, mirror_periods=5, **opts) - lib['tri_wg28'] = lambda: devices.waveguide(length=28, mirror_periods=5, **opts) - lib['tri_bend0'] = lambda: devices.bend(mirror_periods=5, **opts) - lib['tri_ysplit'] = lambda: devices.y_splitter(mirror_periods=5, **opts) - lib['tri_l3cav'] = lambda: devices.perturbed_l3(xy_size=(4, 10), **opts, hole_lib=lib) + lib['tri_wg10'] = devices.waveguide(length=10, mirror_periods=5, **opts) + lib['tri_wg05'] = devices.waveguide(length=5, mirror_periods=5, **opts) + lib['tri_wg28'] = devices.waveguide(length=28, mirror_periods=5, **opts) + lib['tri_bend0'] = devices.bend(mirror_periods=5, **opts) + lib['tri_ysplit'] = devices.y_splitter(mirror_periods=5, **opts) + lib['tri_l3cav'] = devices.perturbed_l3(xy_size=(4, 10), **opts, hole_lib=lib) # # Build a mixed waveguide with an L3 cavity in the middle diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 1d8c3d1..21c6f94 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -22,8 +22,6 @@ Notes: from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable from types import MappingProxyType -import io -import mmap import logging import pathlib import gzip @@ -40,7 +38,7 @@ from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape from ..shapes import Polygon, Path, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t -from ..library import LazyLibrary, Library, ILibrary, ILibraryView +from ..library import Library, ILibrary logger = logging.getLogger(__name__) @@ -542,117 +540,6 @@ def _labels_to_texts(labels: dict[layer_t, list[Label]]) -> list[klamath.element return texts -def load_library( - stream: IO[bytes], - *, - full_load: bool = False, - postprocess: Callable[[ILibraryView, str, Pattern], Pattern] | None = None - ) -> tuple[LazyLibrary, dict[str, Any]]: - """ - Scan a GDSII stream to determine what structures are present, and create - a library from them. This enables deferred reading of structures - on an as-needed basis. - All structures are loaded as secondary - - Args: - stream: Seekable stream. Position 0 should be the start of the file. - The caller should leave the stream open while the library - is still in use, since the library will need to access it - in order to read the structure contents. - full_load: If True, force all structures to be read immediately rather - than as-needed. Since data is read sequentially from the file, this - will be faster than using the resulting library's `precache` method. - postprocess: If given, this function is used to post-process each - pattern *upon first load only*. - - Returns: - LazyLibrary object, allowing for deferred load of structures. - Additional library info (dict, same format as from `read`). - """ - stream.seek(0) - lib = LazyLibrary() - - if full_load: - # Full load approach (immediately load everything) - patterns, library_info = read(stream) - for name, pattern in patterns.items(): - if postprocess is not None: - lib[name] = postprocess(lib, name, pattern) - else: - lib[name] = pattern - return lib, library_info - - # Normal approach (scan and defer load) - library_info = _read_header(stream) - structs = klamath.library.scan_structs(stream) - - for name_bytes, pos in structs.items(): - name = name_bytes.decode('ASCII') - - def mkstruct(pos: int = pos, name: str = name) -> Pattern: - stream.seek(pos) - pat = read_elements(stream, raw_mode=True) - if postprocess is not None: - pat = postprocess(lib, name, pat) - return pat - - lib[name] = mkstruct - - return lib, library_info - - -def load_libraryfile( - filename: str | pathlib.Path, - *, - use_mmap: bool = True, - full_load: bool = False, - postprocess: Callable[[ILibraryView, str, Pattern], Pattern] | None = None - ) -> tuple[LazyLibrary, dict[str, Any]]: - """ - Wrapper for `load_library()` that takes a filename or path instead of a stream. - - Will automatically decompress the file if it is gzipped. - - NOTE that any streams/mmaps opened will remain open until ALL of the - `PatternGenerator` objects in the library are garbage collected. - - Args: - path: filename or path to read from - use_mmap: If `True`, will attempt to memory-map the file instead - of buffering. In the case of gzipped files, the file - is decompressed into a python `bytes` object in memory - and reopened as an `io.BytesIO` stream. - full_load: If `True`, immediately loads all data. See `load_library`. - postprocess: Passed to `load_library` - - Returns: - LazyLibrary object, allowing for deferred load of structures. - Additional library info (dict, same format as from `read`). - """ - path = pathlib.Path(filename) - stream: IO[bytes] - if is_gzipped(path): - if use_mmap: - logger.info('Asked to mmap a gzipped file, reading into memory instead...') - gz_stream = gzip.open(path, mode='rb') # noqa: SIM115 - stream = io.BytesIO(gz_stream.read()) # type: ignore - else: - gz_stream = gzip.open(path, mode='rb') # noqa: SIM115 - stream = io.BufferedReader(gz_stream) # type: ignore - else: # noqa: PLR5501 - if use_mmap: - base_stream = path.open(mode='rb', buffering=0) # noqa: SIM115 - stream = mmap.mmap(base_stream.fileno(), 0, access=mmap.ACCESS_READ) # type: ignore - else: - stream = path.open(mode='rb') # noqa: SIM115 - - try: - return load_library(stream, full_load=full_load, postprocess=postprocess) - finally: - if full_load: - stream.close() - - def check_valid_names( names: Iterable[str], max_length: int = 32, diff --git a/masque/file/gdsii_lazy.py b/masque/file/gdsii_lazy.py new file mode 100644 index 0000000..f9561af --- /dev/null +++ b/masque/file/gdsii_lazy.py @@ -0,0 +1,373 @@ +""" +Source-backed lazy GDSII reader using the pure-python klamath path. + +This module mirrors the lazy Arrow reader's interface closely enough to share +the same overlay and ports-import helpers, while still materializing cells +through the classic `gdsii` decoder. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Iterator, Sequence +import gzip +import io +import logging +import mmap +import pathlib + +import klamath +import numpy +from numpy.typing import NDArray +from klamath import records + +from . import gdsii +from .utils import is_gzipped +from .gdsii_lazy_core import OverlayLibrary, PortsLibraryView, _pattern_children, write, writefile +from ..error import LibraryError +from ..library import ILibraryView, LibraryView, dangling_mode_t +from ..pattern import Pattern +from ..utils import apply_transforms + + +logger = logging.getLogger(__name__) + + +@dataclass +class _SourceHandle: + path: pathlib.Path | None + stream: IO[bytes] + handle: IO[bytes] | None = None + + def close(self) -> None: + self.stream.close() + if self.handle is not None and self.handle is not self.stream: + self.handle.close() + self.handle = None + + +@dataclass(frozen=True) +class _CellScan: + offset: int + children: set[str] + + +def _open_source_stream( + filename: str | pathlib.Path, + *, + use_mmap: bool, + ) -> _SourceHandle: + path = pathlib.Path(filename).expanduser().resolve() + if is_gzipped(path): + if use_mmap: + logger.info('Asked to mmap a gzipped file, reading into memory instead...') + with gzip.open(path, mode='rb') as stream: + data = stream.read() + return _SourceHandle(path=path, stream=io.BytesIO(data)) + stream = cast('IO[bytes]', gzip.open(path, mode='rb')) + return _SourceHandle(path=path, stream=stream) + + if use_mmap: + handle = path.open(mode='rb', buffering=0) + mapped = cast('IO[bytes]', mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ)) + return _SourceHandle(path=path, stream=mapped, handle=handle) + + stream = path.open(mode='rb') + return _SourceHandle(path=path, stream=stream) + + +def _scan_library( + stream: IO[bytes], + ) -> tuple[dict[str, Any], list[str], dict[str, _CellScan]]: + library_info = gdsii._read_header(stream) + order: list[str] = [] + cells: dict[str, _CellScan] = {} + + found_struct = records.BGNSTR.skip_past(stream) + while found_struct: + name = records.STRNAME.skip_and_read(stream).decode('ASCII') + offset = stream.tell() + elements = klamath.library.read_elements(stream) + children = { + element.struct_name.decode('ASCII') + for element in elements + if isinstance(element, klamath.elements.Reference) + } + order.append(name) + cells[name] = _CellScan(offset=offset, children=children) + found_struct = records.BGNSTR.skip_past(stream) + + return library_info, order, cells + + +class GdsLibrarySource(ILibraryView): + """ + Read-only library backed by a seekable GDS stream. + + Cells are scanned once up front to discover order and child edges, then + materialized one at a time through the classic `gdsii.read_elements` path. + """ + + def __init__( + self, + *, + source: _SourceHandle, + library_info: dict[str, Any], + cell_order: Sequence[str], + cells: dict[str, _CellScan], + ) -> None: + self.path = source.path + self.library_info = library_info + self._source = source + self._cell_order = tuple(cell_order) + self._cells = cells + self._cache: dict[str, Pattern] = {} + self._lookups_in_progress: list[str] = [] + + @classmethod + def from_file( + cls, + filename: str | pathlib.Path, + *, + use_mmap: bool = True, + ) -> GdsLibrarySource: + source = _open_source_stream(filename, use_mmap=use_mmap) + source.stream.seek(0) + library_info, cell_order, cells = _scan_library(source.stream) + return cls(source=source, library_info=library_info, cell_order=cell_order, cells=cells) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._cell_order) + + def __len__(self) -> int: + return len(self._cell_order) + + def __contains__(self, key: object) -> bool: + return key in self._cells + + def source_order(self) -> tuple[str, ...]: + return self._cell_order + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = { + name: self._materialize_pattern(name, persist=persist) + for name in dict.fromkeys(names) + } + return LibraryView(mats) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name in self._cache: + return self._cache[name] + + if name not in self._cells: + raise KeyError(name) + + if name in self._lookups_in_progress: + chain = ' -> '.join(self._lookups_in_progress + [name]) + raise LibraryError( + f'Detected circular reference or recursive lookup of "{name}".\n' + f'Lookup chain: {chain}\n' + 'This may be caused by an invalid (cyclical) reference, or buggy code.\n' + 'If you are lazy-loading a file, try a non-lazy load and check for reference cycles.' + ) + + self._lookups_in_progress.append(name) + try: + self._source.stream.seek(self._cells[name].offset) + pat = gdsii.read_elements(self._source.stream, raw_mode=True) + finally: + self._lookups_in_progress.pop() + + if persist: + self._cache[name] = pat + return pat + + def _raw_children(self, name: str) -> set[str]: + return set(self._cells[name].children) + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._cell_order: + if name in self._cache: + graph[name] = _pattern_children(self._cache[name]) + else: + graph[name] = self._raw_children(name) + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + graph = self.child_graph(dangling='ignore') + names = set(graph) + not_toplevel: set[str] = set() + for children in graph.values(): + not_toplevel |= children + return list(names - not_toplevel) + + def with_ports_from_data( + self, + *, + layers: Sequence[tuple[int, int] | int], + max_depth: int = 0, + skip_subcells: bool = True, + ) -> PortsLibraryView: + return PortsLibraryView( + self, + layers=layers, + max_depth=max_depth, + skip_subcells=skip_subcells, + ) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + if parent in self._cache: + for ref in self._cache[parent].refs.get(name, []): + instances[parent].append(ref.as_transforms()) + continue + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def close(self) -> None: + self._source.close() + + def __enter__(self) -> GdsLibrarySource: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + +def read( + stream: IO[bytes], + ) -> tuple[GdsLibrarySource, dict[str, Any]]: + source = _SourceHandle(path=None, stream=stream) + stream.seek(0) + library_info, cell_order, cells = _scan_library(stream) + lib = GdsLibrarySource(source=source, library_info=library_info, cell_order=cell_order, cells=cells) + return lib, library_info + + +def readfile( + filename: str | pathlib.Path, + *, + use_mmap: bool = True, + ) -> tuple[GdsLibrarySource, dict[str, Any]]: + lib = GdsLibrarySource.from_file(filename, use_mmap=use_mmap) + return lib, lib.library_info diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py index 9a03960..f1dcc32 100644 --- a/masque/file/gdsii_lazy_arrow.py +++ b/masque/file/gdsii_lazy_arrow.py @@ -9,8 +9,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import IO, Any, cast from collections import defaultdict -from collections.abc import Callable, Iterator, Mapping, Sequence -import copy +from collections.abc import Iterator, Sequence import gzip import logging import mmap @@ -19,13 +18,12 @@ import pathlib import numpy from numpy.typing import NDArray import pyarrow -import klamath -from . import gdsii, gdsii_arrow -from .utils import is_gzipped, tmpfile -from ..error import LibraryError -from ..library import ILibrary, ILibraryView, Library, LibraryView, dangling_mode_t -from ..pattern import Pattern, map_targets +from . import gdsii_arrow +from .utils import is_gzipped +from .gdsii_lazy_core import OverlayLibrary, PortsLibraryView, _pattern_children, write, writefile +from ..library import ILibraryView, LibraryView, dangling_mode_t +from ..pattern import Pattern from ..utils import apply_transforms @@ -79,22 +77,6 @@ class _ScanPayload: cells: dict[str, _CellScan] refs: _ScanRefs - -@dataclass -class _SourceLayer: - library: ILibraryView - source_to_visible: dict[str, str] - visible_to_source: dict[str, str] - child_graph: dict[str, set[str]] - order: list[str] - - -@dataclass(frozen=True) -class _SourceEntry: - layer_index: int - source_name: str - - def is_available() -> bool: return gdsii_arrow.is_available() @@ -174,30 +156,6 @@ def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: refs=ref_payload, ) - -def _pattern_children(pat: Pattern) -> set[str]: - return {child for child, refs in pat.refs.items() if child is not None and refs} - - -def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: - if not pat.refs: - return pat - pat.refs = map_targets(pat.refs, remap) - return pat - - -def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: - if isinstance(source, ILibraryView): - return source - return LibraryView(source) - - -def _source_order(source: ILibraryView) -> list[str]: - if isinstance(source, ArrowLibrary): - return list(source.source_order()) - return list(source.keys()) - - def _make_ref_rows( xy: NDArray[numpy.integer[Any]], angle_rad: NDArray[numpy.floating[Any]], @@ -285,6 +243,9 @@ class ArrowLibrary(ILibraryView): struct_range = self._payload.cells[name].struct_range return self._source.raw_slice(struct_range.start, struct_range.end) + def can_copy_raw_struct(self, name: str) -> bool: + return name not in self._cache + def materialize_many( self, names: Sequence[str], @@ -435,6 +396,34 @@ class ArrowLibrary(ILibraryView): not_toplevel |= children return list(names - not_toplevel) + def with_ports_from_data( + self, + *, + layers: Sequence[tuple[int, int] | int], + max_depth: int = 0, + skip_subcells: bool = True, + ) -> PortsLibraryView: + return PortsLibraryView( + self, + layers=layers, + max_depth=max_depth, + skip_subcells=skip_subcells, + ) + + def close(self) -> None: + data = self._source.data + if isinstance(data, mmap.mmap): + data.close() + if self._source.handle is not None: + self._source.handle.close() + self._source.handle = None + + def __enter__(self) -> ArrowLibrary: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + def find_refs_local( self, name: str, @@ -517,304 +506,6 @@ class ArrowLibrary(ILibraryView): return result -class OverlayLibrary(ILibrary): - """ - Mutable overlay over one or more source libraries. - - Source-backed cells remain lazy until accessed through `__getitem__`, at - which point that visible cell is promoted into an overlay-owned materialized - `Pattern`. - """ - - def __init__(self) -> None: - self._layers: list[_SourceLayer] = [] - self._entries: dict[str, Pattern | _SourceEntry] = {} - self._order: list[str] = [] - self._target_remap: dict[str, str] = {} - - def __iter__(self) -> Iterator[str]: - return (name for name in self._order if name in self._entries) - - def __len__(self) -> int: - return len(self._entries) - - def __contains__(self, key: object) -> bool: - return key in self._entries - - def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) - - def __setitem__( - self, - key: str, - value: Pattern | Callable[[], Pattern], - ) -> None: - if key in self._entries: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - pattern = value() if callable(value) else value - self._entries[key] = pattern - if key not in self._order: - self._order.append(key) - - def __delitem__(self, key: str) -> None: - if key not in self._entries: - raise KeyError(key) - del self._entries[key] - - def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: - self[key_self] = copy.deepcopy(other[key_other]) - - def add_source( - self, - source: Mapping[str, Pattern] | ILibraryView, - *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, - ) -> dict[str, str]: - view = _coerce_library_view(source) - source_order = _source_order(view) - child_graph = view.child_graph(dangling='include') - - source_to_visible: dict[str, str] = {} - visible_to_source: dict[str, str] = {} - rename_map: dict[str, str] = {} - - for name in source_order: - visible = name - if visible in self._entries or visible in visible_to_source: - if rename_theirs is None: - raise LibraryError(f'Conflicting name while adding source: {name!r}') - visible = rename_theirs(self, name) - if visible in self._entries or visible in visible_to_source: - raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') - rename_map[name] = visible - source_to_visible[name] = visible - visible_to_source[visible] = name - - layer = _SourceLayer( - library=view, - source_to_visible=source_to_visible, - visible_to_source=visible_to_source, - child_graph=child_graph, - order=[source_to_visible[name] for name in source_order], - ) - layer_index = len(self._layers) - self._layers.append(layer) - - for source_name, visible_name in source_to_visible.items(): - self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) - if visible_name not in self._order: - self._order.append(visible_name) - - return rename_map - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> OverlayLibrary: - if old_name not in self._entries: - raise LibraryError(f'"{old_name}" does not exist in the library.') - if old_name == new_name: - return self - if new_name in self._entries: - raise LibraryError(f'"{new_name}" already exists in the library.') - - entry = self._entries.pop(old_name) - self._entries[new_name] = entry - if isinstance(entry, _SourceEntry): - layer = self._layers[entry.layer_index] - layer.source_to_visible[entry.source_name] = new_name - del layer.visible_to_source[old_name] - layer.visible_to_source[new_name] = entry.source_name - - idx = self._order.index(old_name) - self._order[idx] = new_name - - if move_references: - self.move_references(old_name, new_name) - return self - - def _resolve_target(self, target: str) -> str: - seen: set[str] = set() - current = target - while current in self._target_remap: - if current in seen: - raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') - seen.add(current) - current = self._target_remap[current] - return current - - def _set_target_remap(self, old_target: str, new_target: str) -> None: - resolved_new = self._resolve_target(new_target) - if resolved_new == old_target: - raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') - self._target_remap[old_target] = resolved_new - for key in list(self._target_remap): - self._target_remap[key] = self._resolve_target(self._target_remap[key]) - - def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: - if old_target == new_target: - return self - self._set_target_remap(old_target, new_target) - for entry in list(self._entries.values()): - if isinstance(entry, Pattern) and old_target in entry.refs: - entry.refs[new_target].extend(entry.refs[old_target]) - del entry.refs[old_target] - return self - - def _effective_target(self, layer: _SourceLayer, target: str) -> str: - visible = layer.source_to_visible.get(target, target) - return self._resolve_target(visible) - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: - if name not in self._entries: - raise KeyError(name) - entry = self._entries[name] - if isinstance(entry, Pattern): - return entry - - layer = self._layers[entry.layer_index] - source_pat = layer.library[entry.source_name].deepcopy() - remap = lambda target: None if target is None else self._effective_target(layer, target) - pat = _remap_pattern_targets(source_pat, remap) - if persist: - self._entries[name] = pat - return pat - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - graph: dict[str, set[str]] = {} - for name in self._order: - if name not in self._entries: - continue - entry = self._entries[name] - if isinstance(entry, Pattern): - graph[name] = _pattern_children(entry) - continue - layer = self._layers[entry.layer_index] - children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} - graph[name] = children - - existing = set(graph) - dangling_refs = set().union(*(children - existing for children in graph.values())) - if dangling == 'error': - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') - return graph - if dangling == 'ignore': - return {name: {child for child in children if child in existing} for name, children in graph.items()} - - for child in dangling_refs: - graph.setdefault(cast('str', child), set()) - return graph - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - return LibraryView({name: self[name] for name in keep}) - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) - if parent_graph is None: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return instances - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') - if dangling == 'ignore': - return instances - - for parent in parent_graph.get(name, set()): - pat = self._materialize_pattern(parent, persist=False) - for ref in pat.refs.get(name, []): - instances[parent].append(ref.as_transforms()) - return instances - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - result[(parent,) + path] = instances - return result - - def source_order(self) -> tuple[str, ...]: - return tuple(name for name in self._order if name in self._entries) - - def readfile( filename: str | pathlib.Path, ) -> tuple[ArrowLibrary, dict[str, Any]]: @@ -826,135 +517,3 @@ def load_libraryfile( filename: str | pathlib.Path, ) -> tuple[ArrowLibrary, dict[str, Any]]: return readfile(filename) - - -def _get_write_info( - library: Mapping[str, Pattern] | ILibraryView, - *, - meters_per_unit: float | None, - logical_units_per_unit: float | None, - library_name: str | None, - ) -> tuple[float, float, str]: - if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: - return meters_per_unit, logical_units_per_unit, library_name - - infos: list[dict[str, Any]] = [] - if isinstance(library, ArrowLibrary): - infos.append(library.library_info) - elif isinstance(library, OverlayLibrary): - for layer in library._layers: - if isinstance(layer.library, ArrowLibrary): - infos.append(layer.library.library_info) - - if infos: - unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} - if len(unit_pairs) > 1: - raise LibraryError('Merged lazy GDS sources must have identical units before writing') - info = infos[0] - meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit - logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit - name = info['name'] if library_name is None else library_name - return meters, logical, name - - if meters_per_unit is None or logical_units_per_unit is None or library_name is None: - raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') - return meters_per_unit, logical_units_per_unit, library_name - - -def _can_copy_arrow_cell(library: ArrowLibrary, name: str) -> bool: - return name not in library._cache - - -def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: - layer = library._layers[entry.layer_index] - if not isinstance(layer.library, ArrowLibrary): - return False - if name != entry.source_name: - return False - children = layer.child_graph.get(entry.source_name, set()) - return all(library._effective_target(layer, child) == child for child in children) - - -def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: - elements: list[klamath.elements.Element] = [] - elements += gdsii._shapes_to_elements(pat.shapes) - elements += gdsii._labels_to_texts(pat.labels) - elements += gdsii._mrefs_to_grefs(pat.refs) - klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) - - -def write( - library: Mapping[str, Pattern] | ILibraryView, - stream: IO[bytes], - *, - meters_per_unit: float | None = None, - logical_units_per_unit: float | None = None, - library_name: str | None = None, - ) -> None: - meters_per_unit, logical_units_per_unit, library_name = _get_write_info( - library, - meters_per_unit=meters_per_unit, - logical_units_per_unit=logical_units_per_unit, - library_name=library_name, - ) - - header = klamath.library.FileHeader( - name=library_name.encode('ASCII'), - user_units_per_db_unit=logical_units_per_unit, - meters_per_db_unit=meters_per_unit, - ) - header.write(stream) - - if isinstance(library, ArrowLibrary): - for name in library.source_order(): - if _can_copy_arrow_cell(library, name): - stream.write(library.raw_struct_bytes(name)) - else: - _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) - klamath.records.ENDLIB.write(stream, None) - return - - if isinstance(library, OverlayLibrary): - for name in library.source_order(): - entry = library._entries[name] - if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): - layer = library._layers[entry.layer_index] - assert isinstance(layer.library, ArrowLibrary) - stream.write(layer.library.raw_struct_bytes(entry.source_name)) - else: - _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) - klamath.records.ENDLIB.write(stream, None) - return - - gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) - - -def writefile( - library: Mapping[str, Pattern] | ILibraryView, - filename: str | pathlib.Path, - *, - meters_per_unit: float | None = None, - logical_units_per_unit: float | None = None, - library_name: str | None = None, - ) -> None: - path = pathlib.Path(filename) - - with tmpfile(path) as base_stream: - streams: tuple[Any, ...] = (base_stream,) - if path.suffix == '.gz': - stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) - streams = (stream,) + streams - else: - stream = base_stream - - try: - write( - library, - stream, - meters_per_unit=meters_per_unit, - logical_units_per_unit=logical_units_per_unit, - library_name=library_name, - ) - finally: - for ss in streams: - ss.close() diff --git a/masque/file/gdsii_lazy_core.py b/masque/file/gdsii_lazy_core.py new file mode 100644 index 0000000..eaaf863 --- /dev/null +++ b/masque/file/gdsii_lazy_core.py @@ -0,0 +1,665 @@ +""" +Shared helpers for source-backed lazy GDS views. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping, Sequence +import copy +import gzip +import logging +import pathlib + +import klamath +import numpy +from numpy.typing import NDArray + +from . import gdsii +from .utils import tmpfile +from ..error import LibraryError +from ..library import ILibrary, ILibraryView, LibraryView, dangling_mode_t +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms +from ..utils.ports2data import data_to_ports + + +logger = logging.getLogger(__name__) + + +@dataclass +class _SourceLayer: + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + layer_index: int + source_name: str + + +def _pattern_children(pat: Pattern) -> set[str]: + return {child for child, refs in pat.refs.items() if child is not None and refs} + + +def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: + if not pat.refs: + return pat + pat.refs = map_targets(pat.refs, remap) + return pat + + +def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: + if isinstance(source, ILibraryView): + return source + return LibraryView(source) + + +def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: + func = getattr(view, '_materialize_pattern', None) + if callable(func): + return cast('Pattern', func(name, persist=False)) + return view[name].deepcopy() + + +class PortsLibraryView(ILibraryView): + """ + Read-only view which imports ports into cells on first materialization. + + The wrapped source remains untouched; this view owns a separate processed + cache so direct-copy workflows can continue to use the raw source view. + """ + + def __init__( + self, + source: ILibraryView, + *, + layers: Sequence[gdsii.layer_t], + max_depth: int = 0, + skip_subcells: bool = True, + ) -> None: + self._source = source + self._layers = tuple(layers) + self._max_depth = max_depth + self._skip_subcells = skip_subcells + self._cache: dict[str, Pattern] = {} + self._lookups_in_progress: list[str] = [] + if hasattr(source, 'library_info'): + self.library_info = cast('dict[str, Any]', getattr(source, 'library_info')) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._source) + + def __len__(self) -> int: + return len(self._source) + + def __contains__(self, key: object) -> bool: + return key in self._source + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name in self._cache: + return self._cache[name] + + if name in self._lookups_in_progress: + chain = ' -> '.join(self._lookups_in_progress + [name]) + raise LibraryError( + f'Detected circular reference or recursive lookup of "{name}".\n' + f'Lookup chain: {chain}\n' + 'This may be caused by an invalid (cyclical) reference, or buggy code.' + ) + + self._lookups_in_progress.append(name) + try: + pat = _materialize_detached_pattern(self._source, name) + pat = data_to_ports( + layers=self._layers, + library=self, + pattern=pat, + name=name, + max_depth=self._max_depth, + skip_subcells=self._skip_subcells, + ) + finally: + self._lookups_in_progress.pop() + + if persist: + self._cache[name] = pat + return pat + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = { + name: self._materialize_pattern(name, persist=persist) + for name in dict.fromkeys(names) + } + return LibraryView(mats) + + def source_order(self) -> tuple[str, ...]: + return self._source.source_order() + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.child_graph(dangling=dangling) + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.parent_graph(dangling=dangling) + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + return self._source.tops() + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + finder = getattr(self._source, 'find_refs_local', None) + if callable(finder): + return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) + return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + finder = getattr(self._source, 'find_refs_global', None) + if callable(finder): + return cast( + 'dict[tuple[str, ...], NDArray[numpy.float64]]', + finder(name, order=order, parent_graph=parent_graph, dangling=dangling), + ) + return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) + + def raw_struct_bytes(self, name: str) -> bytes: + reader = getattr(self._source, 'raw_struct_bytes', None) + if not callable(reader): + raise AttributeError('raw_struct_bytes') + return cast('bytes', reader(name)) + + def can_copy_raw_struct(self, name: str) -> bool: + can_copy = getattr(self._source, 'can_copy_raw_struct', None) + if not callable(can_copy): + return False + return bool(can_copy(name)) + + def close(self) -> None: + closer = getattr(self._source, 'close', None) + if callable(closer): + closer() + + def __enter__(self) -> PortsLibraryView: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + view = _coerce_library_view(source) + source_order = list(view.source_order()) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + + for name in source_order: + visible = name + if visible in self._entries or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._entries or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return rename_map + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + remap = lambda target: None if target is None else self._effective_target(layer, target) + pat = _remap_pattern_targets(source_pat, remap) + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = _pattern_children(entry) + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + +def _iter_library_infos(library: Mapping[str, Pattern] | ILibraryView) -> Iterator[dict[str, Any]]: + info = getattr(library, 'library_info', None) + if isinstance(info, dict): + yield info + if isinstance(library, OverlayLibrary): + for layer in library._layers: + yield from _iter_library_infos(layer.library) + + +def _get_write_info( + library: Mapping[str, Pattern] | ILibraryView, + *, + meters_per_unit: float | None, + logical_units_per_unit: float | None, + library_name: str | None, + ) -> tuple[float, float, str]: + if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: + return meters_per_unit, logical_units_per_unit, library_name + + infos = list(_iter_library_infos(library)) + if infos: + unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} + if len(unit_pairs) > 1: + raise LibraryError('Merged lazy GDS sources must have identical units before writing') + info = infos[0] + meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit + logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit + name = info['name'] if library_name is None else library_name + return meters, logical, name + + if meters_per_unit is None or logical_units_per_unit is None or library_name is None: + raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') + return meters_per_unit, logical_units_per_unit, library_name + + +def _can_copy_raw_cell(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bool: + can_copy = getattr(library, 'can_copy_raw_struct', None) + if not callable(can_copy): + return False + return bool(can_copy(name)) + + +def _raw_struct_bytes(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bytes: + reader = getattr(library, 'raw_struct_bytes', None) + if not callable(reader): + raise AttributeError('raw_struct_bytes') + return cast('bytes', reader(name)) + + +def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: + layer = library._layers[entry.layer_index] + if name != entry.source_name: + return False + if not _can_copy_raw_cell(layer.library, entry.source_name): + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(library._effective_target(layer, child) == child for child in children) + + +def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: + elements: list[klamath.elements.Element] = [] + elements += gdsii._shapes_to_elements(pat.shapes) + elements += gdsii._labels_to_texts(pat.labels) + elements += gdsii._mrefs_to_grefs(pat.refs) + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) + + +def write( + library: Mapping[str, Pattern] | ILibraryView, + stream: IO[bytes], + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + meters_per_unit, logical_units_per_unit, library_name = _get_write_info( + library, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + + header = klamath.library.FileHeader( + name=library_name.encode('ASCII'), + user_units_per_db_unit=logical_units_per_unit, + meters_per_db_unit=meters_per_unit, + ) + header.write(stream) + + if isinstance(library, OverlayLibrary): + for name in library.source_order(): + entry = library._entries[name] + if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + layer = library._layers[entry.layer_index] + stream.write(_raw_struct_bytes(layer.library, entry.source_name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + if hasattr(library, 'raw_struct_bytes'): + for name in library.source_order(): + if _can_copy_raw_cell(library, name): + stream.write(_raw_struct_bytes(library, name)) + else: + _write_pattern_struct(stream, name, _materialize_detached_pattern(cast('ILibraryView', library), name)) + klamath.records.ENDLIB.write(stream, None) + return + + gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) + + +def writefile( + library: Mapping[str, Pattern] | ILibraryView, + filename: str | pathlib.Path, + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + path = pathlib.Path(filename) + + with tmpfile(path) as base_stream: + streams: tuple[Any, ...] = (base_stream,) + if path.suffix == '.gz': + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + streams = (stream,) + streams + else: + stream = base_stream + + try: + write( + library, + stream, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + finally: + for ss in streams: + ss.close() diff --git a/masque/library.py b/masque/library.py index e98d98d..bc15969 100644 --- a/masque/library.py +++ b/masque/library.py @@ -131,6 +131,15 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): """ return Abstract(name=name, ports=self[name].ports) + def source_order(self) -> tuple[str, ...]: + """ + Return names in the library's preferred source order. + + Source-backed views may override this to preserve on-disk ordering + without materializing patterns. + """ + return tuple(self.keys()) + def dangling_refs( self, tops: str | Sequence[str] | None = None, diff --git a/masque/test/test_gdsii_lazy.py b/masque/test/test_gdsii_lazy.py new file mode 100644 index 0000000..6855783 --- /dev/null +++ b/masque/test/test_gdsii_lazy.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import numpy +from numpy.testing import assert_allclose + +from ..file import gdsii, gdsii_lazy +from ..pattern import Pattern +from ..library import Library + + +def _make_lazy_port_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.label(layer=(10, 0), string='A:type1 0', offset=(5, 0)) + lib['leaf'] = leaf + + child = Pattern() + child.ref('leaf', offset=(10, 20), rotation=numpy.pi / 2) + lib['child'] = child + + top = Pattern() + top.ref('child', offset=(100, 200)) + lib['top'] = top + + return lib + + +def test_gdsii_lazy_source_exposes_order_and_graph_without_materializing(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_source.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-lazy') + + lib, info = gdsii_lazy.readfile(gds_file) + + assert info['name'] == 'classic-lazy' + assert lib.source_order() == ('leaf', 'child', 'top') + assert lib.child_graph(dangling='ignore') == { + 'leaf': set(), + 'child': {'leaf'}, + 'top': {'child'}, + } + assert not lib._cache + + child = lib['child'] + assert list(child.refs.keys()) == ['leaf'] + assert set(lib._cache) == {'child'} + + +def test_gdsii_lazy_ports_view_keeps_raw_source_unmodified(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_ports.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-ports') + + raw, _ = gdsii_lazy.readfile(gds_file) + processed = raw.with_ports_from_data(layers=[(10, 0)], max_depth=2) + + top = processed['top'] + assert set(top.ports) == {'A'} + assert_allclose(top.ports['A'].offset, [110, 225], atol=1e-10) + assert not raw._cache + + raw_top = raw['top'] + assert not raw_top.ports + + +def test_gdsii_lazy_overlay_add_source_stays_lazy_for_processed_view(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_overlay.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-overlay') + + raw, _ = gdsii_lazy.readfile(gds_file) + processed = raw.with_ports_from_data(layers=[(10, 0)], max_depth=2) + + overlay = gdsii_lazy.OverlayLibrary() + overlay.add_source(processed) + + assert not raw._cache + assert not processed._cache + + abstract = overlay.abstract('top') + assert set(abstract.ports) == {'A'} + + +def test_gdsii_lazy_processed_write_roundtrips_without_explicit_units(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_roundtrip.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-roundtrip') + + raw, _ = gdsii_lazy.readfile(gds_file) + processed = raw.with_ports_from_data(layers=[(10, 0)], max_depth=2) + + out_file = tmp_path / 'lazy_roundtrip_out.gds' + gdsii_lazy.writefile(processed, out_file) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_removed_closure_based_lazy_loader() -> None: + assert not hasattr(gdsii, 'load_library') + assert not hasattr(gdsii, 'load_libraryfile') From 80a24539f6f7910fc8545b76f5b4e8e3f2fff3cc Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Wed, 22 Apr 2026 21:24:05 -0700 Subject: [PATCH 36/36] [wip] introduce BuildLibrary and make overlay first-class --- examples/tutorial/README.md | 6 +- examples/tutorial/library.py | 156 +++--- masque/__init__.py | 5 + masque/file/gdsii_lazy.py | 23 +- masque/file/gdsii_lazy_core.py | 41 ++ masque/library.py | 887 +++++++++++++++++++++++++++++- masque/test/test_build_library.py | 315 +++++++++++ 7 files changed, 1335 insertions(+), 98 deletions(-) create mode 100644 masque/test/test_build_library.py diff --git a/examples/tutorial/README.md b/examples/tutorial/README.md index 749915b..f5c4392 100644 --- a/examples/tutorial/README.md +++ b/examples/tutorial/README.md @@ -20,10 +20,10 @@ Contents * Use `Pather` to snap ports together into a circuit * Check for dangling references - [library](library.py) - * Continue from `devices.py` using a lazy library - * Create a `LazyLibrary`, which loads / generates patterns only when they are first used + * Continue from `devices.py` by declaring a mixed library with `BuildLibrary` + * Import source-backed GDS cells and register python-generated recipes together + * Call `build()` to produce a normal library for downstream `Pather` usage and writing * Explore alternate ways of specifying a pattern for `.plug()` and `.place()` - * Design a pattern which is meant to plug into an existing pattern (via `.interface()`) - [pather](pather.py) * Use `Pather` to route individual wires and wire bundles * Use `AutoTool` to generate paths diff --git a/examples/tutorial/library.py b/examples/tutorial/library.py index f6bcc4e..f4eb3f0 100644 --- a/examples/tutorial/library.py +++ b/examples/tutorial/library.py @@ -1,136 +1,114 @@ """ -Tutorial: using a source-backed lazy GDS library and `Pather.interface()`. +Tutorial: authoring a mixed library with `BuildLibrary`. This example assumes you have already read `devices.py` and generated the `circuit.gds` file it writes. The goal here is not the photonic-crystal geometry -itself, but rather how Masque lets you mix lazily loaded GDS content with -python-generated devices inside one library. +itself, but rather how Masque lets you combine imported GDS cells with +python-generated recipes, then turn that declaration set into a normal library +for downstream assembly and writing. """ from typing import Any from pprint import pformat -from masque import Pather +from masque import BuildLibrary, Pather, Pattern, cell from masque.file.gdsii import writefile -from masque.file.gdsii_lazy import OverlayLibrary, readfile +from masque.file.gdsii_lazy import readfile import basic_shapes import devices from basic_shapes import GDS_OPTS +def make_mixed_waveguide(lib: BuildLibrary) -> Pattern: + """ + Recipe which assembles imported and generated cells behind the builder API. + """ + circ = Pather(library=lib, ports='tri_l3cav') + + # First way to specify what we are plugging in: request an explicit abstract. + circ.plug(lib.abstract('wg10'), {'input': 'right'}) + + # Second way: use an AbstractView, which behaves like a mapping of names + # to abstracts. + abstracts = lib.abstract_view() + circ.plug(abstracts['wg10'], {'output': 'left'}) + + # Third way: let Pather resolve a pattern name through its own library. + circ.plug('tri_wg10', {'input': 'right'}) + circ.plug('tri_wg10', {'output': 'left'}) + + return circ.pattern + + def main() -> None: - # `OverlayLibrary` lets us mix source-backed GDS cells with python-generated - # patterns behind the same library interface. - lib = OverlayLibrary() + builder = BuildLibrary() + cells = builder.cells # # Load some devices from a GDS file # # Scan circuit.gds and prepare to lazy-load its contents. Port labels are - # imported on first materialization, but the raw source remains untouched. + # imported on first materialization, but the raw source remains untouched + # until we build the final library. gds_lib, _properties = readfile('circuit.gds') - lib.add_source(gds_lib.with_ports_from_data(layers=[(3, 0)], max_depth=1)) + builder.add_source(gds_lib.with_ports_from_data(layers=[(3, 0)], max_depth=1)) - print('Patterns loaded from GDS into library:\n' + pformat(list(lib.keys()))) + print('Registered imported cells:\n' + pformat(list(gds_lib.keys()))) # - # Add some new devices to the library, this time from python code rather than GDS + # Register some new devices, this time from python code rather than GDS. # - lib['triangle'] = basic_shapes.triangle(devices.RADIUS) + cells.triangle = basic_shapes.triangle(devices.RADIUS) opts: dict[str, Any] = dict( - lattice_constant = devices.LATTICE_CONSTANT, - hole = 'triangle', - ) + lattice_constant=devices.LATTICE_CONSTANT, + hole='triangle', + ) - lib['tri_wg10'] = devices.waveguide(length=10, mirror_periods=5, **opts) - lib['tri_wg05'] = devices.waveguide(length=5, mirror_periods=5, **opts) - lib['tri_wg28'] = devices.waveguide(length=28, mirror_periods=5, **opts) - lib['tri_bend0'] = devices.bend(mirror_periods=5, **opts) - lib['tri_ysplit'] = devices.y_splitter(mirror_periods=5, **opts) - lib['tri_l3cav'] = devices.perturbed_l3(xy_size=(4, 10), **opts, hole_lib=lib) + cells.tri_wg10 = cell(devices.waveguide)(length=10, mirror_periods=5, **opts) + cells.tri_wg05 = cell(devices.waveguide)(length=5, mirror_periods=5, **opts) + cells.tri_wg28 = cell(devices.waveguide)(length=28, mirror_periods=5, **opts) + cells.tri_bend0 = cell(devices.bend)(mirror_periods=5, **opts) + cells.tri_ysplit = cell(devices.y_splitter)(mirror_periods=5, **opts) + cells.tri_l3cav = cell(devices.perturbed_l3)(xy_size=(4, 10), **opts, hole_lib=builder) + cells.mixed_wg_cav = cell(make_mixed_waveguide)(builder) + + print('Declared cells waiting to be built:\n' + pformat(list(builder.keys()))) # - # Build a mixed waveguide with an L3 cavity in the middle + # Build the declaration set into a normal library. # - # Start a new design by copying the ports from an existing library cell. - # This gives `circ2` the same external interface as `tri_l3cav`. - circ2 = Pather(library=lib, ports='tri_l3cav') - - # First way to specify what we are plugging in: request an explicit abstract. - # This works with `Pattern` methods directly as well as with `Pather`. - circ2.plug(lib.abstract('wg10'), {'input': 'right'}) - - # Second way: use an `AbstractView`, which behaves like a mapping of names - # to abstracts. - abstracts = lib.abstract_view() - circ2.plug(abstracts['wg10'], {'output': 'left'}) - - # Third way: let `Pather` resolve a pattern name through its own library. - # This shorthand is convenient, but it is specific to helpers that already - # carry a library reference. - circ2.plug('tri_wg10', {'input': 'right'}) - circ2.plug('tri_wg10', {'output': 'left'}) - - # Add the circuit to the device library. - lib['mixed_wg_cav'] = circ2.pattern - + built = builder.build() + print('Built library contains:\n' + pformat(list(built.keys()))) # - # Build a second device that is explicitly designed to mate with `circ2`. + # Continue designing against the built library. # - # `Pather.interface()` makes a new pattern whose ports mirror an existing - # design's external interface. That is useful when you want to design an - # adapter, continuation, or mating structure. - circ3 = Pather.interface(source=circ2) - - # Continue routing outward from those inherited ports. - circ3.plug('tri_bend0', {'input': 'right'}) - circ3.plug('tri_bend0', {'input': 'left'}, mirrored=True) # mirror since no tri y-symmetry - circ3.plug('tri_bend0', {'input': 'right'}) - circ3.plug('bend0', {'output': 'left'}) - circ3.plug('bend0', {'output': 'left'}) - circ3.plug('bend0', {'output': 'left'}) - circ3.plug('tri_wg10', {'input': 'right'}) - circ3.plug('tri_wg28', {'input': 'right'}) - circ3.plug('tri_wg10', {'input': 'right', 'output': 'left'}) - - lib['loop_segment'] = circ3.pattern + # The built result behaves like a normal mutable library, so downstream code + # can use Pather, abstract views, and writing without going back through the + # builder interface. + circ = Pather.interface(source='mixed_wg_cav', library=built) + circ.plug('tri_bend0', {'input': 'right'}) + circ.plug('tri_bend0', {'input': 'left'}, mirrored=True) # mirror since no tri y-symmetry + circ.plug('tri_bend0', {'input': 'right'}) + circ.plug('bend0', {'output': 'left'}) + circ.plug('bend0', {'output': 'left'}) + circ.plug('bend0', {'output': 'left'}) + circ.plug('tri_wg10', {'input': 'right'}) + circ.plug('tri_wg28', {'input': 'right'}) + circ.plug('tri_wg10', {'input': 'right', 'output': 'left'}) + built['loop_segment'] = circ.pattern # - # Write all devices into a GDS file + # Write all devices into a GDS file. # print('Writing library to file...') - writefile(lib, 'library.gds', **GDS_OPTS) + writefile(built, 'library.gds', **GDS_OPTS) if __name__ == '__main__': main() - - -# -#class prout: -# def place( -# self, -# other: Pattern, -# label_layer: layer_t = 'WATLAYER', -# *, -# port_map: Dict[str, str | None] | None = None, -# **kwargs, -# ) -> 'prout': -# -# Pattern.place(self, other, port_map=port_map, **kwargs) -# name: str | None -# for name in other.ports: -# if port_map: -# assert(name is not None) -# name = port_map.get(name, name) -# if name is None: -# continue -# self.pattern.label(string=name, offset=self.ports[name].offset, layer=label_layer) -# return self -# diff --git a/masque/__init__.py b/masque/__init__.py index b6cc5e9..b6dfb44 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -63,10 +63,15 @@ from .library import ( ILibrary as ILibrary, LibraryView as LibraryView, Library as Library, + BuiltLibrary as BuiltLibrary, + BuildLibrary as BuildLibrary, + BuildReport as BuildReport, + CellProvenance as CellProvenance, LazyLibrary as LazyLibrary, AbstractView as AbstractView, TreeView as TreeView, Tree as Tree, + cell as cell, ) from .ports import ( Port as Port, diff --git a/masque/file/gdsii_lazy.py b/masque/file/gdsii_lazy.py index f9561af..b587c89 100644 --- a/masque/file/gdsii_lazy.py +++ b/masque/file/gdsii_lazy.py @@ -1,9 +1,18 @@ """ -Source-backed lazy GDSII reader using the pure-python klamath path. +Classic source-backed lazy GDSII reader built on the pure-python klamath path. -This module mirrors the lazy Arrow reader's interface closely enough to share -the same overlay and ports-import helpers, while still materializing cells -through the classic `gdsii` decoder. +This module provides the non-Arrow half of Masque's lazy GDS architecture: + +- `GdsLibrarySource` scans a GDS stream once to discover library metadata, + struct order, and child edges without materializing every cell. +- cells are materialized on demand through the classic `gdsii` decoder + whenever a caller indexes the lazy view +- the source can be wrapped in `PortsLibraryView` or merged through + `OverlayLibrary`, both of which live in `gdsii_lazy_core` + +The public surface intentionally parallels `gdsii_lazy_arrow` closely so that +callers can swap between the classic and Arrow-backed implementations with +minimal changes. """ from __future__ import annotations @@ -36,6 +45,7 @@ logger = logging.getLogger(__name__) @dataclass class _SourceHandle: + """ Owns the underlying stream and any companion file handle for a source. """ path: pathlib.Path | None stream: IO[bytes] handle: IO[bytes] | None = None @@ -49,6 +59,7 @@ class _SourceHandle: @dataclass(frozen=True) class _CellScan: + """ Scan-time metadata for one cell in the source stream. """ offset: int children: set[str] @@ -107,6 +118,10 @@ class GdsLibrarySource(ILibraryView): Cells are scanned once up front to discover order and child edges, then materialized one at a time through the classic `gdsii.read_elements` path. + + The source owns the stream lifetime, preserves on-disk ordering through + `source_order()`, and answers graph queries from scan metadata whenever + possible so callers can inspect hierarchy without forcing a full load. """ def __init__( diff --git a/masque/file/gdsii_lazy_core.py b/masque/file/gdsii_lazy_core.py index eaaf863..d730348 100644 --- a/masque/file/gdsii_lazy_core.py +++ b/masque/file/gdsii_lazy_core.py @@ -1,5 +1,18 @@ """ Shared helpers for source-backed lazy GDS views. + +This module contains the reusable pieces that sit between lazy source readers +and ordinary mutable library usage: + +- `PortsLibraryView` layers a processed, ports-importing cache on top of a raw + source view without mutating the source itself +- `OverlayLibrary` exposes a mutable library surface that can mix source-backed + cells with overlay-owned materialized patterns +- the write helpers preserve source-backed copy-through behavior where + possible, falling back to normal pattern serialization when a cell has been + materialized or remapped + +Both the classic and Arrow-backed lazy GDS readers rely on these helpers. """ from __future__ import annotations @@ -30,6 +43,7 @@ logger = logging.getLogger(__name__) @dataclass class _SourceLayer: + """ One imported source layer tracked by an `OverlayLibrary`. """ library: ILibraryView source_to_visible: dict[str, str] visible_to_source: dict[str, str] @@ -39,6 +53,7 @@ class _SourceLayer: @dataclass(frozen=True) class _SourceEntry: + """ Reference to a single visible source-backed cell in an overlay. """ layer_index: int source_name: str @@ -73,6 +88,10 @@ class PortsLibraryView(ILibraryView): The wrapped source remains untouched; this view owns a separate processed cache so direct-copy workflows can continue to use the raw source view. + + Graph queries, source ordering, and copy-through capabilities are delegated + to the wrapped source whenever possible, while `__getitem__` and + `materialize_many()` return port-imported patterns. """ def __init__( @@ -231,6 +250,14 @@ class OverlayLibrary(ILibrary): Source-backed cells remain lazy until accessed through `__getitem__`, at which point that visible cell is promoted into an overlay-owned materialized `Pattern`. + + This is the main mutable integration surface for lazy GDS content. It lets + callers: + - expose one or more source-backed libraries behind a normal `ILibrary` + interface + - add or replace cells with overlay-owned patterns + - rename visible source cells + - remap references without immediately rewriting untouched source structs """ def __init__(self) -> None: @@ -522,6 +549,20 @@ class OverlayLibrary(ILibrary): return tuple(name for name in self._order if name in self._entries) +class BuiltOverlayLibrary(OverlayLibrary): + """ + Internal overlay output returned by `BuildLibrary.build(output='overlay')`. + + The type is intentionally not part of the public API. It exists so build + outputs can carry a `build_report` while still behaving like an + `OverlayLibrary`. + """ + + def __init__(self, *, build_report: Any | None = None) -> None: + super().__init__() + self.build_report = build_report + + def _iter_library_infos(library: Mapping[str, Pattern] | ILibraryView) -> Iterator[dict[str, Any]]: info = getattr(library, 'library_info', None) if isinstance(info, dict): diff --git a/masque/library.py b/masque/library.py index bc15969..942113a 100644 --- a/masque/library.py +++ b/masque/library.py @@ -14,7 +14,7 @@ Classes include: - `AbstractView`: Provides a way to use []-indexing to generate abstracts for patterns in the linked library. Generated with `ILibraryView.abstract_view()`. """ -from typing import Self, TYPE_CHECKING, cast, TypeAlias, Protocol, Literal +from typing import Self, TYPE_CHECKING, Any, cast, TypeAlias, Protocol, Literal from collections.abc import Iterator, Mapping, MutableMapping, Sequence, Callable import logging import re @@ -22,12 +22,14 @@ import copy from pprint import pformat from collections import defaultdict from abc import ABCMeta, abstractmethod +from contextvars import ContextVar +from dataclasses import dataclass, replace from graphlib import TopologicalSorter, CycleError import numpy from numpy.typing import ArrayLike, NDArray -from .error import LibraryError, PatternError +from .error import BuildError, LibraryError, PatternError from .utils import layer_t, apply_transforms from .shapes import Shape, Polygon from .label import Label @@ -40,6 +42,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_ACTIVE_BUILD_SESSIONS: ContextVar[dict[int, '_BuildSessionLibrary'] | None] = ContextVar( + 'masque_active_build_sessions', + default=None, +) + class visitor_function_t(Protocol): """ Signature for `Library.dfs()` visitor functions. """ @@ -62,6 +69,69 @@ Tree: TypeAlias = MutableMapping[str, 'Pattern'] dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include'] """ How helpers should handle refs whose targets are not present in the library. """ +emitted_via_t: TypeAlias = Literal['declaration', 'helper_write', 'tree_merge', 'source_import'] +""" Build-provenance origin tags for emitted cells. """ + + +@dataclass(frozen=True) +class CellProvenance: + """ + Provenance record for one cell in a completed build output. + + Each output name in a `BuildReport` maps to one `CellProvenance`. The + record captures both where the cell came from and how its visible name was + chosen. + + Attributes: + final_name: Name exposed by the completed library. + requested_name: First name requested for this cell during the build. + kind: Whether the cell came from a declaration, helper emission, or an + imported source library. + owner_declared_name: Declared cell responsible for this output cell, if + any. Imported source cells leave this as `None`. + emitted_via: High-level path by which the cell entered the output. + build_chain: Declared-cell dependency chain that was active when the + cell was emitted. + renamed_from: Original requested name when the final name differs. + source_name: Original on-source name for imported cells. + source_metadata: Optional source-library metadata copied through from + lazy GDS readers. + """ + final_name: str + requested_name: str + kind: Literal['declared', 'helper', 'source'] + owner_declared_name: str | None + emitted_via: emitted_via_t + build_chain: tuple[str, ...] + renamed_from: str | None = None + source_name: str | None = None + source_metadata: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class BuildReport: + """ + Immutable summary of one `BuildLibrary.validate()` or `.build()` run. + + The report is designed to answer two questions after a build completes: + which declared cells depended on which other declared cells, and where each + output cell came from. + + Attributes: + requested_roots: Roots explicitly requested for the run. A full + `build()` uses all declared cells. + provenance: Mapping from final output name to provenance metadata. + owned_cells: Mapping from declared cell name to all final output cell + names it owns, including helper cells emitted while that declared + cell was building. + dependency_graph: Declared-cell dependency graph discovered through + library-mediated reads and explicit recipe hints. + """ + requested_roots: tuple[str, ...] + provenance: Mapping[str, CellProvenance] + owned_cells: Mapping[str, tuple[str, ...]] + dependency_graph: Mapping[str, frozenset[str]] + SINGLE_USE_PREFIX = '_' """ @@ -1397,6 +1467,819 @@ class Library(ILibrary): return tree, pat +class BuiltLibrary(Library): + """ + Eager library returned by `BuildLibrary.build(output='library')`. + + This is a normal materialized `Library` with one additional attribute, + `build_report`, which records how the library was assembled from + declarations, helper emissions, and imported source-backed cells. + """ + + def __init__( + self, + mapping: MutableMapping[str, 'Pattern'] | None = None, + *, + build_report: BuildReport | None = None, + ) -> None: + super().__init__(mapping=mapping) + self.build_report = build_report + + +class _CellFactory: + """ + Adapter that turns a plain pattern factory into a deferred recipe factory. + + Calling the wrapper captures arguments and returns a `_BuildRecipe` + instead of executing the function immediately. + """ + + def __init__(self, func: Callable[..., 'Pattern']) -> None: + self.func = func + self.__name__ = getattr(func, '__name__', type(self).__name__) + self.__doc__ = getattr(func, '__doc__') + + def __call__(self, *args: Any, **kwargs: Any) -> '_BuildRecipe': + return _BuildRecipe(func=self.func, args=args, kwargs=kwargs) + + +@dataclass +class _BuildRecipe: + """ Captured deferred call to a pattern factory. """ + func: Callable[..., 'Pattern'] + args: tuple[Any, ...] + kwargs: dict[str, Any] + explicit_dependencies: tuple[str, ...] = () + + def depends_on(self, *names: str) -> '_BuildRecipe': + self.explicit_dependencies += tuple(names) + return self + + +@dataclass(frozen=True) +class _PatternDeclaration: + """ Declared cell backed by an already-built `Pattern`. """ + pattern: 'Pattern' + + +@dataclass(frozen=True) +class _RecipeDeclaration: + """ Declared cell backed by a deferred recipe. """ + recipe: _BuildRecipe + + +@dataclass(frozen=True) +class _SourceDeclaration: + """ + Imported source-backed names registered with a `BuildLibrary`. + + The declaration stores visible-name remapping plus pre-scanned graph + metadata. Underlying source cells stay lazy until a build session + materializes or copies them through. + """ + library: ILibraryView + source_to_visible: Mapping[str, str] + visible_to_source: Mapping[str, str] + child_graph: Mapping[str, set[str]] + order: tuple[str, ...] + + +def cell(func: Callable[..., 'Pattern']) -> _CellFactory: + """ + Wrap a plain pattern factory so calls return deferred build recipes. + + Use as either `cell(fn)(...)` or `@cell`. + """ + return _CellFactory(func) + + +class BuildCellsView: + """ + Attribute-based declaration namespace for `BuildLibrary`. + + This is the ergonomic authoring surface exposed as `builder.cells`. It is + intentionally write-focused: attribute assignment and deletion register + declarations, while attribute reads fail with guidance to build first and + use the returned library. + """ + + def __init__(self, library: 'BuildLibrary') -> None: + object.__setattr__(self, '_library', library) + + def __getattr__(self, name: str) -> 'Pattern': + raise BuildError( + f'BuildLibrary.cells.{name} is write-only during authoring. ' + 'Call build() and index the returned library instead.' + ) + + def __setattr__(self, name: str, value: 'Pattern | _BuildRecipe') -> None: + if name.startswith('_'): + object.__setattr__(self, name, value) + return + self._library[name] = value + + def __delattr__(self, name: str) -> None: + if name.startswith('_'): + raise AttributeError(name) + del self._library[name] + + +class BuildLibrary(ILibrary): + """ + Two-phase declaration surface for mixed imported/generated libraries. + + A `BuildLibrary` collects three kinds of inputs: + - direct declared `Pattern` objects + - deferred recipes created with `cell(...)` + - imported source-backed library views added with `add_source(...)` + + The builder itself is not a normal readable library during authoring. + Instead, `validate()` and `build()` create a temporary build-session library + that recipes can read from and write helper cells into while dependencies + are resolved. `build()` then freezes the builder on success and returns a + normal library-like object carrying a `build_report`. + """ + + def __init__(self, *, check_on_register: bool = False) -> None: + self.check_on_register = check_on_register + self.cells = BuildCellsView(self) + self.last_build_report: BuildReport | None = None + self._frozen = False + self._declarations: dict[str, _PatternDeclaration | _RecipeDeclaration] = {} + self._sources: list[_SourceDeclaration] = [] + self._names: set[str] = set() + self._order: list[str] = [] + + def _active_session(self) -> '_BuildSessionLibrary | None': + sessions = _ACTIVE_BUILD_SESSIONS.get() + if sessions is None: + return None + return sessions.get(id(self)) + + def _require_active_session(self, operation: str) -> '_BuildSessionLibrary': + session = self._active_session() + if session is None: + raise BuildError( + f'BuildLibrary.{operation}() is only available while validate() or build() is running. ' + 'Use the built output library for reads.' + ) + return session + + def _assert_editable(self) -> None: + if self._frozen: + raise BuildError('This BuildLibrary has already been built successfully and is now frozen.') + + def __iter__(self) -> Iterator[str]: + session = self._active_session() + if session is not None: + return iter(session) + return iter(self._order) + + def __len__(self) -> int: + session = self._active_session() + if session is not None: + return len(session) + return len(self._names) + + def __contains__(self, key: object) -> bool: + session = self._active_session() + if session is not None: + return key in session + return key in self._names + + def __getitem__(self, key: str) -> 'Pattern': + return self._require_active_session('__getitem__')[key] + + def __setitem__( + self, + key: str, + value: 'Pattern | _BuildRecipe | Callable[[], Pattern]', + ) -> None: + session = self._active_session() + if session is not None: + session[key] = value + return + + self._assert_editable() + if key in self._names: + raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!') + + declaration: _PatternDeclaration | _RecipeDeclaration + if isinstance(value, _BuildRecipe): + declaration = _RecipeDeclaration(value) + else: + if callable(value): + raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.') + declaration = _PatternDeclaration(value) + + self._declarations[key] = declaration + self._names.add(key) + self._order.append(key) + + if self.check_on_register: + try: + self.validate(names=(key,)) + except Exception: + del self._declarations[key] + self._names.remove(key) + self._order.remove(key) + raise + + def __delitem__(self, key: str) -> None: + session = self._active_session() + if session is not None: + del session[key] + return + + self._assert_editable() + if key not in self._declarations: + raise KeyError(key) + del self._declarations[key] + self._names.remove(key) + self._order.remove(key) + + def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: + session = self._active_session() + if session is not None: + session._merge(key_self, other, key_other) + return + self[key_self] = copy.deepcopy(other[key_other]) + + def add( + self, + other: Mapping[str, 'Pattern'], + rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns, + mutate_other: bool = False, + ) -> dict[str, str]: + session = self._active_session() + if session is not None: + return session.add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) + return super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + """ + Rename an imported source-backed visible name during authoring. + + Only imported source-backed cells may be renamed on the builder itself. + Declared/generated cells must be registered under their intended final + names. `move_references=True` is intentionally unsupported here because + deferred recipes and declaration internals cannot be rewritten safely. + """ + session = self._active_session() + if session is not None: + session.rename(old_name, new_name, move_references=move_references) + return self + + self._assert_editable() + if old_name == new_name: + return self + if old_name in self._declarations: + raise BuildError( + f'Cannot rename declared build cell "{old_name}" during authoring. ' + 'Register it under the intended final name instead.' + ) + if old_name not in self._names: + raise LibraryError(f'"{old_name}" does not exist in the builder.') + if new_name in self._names: + raise LibraryError(f'"{new_name}" already exists in the builder.') + if move_references: + raise BuildError( + 'BuildLibrary.rename(..., move_references=True) is not supported for imported source cells. ' + 'Builder-level renames only change the visible imported name.' + ) + + source_index = next( + (idx for idx, spec in enumerate(self._sources) if old_name in spec.visible_to_source), + None, + ) + if source_index is None: + raise BuildError( + f'Cannot rename "{old_name}" during authoring because only imported source-backed ' + 'cells may be renamed on a BuildLibrary.' + ) + + spec = self._sources[source_index] + source_name = spec.visible_to_source[old_name] + source_to_visible = dict(spec.source_to_visible) + visible_to_source = dict(spec.visible_to_source) + order = list(spec.order) + + source_to_visible[source_name] = new_name + del visible_to_source[old_name] + visible_to_source[new_name] = source_name + order[order.index(old_name)] = new_name + + self._sources[source_index] = replace( + spec, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + order=tuple(order), + ) + self._names.remove(old_name) + self._names.add(new_name) + self._order[self._order.index(old_name)] = new_name + return self + + def abstract(self, name: str) -> Abstract: + return self._require_active_session('abstract').abstract(name) + + def resolve( + self, + other: 'Abstract | str | Pattern | TreeView', + append: bool = False, + ) -> 'Abstract | Pattern': + return self._require_active_session('resolve').resolve(other, append=append) + + def add_source( + self, + source: Mapping[str, 'Pattern'] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + """ + Register an imported source-backed library with the builder. + + The source is not materialized immediately. Instead, its names and + child graph are scanned once and stored as an import declaration. The + source may be renamed on entry to avoid collisions with existing + declarations or other imported sources. + + Returns: + Mapping of `{source_name: visible_name}` for imported names that + were renamed while being added. + """ + self._assert_editable() + + view = source if isinstance(source, ILibraryView) else LibraryView(source) + source_order = tuple(view.source_order()) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + new_names: list[str] = [] + + for name in source_order: + visible = name + if visible in self._names or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._names or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + new_names.append(visible) + + self._sources.append(_SourceDeclaration( + library=view, + source_to_visible=dict(source_to_visible), + visible_to_source=dict(visible_to_source), + child_graph={name: set(children) for name, children in child_graph.items()}, + order=tuple(source_to_visible[name] for name in source_order), + )) + for visible in new_names: + self._names.add(visible) + self._order.append(visible) + return rename_map + + def validate( + self, + names: Sequence[str] | None = None, + *, + allow_dangling: bool = False, + ) -> BuildReport: + """ + Run the full build logic and return a `BuildReport` without producing output. + + This is a dry run over the same dependency resolution and recipe + execution path used by `build()`. Any generated library is discarded + after validation completes. + """ + report, _output = self._run_build(names=names, output='overlay', allow_dangling=allow_dangling, persist_output=False) + self.last_build_report = report + return report + + def build( + self, + *, + output: Literal['overlay', 'library'] = 'overlay', + allow_dangling: bool = False, + ) -> 'BuiltLibrary | ILibrary': + """ + Materialize declarations and return a usable output library. + + Args: + output: `'overlay'` preserves imported source-backed cells where + possible, while `'library'` eagerly materializes the full + result. + allow_dangling: If `False`, fail the build when the completed + library still contains dangling references. + """ + self._assert_editable() + report, built_output = self._run_build(names=None, output=output, allow_dangling=allow_dangling, persist_output=True) + self._frozen = True + self.last_build_report = report + return built_output + + def _run_build( + self, + *, + names: Sequence[str] | None, + output: Literal['overlay', 'library'], + allow_dangling: bool, + persist_output: bool, + ) -> tuple[BuildReport, BuiltLibrary | ILibrary | None]: + roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys())) + unknown = [name for name in roots if name not in self._names] + if unknown: + raise BuildError(f'Unknown build roots requested: {unknown}') + + session = _BuildSessionLibrary(self) + sessions = dict(_ACTIVE_BUILD_SESSIONS.get() or {}) + sessions[id(self)] = session + token = _ACTIVE_BUILD_SESSIONS.set(sessions) + try: + session.materialize_many(roots) + if not allow_dangling: + session.child_graph(dangling='error') + if output == 'library': + built_output = session.to_library() if persist_output else None + elif persist_output: + built_output = session.to_overlay() + else: + built_output = None + finally: + _ACTIVE_BUILD_SESSIONS.reset(token) + + report = session.build_report(roots) + if built_output is not None: + built_output.build_report = report + return report, built_output + + +class _BuildSessionLibrary(ILibrary): + """ + Internal overlay-backed library used while a `BuildLibrary` is executing. + + This object provides the mutable-library surface that recipes expect while + also tracking declared-cell dependencies, helper-cell provenance, and + imported source cells. It exists only for the duration of a validation or + build run. + """ + + def __init__(self, builder: BuildLibrary) -> None: + from .file.gdsii_lazy_core import BuiltOverlayLibrary, _SourceEntry, _SourceLayer # noqa: PLC0415 + + self._builder = builder + self._overlay = BuiltOverlayLibrary() + self._source_entry_type = _SourceEntry + self._source_layer_type = _SourceLayer + self._states: dict[str, Literal['unbuilt', 'building', 'built']] = { + name: 'unbuilt' for name in builder._declarations + } + self._declared_stack: list[str] = [] + self._emission_stack: list[str] = [] + self._emission_via_stack: list[emitted_via_t] = [] + self._names = set(builder._names) + self._order = list(builder._order) + self._provenance: dict[str, CellProvenance] = {} + self._owned_cells: defaultdict[str, list[str]] = defaultdict(list) + self._dependency_graph: defaultdict[str, set[str]] = defaultdict(set) + self._install_sources() + + def _install_sources(self) -> None: + for spec in self._builder._sources: + layer = self._source_layer_type( + library=spec.library, + source_to_visible=dict(spec.source_to_visible), + visible_to_source=dict(spec.visible_to_source), + child_graph={name: set(children) for name, children in spec.child_graph.items()}, + order=list(spec.order), + ) + layer_index = len(self._overlay._layers) + self._overlay._layers.append(layer) + source_info = getattr(spec.library, 'library_info', None) + source_meta = dict(source_info) if isinstance(source_info, dict) else None + + for source_name, visible_name in spec.source_to_visible.items(): + self._overlay._entries[visible_name] = self._source_entry_type( + layer_index=layer_index, + source_name=source_name, + ) + if visible_name not in self._overlay._order: + self._overlay._order.append(visible_name) + self._provenance[visible_name] = CellProvenance( + final_name=visible_name, + requested_name=source_name, + kind='source', + owner_declared_name=None, + emitted_via='source_import', + build_chain=(), + renamed_from=source_name if visible_name != source_name else None, + source_name=source_name, + source_metadata=source_meta, + ) + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._names) + + def __len__(self) -> int: + return len(self._names) + + def __contains__(self, key: object) -> bool: + return key in self._names or key in self._overlay + + def _touch_name(self, key: str) -> None: + if key not in self._names: + self._names.add(key) + self._order.append(key) + + def _current_declared(self) -> str | None: + if not self._declared_stack: + return None + return self._declared_stack[-1] + + def _record_dependency(self, target: str) -> None: + current = self._current_declared() + if current is None or current == target or target not in self._builder._declarations: + return + self._dependency_graph[current].add(target) + + def _guard_mutable_output_name(self, key: str, *, operation: str) -> None: + if key in self._builder._declarations: + raise BuildError(f'Cannot {operation} declared build cell "{key}" during an active build session.') + + provenance = self._provenance.get(key) + if provenance is not None and provenance.kind == 'source': + raise BuildError(f'Cannot {operation} imported source cell "{key}" during an active build session.') + + def _remove_owned_cell(self, owner: str | None, name: str) -> None: + if owner is None or owner not in self._owned_cells: + return + cells = self._owned_cells[owner] + self._owned_cells[owner] = [cell for cell in cells if cell != name] + if not self._owned_cells[owner]: + del self._owned_cells[owner] + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + if old_name == new_name: + return self + if old_name not in self._overlay: + if old_name in self._builder._declarations: + self._guard_mutable_output_name(old_name, operation='rename') + raise LibraryError(f'"{old_name}" does not exist in the library.') + + self._guard_mutable_output_name(old_name, operation='rename') + if new_name in self._names: + raise LibraryError(f'"{new_name}" already exists in the library.') + + self._overlay.rename(old_name, new_name, move_references=move_references) + self._names.discard(old_name) + self._names.add(new_name) + if old_name in self._order: + idx = self._order.index(old_name) + self._order[idx] = new_name + + provenance = self._provenance.pop(old_name) + requested_name = provenance.requested_name + self._provenance[new_name] = replace( + provenance, + final_name=new_name, + renamed_from=requested_name if new_name != requested_name else None, + ) + + owner = provenance.owner_declared_name + if owner is not None and owner in self._owned_cells: + self._owned_cells[owner] = [ + new_name if cell_name == old_name else cell_name + for cell_name in self._owned_cells[owner] + ] + return self + + def __getitem__(self, key: str) -> 'Pattern': + if key in self._builder._declarations: + self._record_dependency(key) + self._ensure_declared(key) + return self._overlay[key] + + def __setitem__( + self, + key: str, + value: 'Pattern | Callable[[], Pattern]', + ) -> None: + if key in self._overlay: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + current = self._current_declared() + if key in self._builder._declarations and key != current: + raise LibraryError(f'"{key}" is reserved for a declared cell and cannot be used as a helper name.') + + pattern = value() if callable(value) else value + self._overlay[key] = pattern + self._touch_name(key) + + kind: Literal['declared', 'helper'] + via = self._emission_via_stack[-1] if self._emission_via_stack else 'helper_write' + if current is not None and key == current: + kind = 'declared' + via = 'declaration' + else: + kind = 'helper' + if not self._emission_via_stack: + via = 'helper_write' + + self._emission_stack.append(key) + try: + self._record_provenance( + final_name=key, + requested_name=key, + kind=kind, + owner_declared_name=current if kind == 'helper' else key, + emitted_via=via, + build_chain=tuple(self._declared_stack), + renamed_from=None, + ) + finally: + self._emission_stack.pop() + + def __delitem__(self, key: str) -> None: + if key not in self._overlay: + if key in self._builder._declarations: + self._guard_mutable_output_name(key, operation='delete') + raise KeyError(key) + + self._guard_mutable_output_name(key, operation='delete') + provenance = self._provenance.get(key) + if key in self._overlay: + del self._overlay[key] + self._names.discard(key) + if key in self._order: + self._order.remove(key) + self._provenance.pop(key, None) + if provenance is not None: + self._remove_owned_cell(provenance.owner_declared_name, key) + + def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add( + self, + other: Mapping[str, 'Pattern'], + rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns, + mutate_other: bool = False, + ) -> dict[str, str]: + self._emission_via_stack.append('tree_merge') + try: + rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) + finally: + self._emission_via_stack.pop() + + current = self._current_declared() + for old_name, new_name in rename_map.items(): + if new_name in self._provenance: + self._provenance[new_name] = replace( + self._provenance[new_name], + requested_name=old_name, + renamed_from=old_name, + owner_declared_name=current if current is not None else self._provenance[new_name].owner_declared_name, + ) + return rename_map + + def _record_provenance( + self, + *, + final_name: str, + requested_name: str, + kind: Literal['declared', 'helper'], + owner_declared_name: str | None, + emitted_via: emitted_via_t, + build_chain: tuple[str, ...], + renamed_from: str | None, + ) -> None: + self._provenance[final_name] = CellProvenance( + final_name=final_name, + requested_name=requested_name, + kind=kind, + owner_declared_name=owner_declared_name, + emitted_via=emitted_via, + build_chain=build_chain, + renamed_from=renamed_from, + ) + if owner_declared_name is not None and final_name not in self._owned_cells[owner_declared_name]: + self._owned_cells[owner_declared_name].append(final_name) + + def _wrap_error(self, name: str, exc: Exception) -> BuildError: + helper = self._emission_stack[-1] if self._emission_stack else None + chain = tuple(self._declared_stack) + msg = [f'Failed while building declared cell "{name}"'] + if helper is not None and helper != name: + msg.append(f'while materializing helper/output "{helper}"') + if chain: + msg.append(f'Dependency chain: {" -> ".join(chain)}') + msg.append(f'Cause: {exc}') + return BuildError('\n'.join(msg)) + + def _ensure_named(self, name: str) -> None: + if name in self._builder._declarations: + self._record_dependency(name) + self._ensure_declared(name) + return + if name in self._overlay: + return + raise BuildError(f'Missing dependency "{name}"') + + def _ensure_declared(self, name: str) -> None: + from .pattern import Pattern # noqa: PLC0415 + + state = self._states[name] + if state == 'built': + return + if state == 'building': + chain = ' -> '.join(self._declared_stack + [name]) + raise BuildError(f'Cycle detected while building declared cells: {chain}') + + declaration = self._builder._declarations[name] + self._states[name] = 'building' + self._declared_stack.append(name) + try: + if isinstance(declaration, _PatternDeclaration): + pattern = declaration.pattern.deepcopy() + else: + for dep in declaration.recipe.explicit_dependencies: + self._ensure_named(dep) + pattern = declaration.recipe.func(*declaration.recipe.args, **declaration.recipe.kwargs) + if not isinstance(pattern, Pattern): + raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') + + if name in self._overlay: + if self._overlay[name] is not pattern: + raise BuildError( + f'Recipe for "{name}" wrote a different pattern into the session under its own name.' + ) + else: + self[name] = pattern + self._states[name] = 'built' + except Exception as exc: + self._states[name] = 'unbuilt' + raise self._wrap_error(name, exc) from exc + finally: + self._declared_stack.pop() + + def materialize_many(self, names: Sequence[str]) -> None: + for name in dict.fromkeys(names): + self._ensure_named(name) + + def source_order(self) -> tuple[str, ...]: + return self._overlay.source_order() + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._overlay.child_graph(dangling=dangling) + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._overlay.parent_graph(dangling=dangling) + + def build_report(self, requested_roots: Sequence[str]) -> BuildReport: + dependency_graph = { + name: frozenset(self._dependency_graph.get(name, set())) + for name in self._builder._declarations + if name in self._dependency_graph or name in requested_roots + } + owned_cells = { + name: tuple(cells) + for name, cells in self._owned_cells.items() + } + return BuildReport( + requested_roots=tuple(dict.fromkeys(requested_roots)), + provenance=dict(self._provenance), + owned_cells=owned_cells, + dependency_graph=dependency_graph, + ) + + def to_overlay(self) -> ILibrary: + return self._overlay + + def to_library(self) -> BuiltLibrary: + mapping = {name: self._overlay[name] for name in self._overlay.source_order()} + return BuiltLibrary(mapping) + + class LazyLibrary(ILibrary): """ This class is usually used to create a library of Patterns by mapping names to diff --git a/masque/test/test_build_library.py b/masque/test/test_build_library.py new file mode 100644 index 0000000..ad43aba --- /dev/null +++ b/masque/test/test_build_library.py @@ -0,0 +1,315 @@ +import pytest + +from ..builder import Pather +from ..error import BuildError +from ..library import BuildLibrary, BuiltLibrary, Library, cell +from ..pattern import Pattern +from ..ports import Port + + +def test_build_library_traces_declared_dependencies_out_of_order() -> None: + builder = BuildLibrary() + + def make_parent(lib: BuildLibrary) -> Pattern: + pat = Pattern() + pat.ref("child") + assert lib.abstract("child").name == "child" + return pat + + builder.cells.parent = cell(make_parent)(builder) + builder["child"] = Pattern(ports={"p": Port((0, 0), 0)}) + + built = builder.build() + + assert "parent" in built + assert "child" in built + assert built.build_report.dependency_graph["parent"] == frozenset({"child"}) + assert built.build_report.provenance["parent"].kind == "declared" + + +def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None: + builder = BuildLibrary() + + def make_top(lib: BuildLibrary) -> Pattern: + tree = Library({"_helper": Pattern()}) + name_a = lib << tree + name_b = lib << tree + top = Pattern() + top.ref(name_a) + top.ref(name_b) + return top + + builder.cells.top = cell(make_top)(builder) + built = builder.build() + report = built.build_report + + helpers = [ + prov for prov in report.provenance.values() + if prov.owner_declared_name == "top" and prov.kind == "helper" + ] + + assert "top" in report.owned_cells["top"] + assert len(helpers) == 2 + assert all(prov.emitted_via == "tree_merge" for prov in helpers) + assert any(prov.renamed_from == "_helper" for prov in helpers) + + +def test_build_library_requires_build_session_for_reads_and_freezes_after_build() -> None: + builder = BuildLibrary() + builder["leaf"] = Pattern() + + with pytest.raises(BuildError, match="validate\\(\\) or build\\(\\)"): + _ = builder["leaf"] + + with pytest.raises(BuildError, match="write-only"): + _ = builder.cells.leaf + + built = builder.build(output="library") + + assert isinstance(built, BuiltLibrary) + assert built.build_report.requested_roots == ("leaf",) + + with pytest.raises(BuildError, match="frozen"): + builder["later"] = Pattern() + with pytest.raises(BuildError, match="frozen"): + builder.build() + + +def test_build_library_validate_is_retryable_after_failure() -> None: + builder = BuildLibrary() + + def make_parent(lib: BuildLibrary) -> Pattern: + pat = Pattern() + pat.ref("child") + lib.abstract("child") + return pat + + builder.cells.parent = cell(make_parent)(builder) + + with pytest.raises(BuildError, match='Failed while building declared cell "parent"'): + builder.validate() + + builder["child"] = Pattern(ports={"p": Port((0, 0), 0)}) + report = builder.validate() + + assert report.dependency_graph["parent"] == frozenset({"child"}) + + +def test_build_library_check_on_register_rolls_back_failed_declarations() -> None: + builder = BuildLibrary(check_on_register=True) + + def make_parent(lib: BuildLibrary) -> Pattern: + pat = Pattern() + pat.ref("child") + lib.abstract("child") + return pat + + with pytest.raises(BuildError, match='Failed while building declared cell "parent"'): + builder.cells.parent = cell(make_parent)(builder) + + assert "parent" not in builder + + +def test_build_library_depends_on_supports_hidden_dependencies_for_partial_validation() -> None: + builder = BuildLibrary() + builder["child"] = Pattern() + + def make_parent() -> Pattern: + pat = Pattern() + pat.ref("child") + return pat + + builder.cells.parent = cell(make_parent)().depends_on("child") + report = builder.validate(names=("parent",)) + + assert report.requested_roots == ("parent",) + assert report.dependency_graph["parent"] == frozenset({"child"}) + + +def test_build_library_validate_rejects_removed_output_argument() -> None: + builder = BuildLibrary() + builder["leaf"] = Pattern() + + with pytest.raises(TypeError): + builder.validate(output="library") # type: ignore[call-arg] + + +def test_build_library_allows_helper_writes_via_pather() -> None: + builder = BuildLibrary() + builder["leaf"] = Pattern(ports={"a": Port((0, 0), 0)}) + + def make_top(lib: BuildLibrary) -> Pattern: + helper = Pather(library=lib, ports="leaf", name="_route") + top = Pattern() + top.ref("_route") + top.ref("leaf") + top.ports.update(helper.pattern.ports) + return top + + builder.cells.top = cell(make_top)(builder) + built = builder.build() + + helper_prov = built.build_report.provenance["_route"] + assert helper_prov.kind == "helper" + assert helper_prov.owner_declared_name == "top" + + +def test_build_library_preserves_source_cells_and_records_source_provenance() -> None: + source = Library({"src": Pattern()}) + builder = BuildLibrary() + builder.add_source(source) + builder.cells.top = cell(lambda: Pattern())() + + built = builder.build() + + assert "src" in built + assert built.build_report.provenance["src"].kind == "source" + assert built.build_report.provenance["src"].emitted_via == "source_import" + + +def test_build_library_can_rename_imported_source_cells_during_authoring() -> None: + source = Library() + source["child"] = Pattern() + parent = Pattern() + parent.ref("child") + source["parent"] = parent + + builder = BuildLibrary() + builder.add_source(source) + builder.rename("child", "renamed_child") + + built = builder.build() + + assert "renamed_child" in built + assert "child" not in built + assert "renamed_child" in built["parent"].refs + assert built.build_report.provenance["renamed_child"].source_name == "child" + + +def test_build_library_rejects_move_references_for_source_rename() -> None: + builder = BuildLibrary() + builder.add_source(Library({"src": Pattern()})) + + with pytest.raises(BuildError, match="move_references=True"): + builder.rename("src", "renamed_src", move_references=True) + + +def test_build_library_rejects_renaming_declared_cells_during_authoring() -> None: + builder = BuildLibrary() + builder["declared"] = Pattern() + + with pytest.raises(BuildError, match='Cannot rename declared build cell "declared"'): + builder.rename("declared", "renamed_declared") + + +def test_build_library_helper_rename_updates_provenance_and_owned_cells() -> None: + builder = BuildLibrary() + + def make_top(lib: BuildLibrary) -> Pattern: + lib["_helper"] = Pattern() + lib.rename("_helper", "final_helper") + top = Pattern() + top.ref("final_helper") + return top + + builder.cells.top = cell(make_top)(builder) + built = builder.build() + report = built.build_report + + assert "final_helper" in built + assert "_helper" not in built + assert "final_helper" in report.owned_cells["top"] + assert "_helper" not in report.owned_cells["top"] + prov = report.provenance["final_helper"] + assert prov.kind == "helper" + assert prov.requested_name == "_helper" + assert prov.renamed_from == "_helper" + assert prov.final_name == "final_helper" + + +def test_build_library_helper_delete_removes_provenance_and_ownership() -> None: + builder = BuildLibrary() + + def make_top(lib: BuildLibrary) -> Pattern: + lib["_helper"] = Pattern() + del lib["_helper"] + return Pattern() + + builder.cells.top = cell(make_top)(builder) + built = builder.build() + report = built.build_report + + assert "_helper" not in built + assert "_helper" not in report.provenance + assert report.owned_cells["top"] == ("top",) + + +def test_build_library_helper_rename_after_auto_rename_preserves_requested_name() -> None: + builder = BuildLibrary() + + def make_top(lib: BuildLibrary) -> Pattern: + tree = Library({"_helper": Pattern()}) + _ = lib << tree + renamed = lib << tree + lib.rename(renamed, "final_helper") + top = Pattern() + top.ref("_helper") + top.ref("final_helper") + return top + + builder.cells.top = cell(make_top)(builder) + built = builder.build() + report = built.build_report + + assert "final_helper" in built + prov = report.provenance["final_helper"] + assert prov.requested_name == "_helper" + assert prov.renamed_from == "_helper" + + +def test_build_library_rejects_renaming_declared_or_source_cells_during_build() -> None: + declared = BuildLibrary() + declared["leaf"] = Pattern() + + def rename_declared(lib: BuildLibrary) -> Pattern: + lib.rename("leaf", "renamed_leaf") + return Pattern() + + declared.cells.top = cell(rename_declared)(declared) + with pytest.raises(BuildError, match='Cannot rename declared build cell "leaf"'): + declared.build() + + source = BuildLibrary() + source.add_source(Library({"src": Pattern()})) + + def rename_source(lib: BuildLibrary) -> Pattern: + lib.rename("src", "renamed_src") + return Pattern() + + source.cells.top = cell(rename_source)(source) + with pytest.raises(BuildError, match='Cannot rename imported source cell "src"'): + source.build() + + +def test_build_library_rejects_deleting_declared_or_source_cells_during_build() -> None: + declared = BuildLibrary() + declared["leaf"] = Pattern() + + def delete_declared(lib: BuildLibrary) -> Pattern: + del lib["leaf"] + return Pattern() + + declared.cells.top = cell(delete_declared)(declared) + with pytest.raises(BuildError, match='Cannot delete declared build cell "leaf"'): + declared.build() + + source = BuildLibrary() + source.add_source(Library({"src": Pattern()})) + + def delete_source(lib: BuildLibrary) -> Pattern: + del lib["src"] + return Pattern() + + source.cells.top = cell(delete_source)(source) + with pytest.raises(BuildError, match='Cannot delete imported source cell "src"'): + source.build()