diff --git a/examples/generate_gds_perf.py b/examples/generate_gds_perf.py new file mode 100644 index 0000000..6bdd999 --- /dev/null +++ b/examples/generate_gds_perf.py @@ -0,0 +1,5 @@ +from masque.file.gdsii_perf import main + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py new file mode 100644 index 0000000..fb7c99e --- /dev/null +++ b/examples/profile_gdsii_readers.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import argparse +import importlib +import json +import time +from pathlib import Path +from typing import Any + +from masque import LibraryError + + +READERS: dict[str, tuple[str, tuple[str, ...]]] = { + 'gdsii': ('masque.file.gdsii', ('readfile',)), + 'gdsii_arrow': ('masque.file.gdsii_arrow', ('readfile', 'arrow_import', 'arrow_convert')), + } + + +def _summarize_library(path: Path, elapsed_s: float, info: dict[str, object], lib: object) -> dict[str, object]: + assert hasattr(lib, '__len__') + assert hasattr(lib, 'tops') + tops = lib.tops() # type: ignore[no-any-return, attr-defined] + try: + unique_top = lib.top() # type: ignore[no-any-return, attr-defined] + except LibraryError: + unique_top = None + + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'library_name': info['name'], + 'cell_count': len(lib), # type: ignore[arg-type] + 'topcells': tops, + 'topcell': unique_top, + } + + +def _summarize_arrow_import(path: Path, elapsed_s: float, arrow_arr: Any) -> dict[str, object]: + libarr = arrow_arr[0] + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': len(arrow_arr), + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + + +def _profile_stage(module: Any, stage: str, path: Path) -> dict[str, object]: + start = time.perf_counter() + + if stage == 'readfile': + lib, info = module.readfile(path) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + if stage == 'arrow_import': + if hasattr(module, 'readfile_arrow'): + libarr, _info = module.readfile_arrow(path) + elapsed_s = time.perf_counter() - start + return { + 'path': str(path), + 'elapsed_s': elapsed_s, + 'arrow_rows': 1, + 'library_name': libarr['lib_name'].as_py(), + 'cell_count': len(libarr['cells']), + 'layer_count': len(libarr['layers']), + } + + arrow_arr = module._read_to_arrow(path) + elapsed_s = time.perf_counter() - start + return _summarize_arrow_import(path, elapsed_s, arrow_arr) + + if stage == 'arrow_convert': + arrow_arr = module._read_to_arrow(path) + libarr = arrow_arr[0] + start = time.perf_counter() + lib, info = module.read_arrow(libarr) + elapsed_s = time.perf_counter() - start + return _summarize_library(path, elapsed_s, info, lib) + + raise ValueError(f'Unsupported stage {stage!r}') + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Profile GDS readers with a stable end-to-end workload.') + parser.add_argument('--reader', choices=sorted(READERS), required=True) + parser.add_argument('--stage', default='readfile') + parser.add_argument('--path', type=Path, required=True) + parser.add_argument('--warmup', type=int, default=1) + parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--output-json', type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + module_name, stages = READERS[args.reader] + if args.stage not in stages: + parser.error(f'reader {args.reader!r} only supports stages: {", ".join(stages)}') + + module = importlib.import_module(module_name) + path = args.path.expanduser().resolve() + + for _ in range(args.warmup): + _profile_stage(module, args.stage, path) + + runs = [] + for _ in range(args.repeat): + runs.append(_profile_stage(module, args.stage, path)) + + payload = { + 'reader': args.reader, + 'stage': args.stage, + 'warmup': args.warmup, + 'repeat': args.repeat, + 'runs': runs, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(rendered + '\n') + print(rendered) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/__init__.py b/masque/__init__.py index 8e13bb9..b6cc5e9 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -42,6 +42,7 @@ from .error import ( from .shapes import ( Shape as Shape, Polygon as Polygon, + RectCollection as RectCollection, Path as Path, Circle as Circle, Arc as Arc, diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 116fa07..1d8c3d1 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -37,7 +37,7 @@ from klamath import records from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path +from ..shapes import Polygon, Path, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -323,26 +323,40 @@ def _gpath_to_mpath(gpath: klamath.library.Path, raw_mode: bool) -> tuple[layer_ else: raise PatternError(f'Unrecognized path type: {gpath.path_type}') - mpath = Path( - vertices=gpath.xy.astype(float), - width=gpath.width, - cap=cap, - offset=numpy.zeros(2), - annotations=_properties_to_annotations(gpath.properties), - raw=raw_mode, - ) + vertices = gpath.xy.astype(float) + annotations = _properties_to_annotations(gpath.properties) + cap_extensions = None if cap == Path.Cap.SquareCustom: - mpath.cap_extensions = gpath.extension + cap_extensions = numpy.asarray(gpath.extension, dtype=float) + + if raw_mode: + mpath = Path._from_raw( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + else: + mpath = Path( + vertices=vertices, + width=gpath.width, + cap=cap, + cap_extensions=cap_extensions, + offset=numpy.zeros(2), + annotations=annotations, + ) return gpath.layer, mpath def _boundary_to_polygon(boundary: klamath.library.Boundary, raw_mode: bool) -> tuple[layer_t, Polygon]: - return boundary.layer, Polygon( - vertices=boundary.xy[:-1].astype(float), - offset=numpy.zeros(2), - annotations=_properties_to_annotations(boundary.properties), - raw=raw_mode, - ) + vertices = boundary.xy[:-1].astype(float) + annotations = _properties_to_annotations(boundary.properties) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=numpy.zeros(2), annotations=annotations) + return boundary.layer, poly def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.Reference]: @@ -466,6 +480,20 @@ def _shapes_to_elements( properties=properties, ) elements.append(path) + elif isinstance(shape, RectCollection): + for rect in shape.rects: + xy_closed = numpy.empty((5, 2), dtype=numpy.int32) + xy_closed[0] = rint_cast((rect[0], rect[1])) + xy_closed[1] = rint_cast((rect[0], rect[3])) + xy_closed[2] = rint_cast((rect[2], rect[3])) + xy_closed[3] = rint_cast((rect[2], rect[1])) + xy_closed[4] = xy_closed[0] + boundary = klamath.elements.Boundary( + layer=(layer, data_type), + xy=xy_closed, + properties=properties, + ) + elements.append(boundary) elif isinstance(shape, Polygon): polygon = shape xy_closed = numpy.empty((polygon.vertices.shape[0] + 1, 2), dtype=numpy.int32) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..95229ea --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,882 @@ +# ruff: noqa: ARG001, F401 +""" +GDSII file format readers and writers using the `TODO` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +from importlib.machinery import EXTENSION_SUFFIXES +import importlib.util +import mmap +import logging +import os +import pathlib +import gzip +import string +import sys +import tempfile +from pprint import pformat + +from klamath.basic import KlamathError +import numpy +from numpy.typing import ArrayLike, NDArray +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path, PolyCollection, RectCollection +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +ffi.cdef( + """ + const char* last_error_message(void); + int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema); + int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema); + int read_cells_bytes( + uint8_t* data, + size_t size, + uint64_t* ranges, + size_t range_count, + struct ArrowArray* array, + struct ArrowSchema* schema + ); + """ +) + +clib: Any | None = None + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]: + layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16) + dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16) + return numpy.stack((layer, dtype), axis=-1) + + +def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]: + a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64) + b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64) + return numpy.stack((a_count, b_count), axis=-1) + + +def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]: + xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32) + yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32) + return numpy.stack((xx, yy), axis=-1) + + +def _local_library_filename() -> str: + if sys.platform.startswith('linux'): + return 'libklamath_rs_ext.so' + if sys.platform == 'darwin': + return 'libklamath_rs_ext.dylib' + if sys.platform == 'win32': + return 'klamath_rs_ext.dll' + raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}') + + +def _installed_library_candidates() -> list[pathlib.Path]: + candidates: list[pathlib.Path] = [] + + try: + spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext') + except ModuleNotFoundError: + spec = None + if spec is not None and spec.origin is not None: + candidates.append(pathlib.Path(spec.origin)) + + try: + pkg_spec = importlib.util.find_spec('klamath_rs_ext') + except ModuleNotFoundError: + pkg_spec = None + if pkg_spec is not None and pkg_spec.submodule_search_locations is not None: + for location in pkg_spec.submodule_search_locations: + pkg_dir = pathlib.Path(location) + for suffix in EXTENSION_SUFFIXES: + candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}'))) + + return candidates + + +def _repo_library_candidates() -> list[pathlib.Path]: + repo_root = pathlib.Path(__file__).resolve().parents[2] + library_name = _local_library_filename() + return [ + repo_root / 'klamath-rs' / 'target' / 'release' / library_name, + repo_root / 'klamath-rs' / 'target' / 'debug' / library_name, + ] + + +def find_klamath_rs_library() -> pathlib.Path | None: + env_path = os.environ.get('KLAMATH_RS_EXT_LIB') + if env_path: + candidate = pathlib.Path(env_path).expanduser() + if candidate.exists(): + return candidate.resolve() + + seen: set[pathlib.Path] = set() + for candidate in _installed_library_candidates() + _repo_library_candidates(): + resolved = candidate.expanduser() + if resolved in seen: + continue + seen.add(resolved) + if resolved.exists(): + return resolved.resolve() + return None + + +def is_available() -> bool: + return find_klamath_rs_library() is not None + + +def _get_clib() -> Any: + global clib + if clib is None: + lib_path = find_klamath_rs_library() + if lib_path is None: + raise ImportError( + 'Could not locate klamath_rs_ext shared library. ' + 'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` ' + 'or set KLAMATH_RS_EXT_LIB to the built library path.' + ) + clib = ffi.dlopen(str(lib_path)) + return clib + + +def _read_annotations( + prop_offs: NDArray[numpy.integer[Any]], + prop_key: NDArray[numpy.integer[Any]], + prop_val: list[str], + ee: int, + ) -> annotations_t: + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii >= prop_ff: + return None + return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)} + + +def _read_to_arrow( + filename: str | pathlib.Path, + ) -> pyarrow.Array: + path = pathlib.Path(filename).expanduser().resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + if is_gzipped(path): + with gzip.open(path, mode='rb') as src: + data = src.read() + with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream: + tmp_stream.write(data) + tmp_name = tmp_stream.name + try: + _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path') + finally: + pathlib.Path(tmp_name).unlink(missing_ok=True) + else: + _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path') + return _import_arrow_array(ptr_array, ptr_schema) + + +def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array: + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + return pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + +def _call_native(status: int, action: str) -> None: + if status == 0: + return + + err_ptr = _get_clib().last_error_message() + if err_ptr == ffi.NULL: + raise KlamathError(f'{action} failed') + + message = ffi.string(err_ptr).decode(errors='replace') + raise KlamathError(message) + + +def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes') + return _import_arrow_array(ptr_array, ptr_schema) + + +def _read_selected_cells_to_arrow( + buffer: bytes | mmap.mmap | memoryview, + ranges: NDArray[numpy.uint64], + ) -> pyarrow.Array: + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + buf_view = memoryview(buffer) + cbuf = ffi.from_buffer('uint8_t[]', buf_view) + flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED')) + cranges = ffi.from_buffer('uint64_t[]', flat_ranges) + _call_native( + _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema), + 'read_cells_bytes', + ) + return _import_arrow_array(ptr_array, ptr_schema) + + +def readfile( + filename: str | pathlib.Path, + ) -> tuple[Library, dict[str, Any]]: + """ + Read a GDSII file from a path into `masque.Library` / `Pattern` objects. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to read. + + For callers that can consume Arrow directly, prefer `readfile_arrow()` + to skip Python `Pattern` construction entirely. + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def readfile_arrow( + filename: str | pathlib.Path, + ) -> tuple[pyarrow.StructScalar, dict[str, Any]]: + """ + Read a GDSII file into the native Arrow representation without converting + it into `masque.Library` / `Pattern` objects. + + This is the lowest-overhead public read path exposed by this module. + + Args: + filename: Filename to read. + + Returns: + - Arrow struct scalar for the library payload + - dict of GDSII library info + """ + arrow_arr = _read_to_arrow(filename) + assert len(arrow_arr) == 1 + libarr = arrow_arr[0] + return libarr, _read_header(libarr) + + +def read_arrow( + libarr: pyarrow.Array, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + libarr: Arrow library payload as returned by `readfile_arrow()`. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy()) + layer_tups = [(int(pair[0]), int(pair[1])) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('boundary_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = batches.values.field('vertices').offsets.to_numpy() // 2, + poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(), + poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), + ) + + def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('rect_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_off = batches.values.field('rects').offsets.to_numpy() // 4, + ) + + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: + boundaries = libarr['cells'].values.field('boundary_props') + return dict( + offsets = boundaries.offsets.to_numpy(), + layer_inds = boundaries.values.field('layer').to_numpy(), + vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2, + prop_off = boundaries.values.field('properties').offsets.to_numpy(), + prop_key = boundaries.values.field('properties').values.field('key').to_numpy(), + prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), + ) + + def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + prop_off = values.field('properties').offsets.to_numpy(), + prop_key = values.field('properties').values.field('key').to_numpy(), + prop_val = values.field('properties').values.field('value').to_pylist(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), + ) + + elements = dict( + srefs = get_refs(libarr, 'srefs', has_repetition=False), + arefs = get_refs(libarr, 'arefs', has_repetition=True), + sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False), + aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True), + rect_batches = get_rect_batches(libarr), + boundary_batches = get_boundary_batches(libarr), + boundary_props = get_boundary_props(libarr), + paths = get_geom(libarr, 'paths'), + texts = texts, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').fill_null(0).to_numpy(), + path_type = paths.values.field('path_type').fill_null(0).to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').fill_null(0).to_numpy(), + paths.values.field('extension_end').fill_null(0).to_numpy(), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + ) + + mlib = Library() + for cc in range(len(libarr['cells'])): + name = cell_names[int(cell_ids[cc])] + pat = Pattern() + _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) + _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) + _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _srefs_to_mrefs(pat, global_args, elements['srefs'], cc) + _arefs_to_mrefs(pat, global_args, elements['arefs'], cc) + _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc) + _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'].as_py(), + meters_per_unit = libarr['meters_per_db_unit'].as_py(), + logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(), + ) + return library_info + + +def _srefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + + _append_plain_refs_sorted( + pat=pat, + cell_names=cell_names, + elem_targets=elem_targets, + elem_xy=elem_xy, + elem_invert_y=elem_invert_y, + elem_angle_rad=elem_angle_rad, + elem_scale=elem_scale, + ) + + +def _append_plain_refs_sorted( + *, + pat: Pattern, + cell_names: list[str], + elem_targets: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.integer[Any]], + elem_invert_y: NDArray[numpy.bool_ | numpy.bool], + elem_angle_rad: NDArray[numpy.floating[Any]], + elem_scale: NDArray[numpy.floating[Any]], + ) -> None: + elem_count = len(elem_targets) + if elem_count == 0: + return + + target_start = 0 + while target_start < elem_count: + target_id = int(elem_targets[target_start]) + target_stop = target_start + 1 + while target_stop < elem_count and elem_targets[target_stop] == target_id: + target_stop += 1 + + append_refs = pat.refs[cell_names[target_id]].extend + append_refs( + Ref._from_raw( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=None, + ) + for ee in range(target_start, target_stop) + ) + + target_start = target_stop + + +def _arefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + elem_xy0 = elem['xy0'][start:stop] + elem_xy1 = elem['xy1'][start:stop] + elem_counts = elem['counts'][start:stop] + + if len(elem_targets) == 0: + return + + target = None + append_ref: Callable[[Ref], Any] | None = None + for ee in range(len(elem_targets)): + target_id = int(elem_targets[ee]) + if target != target_id: + target = target_id + append_ref = pat.refs[cell_names[target_id]].append + assert append_ref is not None + a_count, b_count = elem_counts[ee] + append_ref(Ref._from_raw( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=None, + )) + + +def _sref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + + for ee in range(elem_count): + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = Ref._from_raw( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=annotations, + ) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) + + +def _aref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] + elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] + elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] + + for ee in range(elem_count): + a_count, b_count = elem_counts[ee] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = Ref._from_raw( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=annotations, + ) + pat.refs[cell_names[int(elem_targets[ee])]].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_xy = xy[elem_slc][:elem_count] + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_strings = elem['string'][elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[int(elem_layer_inds[ee])] + offset = elem_xy[ee] + string = elem_strings[ee] + + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + elem_layer_inds = layer_inds[elem_slc][:elem_count] + elem_widths = elem['width'][elem_slc][:elem_count] + elem_path_types = elem['path_type'][elem_slc][:elem_count] + elem_extensions = elem['extensions'][elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[int(elem_layer_inds[ee])] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem_widths[ee] + cap_int = int(elem_path_types[ee]) + if cap_int not in path_cap_map: + raise PatternError(f'Unrecognized path type: {cap_int}') + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem_extensions[ee] + else: + cap_extensions = None + + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + path = Path._from_raw( + vertices=vertices, + width=width, + cap=cap, + cap_extensions=cap_extensions, + annotations=annotations, + ) + pat.shapes[layer].append(path) + + +def _boundary_batches_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + poly_off = elem['poly_off'] + poly_offsets = elem['poly_offsets'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) # +1 to capture ending location for last elem + elem_vert_off = vert_off[elem_slc] + elem_poly_off = poly_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + for bb in range(batch_count): + layer = layer_tups[int(elem_layer_inds[bb])] + vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] + vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] + + if vertex_offsets.size == 1: + poly = Polygon._from_raw(vertices=vertices, annotations=None) + pat.shapes[layer].append(poly) + else: + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) + pat.shapes[layer].append(polys) + + +def _rect_batches_to_rectcollections( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + rect_arr = elem['rect_arr'] + rect_off = elem['rect_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) + elem_rect_off = rect_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + for bb in range(batch_count): + layer = layer_tups[int(elem_layer_inds[bb])] + rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) + pat.shapes[layer].append(rect_collection) + + +def _boundary_props_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + vert_arr = elem['vert_arr'] + vert_off = elem['vert_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + elem_vert_off = vert_off[elem_slc] + prop_offs = elem['prop_off'][elem_slc] + elem_layer_inds = layer_inds[elem_slc][:elem_count] + + for ee in range(elem_count): + layer = layer_tups[int(elem_layer_inds[ee])] + vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + pat.shapes[layer].append(poly) + + +#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: +# return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py new file mode 100644 index 0000000..9a03960 --- /dev/null +++ b/masque/file/gdsii_lazy_arrow.py @@ -0,0 +1,960 @@ +""" +Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. + +This module is intentionally separate from `gdsii_arrow` so the eager read path +keeps its current behavior and performance profile. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import IO, Any, cast +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping, Sequence +import copy +import gzip +import logging +import mmap +import pathlib + +import numpy +from numpy.typing import NDArray +import pyarrow +import klamath + +from . import gdsii, gdsii_arrow +from .utils import is_gzipped, tmpfile +from ..error import LibraryError +from ..library import ILibrary, ILibraryView, Library, LibraryView, dangling_mode_t +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _StructRange: + start: int + end: int + + +@dataclass +class _SourceBuffer: + path: pathlib.Path + data: bytes | mmap.mmap + handle: IO[bytes] | None = None + + def raw_slice(self, start: int, end: int) -> bytes: + return self.data[start:end] + + +@dataclass +class _ScanRefs: + offsets: NDArray[numpy.integer[Any]] + targets: NDArray[numpy.integer[Any]] + xy: NDArray[numpy.int32] + xy0: NDArray[numpy.int32] + xy1: NDArray[numpy.int32] + counts: NDArray[numpy.int64] + invert_y: NDArray[numpy.bool_ | numpy.bool] + angle_rad: NDArray[numpy.floating[Any]] + scale: NDArray[numpy.floating[Any]] + + +@dataclass(frozen=True) +class _CellScan: + cell_id: int + struct_range: _StructRange + ref_start: int + ref_stop: int + children: set[str] + + +@dataclass +class _ScanPayload: + libarr: pyarrow.StructScalar + library_info: dict[str, Any] + cell_names: list[str] + cell_order: list[str] + cells: dict[str, _CellScan] + refs: _ScanRefs + + +@dataclass +class _SourceLayer: + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + layer_index: int + source_name: str + + +def is_available() -> bool: + return gdsii_arrow.is_available() + + +def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: + return gdsii_arrow._read_header(libarr) + + +def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: + if is_gzipped(path): + with gzip.open(path, mode='rb') as stream: + data = stream.read() + return _SourceBuffer(path=path, data=data) + + handle = path.open(mode='rb', buffering=0) + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + return _SourceBuffer(path=path, data=mapped, handle=handle) + + +def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: + library_info = _read_header(libarr) + cell_names = libarr['cell_names'].as_py() + + cells = libarr['cells'] + cell_values = cells.values + cell_ids = cell_values.field('id').to_numpy() + struct_starts = cell_values.field('struct_start_offset').to_numpy() + struct_ends = cell_values.field('struct_end_offset').to_numpy() + + refs = cell_values.field('refs') + ref_values = refs.values + ref_offsets = refs.offsets.to_numpy() + targets = ref_values.field('target').to_numpy() + xy = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) + xy0 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) + xy1 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) + counts = gdsii_arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) + invert_y = ref_values.field('invert_y').to_numpy(zero_copy_only=False) + angle_rad = ref_values.field('angle_rad').to_numpy() + scale = ref_values.field('scale').to_numpy() + + ref_payload = _ScanRefs( + offsets=ref_offsets, + targets=targets, + xy=xy, + xy0=xy0, + xy1=xy1, + counts=counts, + invert_y=invert_y, + angle_rad=angle_rad, + scale=scale, + ) + + cell_order = [cell_names[int(cell_id)] for cell_id in cell_ids] + cell_scan: dict[str, _CellScan] = {} + for cc, name in enumerate(cell_order): + ref_start = int(ref_offsets[cc]) + ref_stop = int(ref_offsets[cc + 1]) + children = { + cell_names[int(target)] + for target in targets[ref_start:ref_stop] + } + cell_scan[name] = _CellScan( + cell_id=int(cell_ids[cc]), + struct_range=_StructRange(int(struct_starts[cc]), int(struct_ends[cc])), + ref_start=ref_start, + ref_stop=ref_stop, + children=children, + ) + + return _ScanPayload( + libarr=libarr, + library_info=library_info, + cell_names=cell_names, + cell_order=cell_order, + cells=cell_scan, + refs=ref_payload, + ) + + +def _pattern_children(pat: Pattern) -> set[str]: + return {child for child, refs in pat.refs.items() if child is not None and refs} + + +def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: + if not pat.refs: + return pat + pat.refs = map_targets(pat.refs, remap) + return pat + + +def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: + if isinstance(source, ILibraryView): + return source + return LibraryView(source) + + +def _source_order(source: ILibraryView) -> list[str]: + if isinstance(source, ArrowLibrary): + return list(source.source_order()) + return list(source.keys()) + + +def _make_ref_rows( + xy: NDArray[numpy.integer[Any]], + angle_rad: NDArray[numpy.floating[Any]], + invert_y: NDArray[numpy.bool_ | numpy.bool], + scale: NDArray[numpy.floating[Any]], + ) -> NDArray[numpy.float64]: + rows = numpy.empty((len(xy), 5), dtype=float) + rows[:, :2] = xy + rows[:, 2] = angle_rad + rows[:, 3] = invert_y.astype(float) + rows[:, 4] = scale + return rows + + +def _expand_aref_row( + xy: NDArray[numpy.integer[Any]], + xy0: NDArray[numpy.integer[Any]], + xy1: NDArray[numpy.integer[Any]], + counts: NDArray[numpy.integer[Any]], + angle_rad: float, + invert_y: bool, + scale: float, + ) -> NDArray[numpy.float64]: + a_count = int(counts[0]) + b_count = int(counts[1]) + aa, bb = numpy.meshgrid(numpy.arange(a_count), numpy.arange(b_count), indexing='ij') + displacements = aa.reshape(-1, 1) * xy0[None, :] + bb.reshape(-1, 1) * xy1[None, :] + rows = numpy.empty((displacements.shape[0], 5), dtype=float) + rows[:, :2] = xy + displacements + rows[:, 2] = angle_rad + rows[:, 3] = float(invert_y) + rows[:, 4] = scale + return rows + + +class ArrowLibrary(ILibraryView): + """ + Read-only library backed by the native lazy Arrow scan schema. + + Materializing a cell via `__getitem__` caches a real `Pattern` for that cell. + Cached cells are treated as edited for future writes from this module. + """ + + path: pathlib.Path + library_info: dict[str, Any] + + def __init__( + self, + *, + path: pathlib.Path, + payload: _ScanPayload, + source: _SourceBuffer, + ) -> None: + self.path = path + self.library_info = payload.library_info + self._payload = payload + self._source = source + self._cache: dict[str, Pattern] = {} + + @classmethod + def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: + path = pathlib.Path(filename).expanduser().resolve() + source = _open_source_buffer(path) + scan_arr = gdsii_arrow._scan_buffer_to_arrow(source.data) + assert len(scan_arr) == 1 + payload = _extract_scan_payload(scan_arr[0]) + return cls(path=path, payload=payload, source=source) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._payload.cell_order) + + def __len__(self) -> int: + return len(self._payload.cell_order) + + def __contains__(self, key: object) -> bool: + return key in self._payload.cells + + def source_order(self) -> tuple[str, ...]: + return tuple(self._payload.cell_order) + + def raw_struct_bytes(self, name: str) -> bytes: + struct_range = self._payload.cells[name].struct_range + return self._source.raw_slice(struct_range.start, struct_range.end) + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = self._materialize_patterns(names, persist=persist) + return LibraryView(mats) + + def _materialize_patterns( + self, + names: Sequence[str], + *, + persist: bool, + ) -> dict[str, Pattern]: + ordered_names = list(dict.fromkeys(names)) + missing = [name for name in ordered_names if name not in self._payload.cells] + if missing: + raise KeyError(missing[0]) + + materialized: dict[str, Pattern] = {} + uncached = [name for name in ordered_names if name not in self._cache] + if uncached: + ranges = numpy.asarray( + [ + [ + self._payload.cells[name].struct_range.start, + self._payload.cells[name].struct_range.end, + ] + for name in uncached + ], + dtype=numpy.uint64, + ) + arrow_arr = gdsii_arrow._read_selected_cells_to_arrow(self._source.data, ranges) + assert len(arrow_arr) == 1 + selected_lib, _info = gdsii_arrow.read_arrow(arrow_arr[0]) + for name in uncached: + pat = selected_lib[name] + materialized[name] = pat + if persist: + self._cache[name] = pat + + for name in ordered_names: + if name in self._cache: + materialized[name] = self._cache[name] + return materialized + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + return self._materialize_patterns((name,), persist=persist)[name] + + def _raw_children(self, name: str) -> set[str]: + return set(self._payload.cells[name].children) + + def _collect_raw_transforms(self, cell: _CellScan, target_id: int) -> list[NDArray[numpy.float64]]: + refs = self._payload.refs + start = cell.ref_start + stop = cell.ref_stop + if stop <= start: + return [] + + targets = refs.targets[start:stop] + mask = targets == target_id + if not mask.any(): + return [] + + rows: list[NDArray[numpy.float64]] = [] + counts = refs.counts[start:stop] + unit_mask = mask & (counts[:, 0] == 1) & (counts[:, 1] == 1) + if unit_mask.any(): + rows.append(_make_ref_rows( + refs.xy[start:stop][unit_mask], + refs.angle_rad[start:stop][unit_mask], + refs.invert_y[start:stop][unit_mask], + refs.scale[start:stop][unit_mask], + )) + + aref_indices = numpy.nonzero(mask & ~unit_mask)[0] + for idx in aref_indices: + abs_idx = start + int(idx) + rows.append(_expand_aref_row( + xy=refs.xy[abs_idx], + xy0=refs.xy0[abs_idx], + xy1=refs.xy1[abs_idx], + counts=refs.counts[abs_idx], + angle_rad=float(refs.angle_rad[abs_idx]), + invert_y=bool(refs.invert_y[abs_idx]), + scale=float(refs.scale[abs_idx]), + )) + return rows + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._payload.cell_order: + if name in self._cache: + graph[name] = _pattern_children(self._cache[name]) + else: + graph[name] = self._raw_children(name) + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + graph = self.child_graph(dangling='ignore') + names = set(graph) + not_toplevel: set[str] = set() + for children in graph.values(): + not_toplevel |= children + return list(names - not_toplevel) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + target_id = self._payload.cells.get(name) + for parent in parent_graph.get(name, set()): + if parent in self._cache: + for ref in self._cache[parent].refs.get(name, []): + instances[parent].append(ref.as_transforms()) + continue + + if target_id is None or parent not in self._payload.cells: + continue + rows = self._collect_raw_transforms(self._payload.cells[parent], target_id.cell_id) + if rows: + instances[parent].extend(rows) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + full_path = (parent,) + path + result[full_path] = instances + return result + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + ) -> dict[str, str]: + view = _coerce_library_view(source) + source_order = _source_order(view) + child_graph = view.child_graph(dangling='include') + + source_to_visible: dict[str, str] = {} + visible_to_source: dict[str, str] = {} + rename_map: dict[str, str] = {} + + for name in source_order: + visible = name + if visible in self._entries or visible in visible_to_source: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(self, name) + if visible in self._entries or visible in visible_to_source: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + rename_map[name] = visible + source_to_visible[name] = visible + visible_to_source[visible] = name + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return rename_map + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + remap = lambda target: None if target is None else self._effective_target(layer, target) + pat = _remap_pattern_targets(source_pat, remap) + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = _pattern_children(entry) + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + +def readfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + lib = ArrowLibrary.from_file(filename) + return lib, lib.library_info + + +def load_libraryfile( + filename: str | pathlib.Path, + ) -> tuple[ArrowLibrary, dict[str, Any]]: + return readfile(filename) + + +def _get_write_info( + library: Mapping[str, Pattern] | ILibraryView, + *, + meters_per_unit: float | None, + logical_units_per_unit: float | None, + library_name: str | None, + ) -> tuple[float, float, str]: + if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: + return meters_per_unit, logical_units_per_unit, library_name + + infos: list[dict[str, Any]] = [] + if isinstance(library, ArrowLibrary): + infos.append(library.library_info) + elif isinstance(library, OverlayLibrary): + for layer in library._layers: + if isinstance(layer.library, ArrowLibrary): + infos.append(layer.library.library_info) + + if infos: + unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} + if len(unit_pairs) > 1: + raise LibraryError('Merged lazy GDS sources must have identical units before writing') + info = infos[0] + meters = info['meters_per_unit'] if meters_per_unit is None else meters_per_unit + logical = info['logical_units_per_unit'] if logical_units_per_unit is None else logical_units_per_unit + name = info['name'] if library_name is None else library_name + return meters, logical, name + + if meters_per_unit is None or logical_units_per_unit is None or library_name is None: + raise LibraryError('meters_per_unit, logical_units_per_unit, and library_name are required for non-GDS-backed lazy writes') + return meters_per_unit, logical_units_per_unit, library_name + + +def _can_copy_arrow_cell(library: ArrowLibrary, name: str) -> bool: + return name not in library._cache + + +def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: + layer = library._layers[entry.layer_index] + if not isinstance(layer.library, ArrowLibrary): + return False + if name != entry.source_name: + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(library._effective_target(layer, child) == child for child in children) + + +def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: + elements: list[klamath.elements.Element] = [] + elements += gdsii._shapes_to_elements(pat.shapes) + elements += gdsii._labels_to_texts(pat.labels) + elements += gdsii._mrefs_to_grefs(pat.refs) + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=elements) + + +def write( + library: Mapping[str, Pattern] | ILibraryView, + stream: IO[bytes], + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + meters_per_unit, logical_units_per_unit, library_name = _get_write_info( + library, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + + header = klamath.library.FileHeader( + name=library_name.encode('ASCII'), + user_units_per_db_unit=logical_units_per_unit, + meters_per_db_unit=meters_per_unit, + ) + header.write(stream) + + if isinstance(library, ArrowLibrary): + for name in library.source_order(): + if _can_copy_arrow_cell(library, name): + stream.write(library.raw_struct_bytes(name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + if isinstance(library, OverlayLibrary): + for name in library.source_order(): + entry = library._entries[name] + if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + layer = library._layers[entry.layer_index] + assert isinstance(layer.library, ArrowLibrary) + stream.write(layer.library.raw_struct_bytes(entry.source_name)) + else: + _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) + klamath.records.ENDLIB.write(stream, None) + return + + gdsii.write(cast('Mapping[str, Pattern]', library), stream, meters_per_unit, logical_units_per_unit, library_name) + + +def writefile( + library: Mapping[str, Pattern] | ILibraryView, + filename: str | pathlib.Path, + *, + meters_per_unit: float | None = None, + logical_units_per_unit: float | None = None, + library_name: str | None = None, + ) -> None: + path = pathlib.Path(filename) + + with tmpfile(path) as base_stream: + streams: tuple[Any, ...] = (base_stream,) + if path.suffix == '.gz': + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + streams = (stream,) + streams + else: + stream = base_stream + + try: + write( + library, + stream, + meters_per_unit=meters_per_unit, + logical_units_per_unit=logical_units_per_unit, + library_name=library_name, + ) + finally: + for ss in streams: + ss.close() diff --git a/masque/file/gdsii_perf.py b/masque/file/gdsii_perf.py new file mode 100644 index 0000000..38d1a7d --- /dev/null +++ b/masque/file/gdsii_perf.py @@ -0,0 +1,633 @@ +""" +Synthetic GDS fixture generation for reader/writer performance testing. + +The presets here are intentionally hierarchical and deterministic. They aim to +approximate a pair of real-world layout families discussed during GDS reader and +writer work: + +* `many_cells`: tens of thousands of cells, moderate reference count, very heavy + box usage after flattening, and moderate polygon density. +* `many_instances`: a much smaller cell library with very high reference count, + similar box density, and far fewer polygons. + +Fixtures are written by streaming structures through `klamath` directly so large +benchmark files can be produced without first materializing an equally large +`masque.Library` in Python. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +import argparse +import json +import math + +import numpy +import klamath +from klamath import elements + + +EMPTY_PROPERTIES: dict[int, bytes] = {} +METERS_PER_DB_UNIT = 1e-9 +USER_UNITS_PER_DB_UNIT = 1e-3 +TOTAL_LAYERS = 200 + + +@dataclass(frozen=True) +class FixturePreset: + name: str + total_layers: int + box_layers: int + heavy_box_layers: int + polygon_layers: int + box_cells: int + poly_cells: int + box_wrappers: int + poly_wrappers: int + box_clusters: int + poly_clusters: int + box_cluster_refs: int + poly_cluster_refs: int + top_direct_box_refs: int + top_direct_poly_refs: int + heavy_boxes_per_cell: int + regular_boxes_per_cell: int + polygons_per_cell: int + path_stride: int + text_stride: int + box_cluster_array: tuple[int, int] + top_box_array: tuple[int, int] + poly_cluster_array: tuple[int, int] + top_poly_array: tuple[int, int] + rare_annotation_stride: int + + +PRESETS: dict[str, FixturePreset] = { + 'many_cells': FixturePreset( + name='many_cells', + total_layers=TOTAL_LAYERS, + box_layers=20, + heavy_box_layers=3, + polygon_layers=20, + box_cells=17_000, + poly_cells=6_000, + box_wrappers=18_000, + poly_wrappers=6_000, + box_clusters=2_000, + poly_clusters=999, + box_cluster_refs=24, + poly_cluster_refs=16, + top_direct_box_refs=21_000, + top_direct_poly_refs=7_000, + heavy_boxes_per_cell=6, + regular_boxes_per_cell=2, + polygons_per_cell=50, + path_stride=2, + text_stride=3, + box_cluster_array=(24, 16), + top_box_array=(8, 8), + poly_cluster_array=(4, 2), + top_poly_array=(3, 2), + rare_annotation_stride=1_250, + ), + 'many_instances': FixturePreset( + name='many_instances', + total_layers=TOTAL_LAYERS, + box_layers=25, + heavy_box_layers=3, + polygon_layers=10, + box_cells=2_500, + poly_cells=500, + box_wrappers=1_000, + poly_wrappers=500, + box_clusters=1_000, + poly_clusters=499, + box_cluster_refs=1_200, + poly_cluster_refs=400, + top_direct_box_refs=102_001, + top_direct_poly_refs=0, + heavy_boxes_per_cell=40, + regular_boxes_per_cell=16, + polygons_per_cell=60, + path_stride=1, + text_stride=2, + box_cluster_array=(1, 1), + top_box_array=(1, 1), + poly_cluster_array=(1, 1), + top_poly_array=(1, 1), + rare_annotation_stride=250, + ), + } + + +@dataclass(frozen=True) +class FixtureManifest: + preset: str + scale: float + gds_path: str + library_name: str + cells: int + refs: int + layers: int + box_layers: int + heavy_box_layers: list[list[int]] + polygon_layers: list[list[int]] + hierarchical_boxes_per_heavy_layer: int + hierarchical_boxes_per_regular_layer: int + hierarchical_polygons_total: int + hierarchical_paths_total: int + hierarchical_texts_total: int + flattened_box_placements: int + flattened_poly_placements: int + estimated_flat_boxes_per_heavy_layer: int + estimated_flat_polygons_per_active_polygon_layer: int + + +def _scaled_count(value: int, scale: float, minimum: int = 0) -> int: + if value == 0: + return 0 + scaled = int(math.ceil(value * scale)) + return max(minimum, scaled) + + +def _scaled_preset(preset: FixturePreset, scale: float) -> FixturePreset: + if scale <= 0: + raise ValueError(f'scale must be positive, got {scale!r}') + + return FixturePreset( + name=preset.name, + total_layers=preset.total_layers, + box_layers=min(preset.box_layers, preset.total_layers), + heavy_box_layers=min(preset.heavy_box_layers, preset.box_layers), + polygon_layers=min(preset.polygon_layers, preset.total_layers), + box_cells=_scaled_count(preset.box_cells, scale, minimum=1), + poly_cells=_scaled_count(preset.poly_cells, scale, minimum=1), + box_wrappers=_scaled_count(preset.box_wrappers, scale), + poly_wrappers=_scaled_count(preset.poly_wrappers, scale), + box_clusters=_scaled_count(preset.box_clusters, scale, minimum=1), + poly_clusters=_scaled_count(preset.poly_clusters, scale, minimum=1), + box_cluster_refs=_scaled_count(preset.box_cluster_refs, scale, minimum=1), + poly_cluster_refs=_scaled_count(preset.poly_cluster_refs, scale, minimum=1), + top_direct_box_refs=_scaled_count(preset.top_direct_box_refs, scale), + top_direct_poly_refs=_scaled_count(preset.top_direct_poly_refs, scale), + heavy_boxes_per_cell=max(1, preset.heavy_boxes_per_cell), + regular_boxes_per_cell=max(1, preset.regular_boxes_per_cell), + polygons_per_cell=max(1, preset.polygons_per_cell), + path_stride=max(1, preset.path_stride), + text_stride=max(1, preset.text_stride), + box_cluster_array=preset.box_cluster_array, + top_box_array=preset.top_box_array, + poly_cluster_array=preset.poly_cluster_array, + top_poly_array=preset.top_poly_array, + rare_annotation_stride=max(1, _scaled_count(preset.rare_annotation_stride, scale, minimum=1)), + ) + + +def _rect_xy(xmin: int, ymin: int, xmax: int, ymax: int) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + return numpy.array( + [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]], + dtype=numpy.int32, + ) + + +def _poly_xy(points: list[tuple[int, int]]) -> numpy.ndarray[Any, numpy.dtype[numpy.int32]]: + closed = points + [points[0]] + return numpy.array(closed, dtype=numpy.int32) + + +def _sref( + target: str, + xy: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([xy], dtype=numpy.int32), + colrow=None, + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _aref( + target: str, + origin: tuple[int, int], + counts: tuple[int, int], + step: tuple[int, int], + properties: dict[int, bytes] | None = None, + ) -> elements.Reference: + cols, rows = counts + dx, dy = step + xy = numpy.array( + [ + origin, + (origin[0] + cols * dx, origin[1]), + (origin[0], origin[1] + rows * dy), + ], + dtype=numpy.int32, + ) + return klamath.library.Reference( + struct_name=target.encode('ASCII'), + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=xy, + colrow=(cols, rows), + properties=EMPTY_PROPERTIES if properties is None else properties, + ) + + +def _annotation(index: int) -> dict[int, bytes]: + return {1: f'perf-{index}'.encode('ASCII')} + + +def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 17) * 600 + ybase = (index // 17) * 180 + + for layer in range(cfg.heavy_box_layers): + for box_idx in range(cfg.heavy_boxes_per_cell): + x0 = xbase + box_idx * 22 + y0 = ybase + layer * 40 + width = 10 + ((index + box_idx + layer) % 7) * 6 + height = 10 + ((index * 3 + box_idx + layer) % 5) * 8 + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 and box_idx == 0 and layer == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=properties, + )) + + for layer in range(cfg.heavy_box_layers, cfg.box_layers): + for box_idx in range(cfg.regular_boxes_per_cell): + x0 = xbase + box_idx * 38 + y0 = ybase + (layer - cfg.heavy_box_layers) * 28 + 400 + width = 18 + ((index + layer + box_idx) % 9) * 4 + height = 12 + ((index + 2 * layer + box_idx) % 6) * 5 + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_rect_xy(x0, y0, x0 + width, y0 + height), + properties=EMPTY_PROPERTIES, + )) + + return cell_elements + + +def _make_poly_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: + cell_elements: list[elements.Element] = [] + xbase = (index % 19) * 900 + ybase = (index // 19) * 260 + + for poly_idx in range(cfg.polygons_per_cell): + layer = poly_idx % cfg.polygon_layers + dx = xbase + (poly_idx % 5) * 120 + dy = ybase + (poly_idx // 5) * 80 + size = 18 + ((index + poly_idx + layer) % 11) * 7 + points = [ + (dx, dy), + (dx + size, dy + size // 5), + (dx + size + size // 3, dy + size), + (dx + size // 2, dy + size + size // 2), + (dx - size // 4, dy + size // 2), + ] + properties = _annotation(index) if poly_idx == 0 and index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Boundary( + layer=(layer, 0), + xy=_poly_xy(points), + properties=properties, + )) + + if index % cfg.path_stride == 0: + layer = index % cfg.polygon_layers + cell_elements.append(elements.Path( + layer=(layer, 1), + path_type=2, + width=12 + (index % 5) * 4, + extension=(0, 0), + xy=numpy.array( + [ + [xbase, ybase + 900], + [xbase + 240, ybase + 930], + [xbase + 420, ybase + 960], + ], + dtype=numpy.int32, + ), + properties=EMPTY_PROPERTIES, + )) + + if index % cfg.text_stride == 0: + layer = index % cfg.polygon_layers + properties = _annotation(index) if index % cfg.rare_annotation_stride == 0 else EMPTY_PROPERTIES + cell_elements.append(elements.Text( + layer=(layer, 2), + presentation=0, + path_type=0, + width=0, + invert_y=False, + mag=1.0, + angle_deg=0.0, + xy=numpy.array([[xbase + 64, ybase + 1536]], dtype=numpy.int32), + string=f'T{index:05d}'.encode('ASCII'), + properties=properties, + )) + + return cell_elements + + +def _write_struct(stream: Any, name: str, cell_elements: list[elements.Element]) -> None: + klamath.library.write_struct(stream, name=name.encode('ASCII'), elements=cell_elements) + + +def _box_name(index: int) -> str: + return f'box_{index:05d}' + + +def _poly_name(index: int) -> str: + return f'poly_{index:05d}' + + +def _box_wrapper_name(index: int) -> str: + return f'box_wrap_{index:05d}' + + +def _poly_wrapper_name(index: int) -> str: + return f'poly_wrap_{index:05d}' + + +def _box_cluster_name(index: int) -> str: + return f'box_cluster_{index:05d}' + + +def _poly_cluster_name(index: int) -> str: + return f'poly_cluster_{index:05d}' + + +def _write_box_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_cells): + _write_struct(stream, _box_name(idx), _make_box_cell(_box_name(idx), idx, cfg)) + + +def _write_poly_cells(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.poly_cells): + _write_struct(stream, _poly_name(idx), _make_poly_cell(_poly_name(idx), idx, cfg)) + + +def _write_wrappers(stream: Any, cfg: FixturePreset) -> None: + for idx in range(cfg.box_wrappers): + target = _box_name(idx % cfg.box_cells) + origin = ((idx % 97) * 2_000, (idx // 97) * 2_000) + _write_struct(stream, _box_wrapper_name(idx), [_sref(target, origin)]) + + for idx in range(cfg.poly_wrappers): + target = _poly_name(idx % cfg.poly_cells) + origin = ((idx % 61) * 3_200, (idx // 61) * 3_200) + _write_struct(stream, _poly_wrapper_name(idx), [_sref(target, origin)]) + + +def _write_box_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + for idx in range(cfg.box_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.box_cluster_refs): + target = _box_name((idx * cfg.box_cluster_refs + ref_idx) % cfg.box_cells) + origin = ( + (ref_idx % 6) * 48_000, + (ref_idx // 6) * 48_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.box_cluster_array, (720, 900))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _box_cluster_name(idx), cell_elements) + + +def _write_poly_clusters(stream: Any, cfg: FixturePreset) -> None: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + for idx in range(cfg.poly_clusters): + cell_elements: list[elements.Element] = [] + for ref_idx in range(cfg.poly_cluster_refs): + target = _poly_name((idx * cfg.poly_cluster_refs + ref_idx) % cfg.poly_cells) + origin = ( + (ref_idx % 10) * 96_000, + (ref_idx // 10) * 96_000, + ) + if ref_idx < array_refs: + cell_elements.append(_aref(target, origin, cfg.poly_cluster_array, (12_000, 8_500))) + else: + cell_elements.append(_sref(target, origin)) + _write_struct(stream, _poly_cluster_name(idx), cell_elements) + + +def _top_box_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.box_wrappers): + refs.append(_sref( + _box_wrapper_name(idx), + ((idx % 240) * 240_000, (idx // 240) * 240_000), + )) + + for idx in range(cfg.box_clusters): + refs.append(_sref( + _box_cluster_name(idx), + ((idx % 100) * 800_000, (idx // 100) * 800_000 + 14_000_000), + )) + + for idx in range(cfg.top_direct_box_refs): + target = _box_name(idx % cfg.box_cells) + origin = ( + (idx % 150) * 160_000, + (idx // 150) * 160_000 + 26_000_000, + ) + if cfg.top_box_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_box_array, (1_100, 1_350))) + + return refs + + +def _top_poly_refs(cfg: FixturePreset) -> list[elements.Reference]: + refs: list[elements.Reference] = [] + + for idx in range(cfg.poly_wrappers): + refs.append(_sref( + _poly_wrapper_name(idx), + ((idx % 180) * 360_000, (idx // 180) * 360_000 + 44_000_000), + )) + + for idx in range(cfg.poly_clusters): + refs.append(_sref( + _poly_cluster_name(idx), + ((idx % 70) * 1_100_000, (idx // 70) * 1_100_000 + 58_000_000), + )) + + for idx in range(cfg.top_direct_poly_refs): + target = _poly_name(idx % cfg.poly_cells) + origin = ( + (idx % 110) * 420_000, + (idx // 110) * 420_000 + 72_000_000, + ) + if cfg.top_poly_array == (1, 1): + refs.append(_sref(target, origin)) + else: + refs.append(_aref(target, origin, cfg.top_poly_array, (16_000, 14_000))) + + return refs + + +def _write_top(stream: Any, cfg: FixturePreset) -> None: + cell_elements: list[elements.Element] = [] + cell_elements.extend(_top_box_refs(cfg)) + cell_elements.extend(_top_poly_refs(cfg)) + _write_struct(stream, 'TOP', cell_elements) + + +def _poly_paths_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.path_stride + 1 + + +def _poly_texts_total(cfg: FixturePreset) -> int: + return (cfg.poly_cells - 1) // cfg.text_stride + 1 + + +def _ref_instances_per_box_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] + return array_refs * array_mult + (cfg.box_cluster_refs - array_refs) + + +def _ref_instances_per_poly_cluster(cfg: FixturePreset) -> int: + array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] + return array_refs * array_mult + (cfg.poly_cluster_refs - array_refs) + + +def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> FixtureManifest: + base = PRESETS[preset] + cfg = _scaled_preset(base, scale) + + flattened_box_placements = ( + cfg.box_wrappers + + cfg.box_clusters * _ref_instances_per_box_cluster(cfg) + + cfg.top_direct_box_refs * cfg.top_box_array[0] * cfg.top_box_array[1] + ) + flattened_poly_placements = ( + cfg.poly_wrappers + + cfg.poly_clusters * _ref_instances_per_poly_cluster(cfg) + + cfg.top_direct_poly_refs * cfg.top_poly_array[0] * cfg.top_poly_array[1] + ) + polygon_layers = max(1, cfg.polygon_layers) + polys_per_layer = (cfg.poly_cells * cfg.polygons_per_cell) // polygon_layers + + return FixtureManifest( + preset=cfg.name, + scale=scale, + gds_path=str(Path(path)), + library_name=f'masque-perf-{cfg.name}', + cells=cfg.box_cells + cfg.poly_cells + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + 1, + refs=( + cfg.box_wrappers + + cfg.poly_wrappers + + cfg.box_clusters * cfg.box_cluster_refs + + cfg.poly_clusters * cfg.poly_cluster_refs + + cfg.box_wrappers + cfg.poly_wrappers + cfg.box_clusters + cfg.poly_clusters + + cfg.top_direct_box_refs + cfg.top_direct_poly_refs + ), + layers=cfg.total_layers, + box_layers=cfg.box_layers, + heavy_box_layers=[[layer, 0] for layer in range(cfg.heavy_box_layers)], + polygon_layers=[[layer, 0] for layer in range(cfg.polygon_layers)], + hierarchical_boxes_per_heavy_layer=cfg.box_cells * cfg.heavy_boxes_per_cell, + hierarchical_boxes_per_regular_layer=cfg.box_cells * cfg.regular_boxes_per_cell, + hierarchical_polygons_total=cfg.poly_cells * cfg.polygons_per_cell, + hierarchical_paths_total=_poly_paths_total(cfg), + hierarchical_texts_total=_poly_texts_total(cfg), + flattened_box_placements=flattened_box_placements, + flattened_poly_placements=flattened_poly_placements, + estimated_flat_boxes_per_heavy_layer=flattened_box_placements * cfg.heavy_boxes_per_cell, + estimated_flat_polygons_per_active_polygon_layer=flattened_poly_placements * polys_per_layer // cfg.poly_cells if cfg.poly_cells else 0, + ) + + +def write_fixture( + path: str | Path, + *, + preset: str, + scale: float = 1.0, + write_manifest: bool = True, + ) -> FixtureManifest: + if preset not in PRESETS: + known = ', '.join(sorted(PRESETS)) + raise KeyError(f'unknown preset {preset!r}; expected one of: {known}') + + manifest = fixture_manifest(path, preset, scale) + cfg = _scaled_preset(PRESETS[preset], scale) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + + with output.open('wb') as stream: + header = klamath.library.FileHeader( + name=manifest.library_name.encode('ASCII'), + user_units_per_db_unit=USER_UNITS_PER_DB_UNIT, + meters_per_db_unit=METERS_PER_DB_UNIT, + ) + header.write(stream) + _write_box_cells(stream, cfg) + _write_poly_cells(stream, cfg) + _write_wrappers(stream, cfg) + _write_box_clusters(stream, cfg) + _write_poly_clusters(stream, cfg) + _write_top(stream, cfg) + klamath.records.ENDLIB.write(stream, None) + + if write_manifest: + manifest_path = output.with_suffix(output.suffix + '.json') + manifest_path.write_text(json.dumps(asdict(manifest), indent=2, sort_keys=True) + '\n') + + return manifest + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description='Generate synthetic GDS fixtures for GDS reader/writer performance work.') + parser.add_argument( + 'preset', + nargs='?', + default='many_cells', + choices=sorted(PRESETS), + help='Fixture family to generate.', + ) + parser.add_argument( + 'output', + nargs='?', + help='Output .gds path. Defaults to build/gds_perf/.gds', + ) + parser.add_argument( + '--scale', + type=float, + default=1.0, + help='Scale the preset counts down or up while keeping the same shape mix. Default: 1.0', + ) + parser.add_argument( + '--no-manifest', + action='store_true', + help='Do not write the sidecar JSON manifest.', + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + output = Path(args.output) if args.output is not None else Path('build/gds_perf') / f'{args.preset}.gds' + manifest = write_fixture(output, preset=args.preset, scale=args.scale, write_manifest=not args.no_manifest) + print(json.dumps(asdict(manifest), indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/masque/file/svg.py b/masque/file/svg.py index 621bcdb..772aa39 100644 --- a/masque/file/svg.py +++ b/masque/file/svg.py @@ -45,6 +45,10 @@ def _make_svg_ids(names: Mapping[str, Pattern]) -> dict[str, str]: return svg_ids +def _detached_library(library: Mapping[str, Pattern]) -> dict[str, Pattern]: + return {name: pat.deepcopy() for name, pat in library.items()} + + def writefile( library: Mapping[str, Pattern], top: str, @@ -53,13 +57,12 @@ def writefile( annotate_ports: bool = False, ) -> None: """ - Write a Pattern to an SVG file, by first calling .polygonize() on it + Write a Pattern to an SVG file, by first calling .polygonize() on a detached + materialized copy to change the shapes into polygons, and then writing patterns as SVG groups (, inside ), polygons as paths (), and refs as elements. - Note that this function modifies the Pattern. - If `custom_attributes` is `True`, a non-standard `pattern_layer` attribute is written to the relevant elements. @@ -71,19 +74,21 @@ def writefile( prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. custom_attributes: Whether to write non-standard `pattern_layer` attribute to the SVG elements. annotate_ports: If True, draw an arrow for each port (similar to `Pattern.visualize(..., ports=True)`). """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize pattern pattern.polygonize() - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) @@ -96,10 +101,10 @@ def writefile( # Create file svg = svgwrite.Drawing(filename, profile='full', viewBox=viewbox_string, debug=(not custom_attributes)) - svg_ids = _make_svg_ids(library) + svg_ids = _make_svg_ids(detached) # Now create a group for each pattern and add in any Boundary and Use elements - for name, pat in library.items(): + for name, pat in detached.items(): svg_group = svg.g(id=svg_ids[name], fill='blue', stroke='red') for layer, shapes in pat.shapes.items(): @@ -158,21 +163,21 @@ def writefile_inverted( box and drawing the polygons with reverse vertex order inside it, all within one `` element. - Note that this function modifies the Pattern. - If you want pattern polygonized with non-default arguments, just call `pattern.polygonize()` prior to calling this function. Args: - pattern: Pattern to write to file. Modified by this function. + library: Mapping of pattern names to patterns. + top: Name of the top-level pattern to render. filename: Filename to write to. """ - pattern = library[top] + detached = _detached_library(library) + pattern = detached[top] # Polygonize and flatten pattern - pattern.polygonize().flatten(library) + pattern.polygonize().flatten(detached) - bounds = pattern.get_bounds(library=library) + bounds = pattern.get_bounds(library=detached) if bounds is None: bounds_min, bounds_max = numpy.array([[-1, -1], [1, 1]]) logger.warning('Pattern had no bounds (empty?); setting arbitrary viewbox', stacklevel=1) diff --git a/masque/label.py b/masque/label.py index 6a2a61f..d220fee 100644 --- a/masque/label.py +++ b/masque/label.py @@ -53,6 +53,22 @@ class Label(PositionableImpl, RepeatableImpl, AnnotatableImpl, Bounded, Pivotabl self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + string: str, + *, + offset: NDArray[numpy.float64], + repetition: Repetition | None = None, + annotations: annotations_t | None = None, + ) -> Self: + new = cls.__new__(cls) + new._string = string + new._offset = offset + new._repetition = repetition + new._annotations = annotations + return new + def __copy__(self) -> Self: return type(self)( string=self.string, diff --git a/masque/ref.py b/masque/ref.py index 268d8d4..0cc911f 100644 --- a/masque/ref.py +++ b/masque/ref.py @@ -86,6 +86,26 @@ class Ref( self.repetition = repetition self.annotations = annotations if annotations is not None else {} + @classmethod + def _from_raw( + cls, + *, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + scale: float, + repetition: Repetition | None, + annotations: annotations_t | None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._rotation = rotation % (2 * pi) + new._scale = scale + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations + return new + def __copy__(self) -> 'Ref': new = Ref( offset=self.offset.copy(), diff --git a/masque/repetition.py b/masque/repetition.py index 1e12fcd..9e8af26 100644 --- a/masque/repetition.py +++ b/masque/repetition.py @@ -113,6 +113,22 @@ class Grid(Repetition): self.a_count = a_count self.b_count = b_count + @classmethod + def _from_raw( + cls: type[GG], + *, + a_vector: NDArray[numpy.float64], + a_count: int, + b_vector: NDArray[numpy.float64], + b_count: int, + ) -> GG: + new = cls.__new__(cls) + new._a_vector = a_vector + new._b_vector = b_vector + new._a_count = int(a_count) + new._b_count = int(b_count) + return new + @classmethod def aligned( cls: type[GG], diff --git a/masque/shapes/__init__.py b/masque/shapes/__init__.py index fd66c59..ac3a14b 100644 --- a/masque/shapes/__init__.py +++ b/masque/shapes/__init__.py @@ -11,6 +11,7 @@ from .shape import ( from .polygon import Polygon as Polygon from .poly_collection import PolyCollection as PolyCollection +from .rect_collection import RectCollection as RectCollection from .circle import Circle as Circle from .ellipse import Ellipse as Ellipse from .arc import Arc as Arc diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index 00c5714..9d5f65d 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -159,27 +159,36 @@ class Arc(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(angles, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._angles = angles - self._width = width - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.angles = angles - self.width = width - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.angles = angles + self.width = width + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + angles: NDArray[numpy.float64], + width: float, + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Arc': + new = cls.__new__(cls) + new._radii = radii + new._angles = angles + new._width = width + new._offset = offset + new._rotation = rotation % (2 * pi) + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Arc': memo = {} if memo is None else memo diff --git a/masque/shapes/circle.py b/masque/shapes/circle.py index 19f3f2b..d7591db 100644 --- a/masque/shapes/circle.py +++ b/masque/shapes/circle.py @@ -50,19 +50,27 @@ class Circle(PositionableImpl, Shape): offset: ArrayLike = (0.0, 0.0), repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._radius = radius - self._offset = offset - self._repetition = repetition - self._annotations = annotations - else: - self.radius = radius - self.offset = offset - self.repetition = repetition - self.annotations = annotations + self.radius = radius + self.offset = offset + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radius: float, + offset: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> 'Circle': + new = cls.__new__(cls) + new._radius = radius + new._offset = offset + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> 'Circle': memo = {} if memo is None else memo diff --git a/masque/shapes/ellipse.py b/masque/shapes/ellipse.py index 38799da..52a3297 100644 --- a/masque/shapes/ellipse.py +++ b/masque/shapes/ellipse.py @@ -95,22 +95,30 @@ class Ellipse(PositionableImpl, Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(radii, numpy.ndarray) - assert isinstance(offset, numpy.ndarray) - self._radii = radii - self._offset = offset - self._rotation = rotation - self._repetition = repetition - self._annotations = annotations - else: - self.radii = radii - self.offset = offset - self.rotation = rotation - self.repetition = repetition - self.annotations = annotations + self.radii = radii + self.offset = offset + self.rotation = rotation + self.repetition = repetition + self.annotations = annotations + + @classmethod + def _from_raw( + cls, + *, + radii: NDArray[numpy.float64], + offset: NDArray[numpy.float64], + rotation: float, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._radii = radii + new._offset = offset + new._rotation = rotation % pi + new._repetition = repetition + new._annotations = annotations + return new def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo diff --git a/masque/shapes/path.py b/masque/shapes/path.py index 7038309..a1e04af 100644 --- a/masque/shapes/path.py +++ b/masque/shapes/path.py @@ -201,34 +201,43 @@ class Path(Shape): rotation: float = 0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: self._cap_extensions = None # Since .cap setter might access it - if raw: - assert isinstance(vertices, numpy.ndarray) - assert isinstance(cap_extensions, numpy.ndarray) or cap_extensions is None - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - self._width = width - self._cap = cap - self._cap_extensions = cap_extensions + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations + self._cap = cap + if cap == PathCap.SquareCustom and cap_extensions is None: + self._cap_extensions = numpy.zeros(2) else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations - self._cap = cap - if cap == PathCap.SquareCustom and cap_extensions is None: - self._cap_extensions = numpy.zeros(2) - else: - self.cap_extensions = cap_extensions - self.width = width + self.cap_extensions = cap_extensions + self.width = width if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + width: float, + cap: PathCap, + cap_extensions: NDArray[numpy.float64] | None = None, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._width = width + new._cap = cap + new._cap_extensions = cap_extensions + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Path': memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 6c23da7..f1c840a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -34,7 +34,7 @@ class PolyCollection(Shape): _vertex_lists: NDArray[numpy.float64] """ 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """ - _vertex_offsets: NDArray[numpy.intp] + _vertex_offsets: NDArray[numpy.integer[Any]] """ 1D NDArray specifying the starting offset for each polygon """ @property @@ -45,7 +45,7 @@ class PolyCollection(Shape): return self._vertex_lists @property - def vertex_offsets(self) -> NDArray[numpy.intp]: + def vertex_offsets(self) -> NDArray[numpy.integer[Any]]: """ Starting offset (in `vertex_lists`) for each polygon """ @@ -63,7 +63,7 @@ class PolyCollection(Shape): chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]), strict=True, ): - yield slice(ii, ff) + yield slice(int(ii), int(ff)) @property def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: @@ -100,25 +100,32 @@ class PolyCollection(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertex_lists, numpy.ndarray) - assert isinstance(vertex_offsets, numpy.ndarray) - self._vertex_lists = vertex_lists - self._vertex_offsets = vertex_offsets - self._repetition = repetition - self._annotations = annotations - else: - self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) - self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) - self.repetition = repetition - self.annotations = annotations + self._vertex_lists = numpy.asarray(vertex_lists, dtype=float) + self._vertex_offsets = numpy.asarray(vertex_offsets, dtype=numpy.intp) + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertex_lists: NDArray[numpy.float64], + vertex_offsets: NDArray[numpy.integer[Any]], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertex_lists = vertex_lists + new._vertex_offsets = vertex_offsets + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) @@ -132,7 +139,7 @@ class PolyCollection(Shape): return ( type(self) is type(other) and numpy.array_equal(self._vertex_lists, other._vertex_lists) - and numpy.array_equal(self._vertex_offsets, other._vertex_offsets) + and numpy.array_equal(self.vertex_offsets, other.vertex_offsets) and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -215,11 +222,11 @@ class PolyCollection(Shape): # TODO: normalize mirroring? - return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()), + return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()), (offset, scale / norm_value, rotation, False), lambda: PolyCollection( vertex_lists=rotated_vertices * norm_value, - vertex_offsets=self._vertex_offsets.copy(), + vertex_offsets=self.vertex_offsets.copy(), ), ) diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index a743de2..06e5c2b 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -115,22 +115,29 @@ class Polygon(Shape): rotation: float = 0.0, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(vertices, numpy.ndarray) - self._vertices = vertices - self._repetition = repetition - self._annotations = annotations - else: - self.vertices = vertices - self.repetition = repetition - self.annotations = annotations + self.vertices = vertices + self.repetition = repetition + self.annotations = annotations if rotation: self.rotate(rotation) if numpy.any(offset): self.translate(offset) + @classmethod + def _from_raw( + cls, + *, + vertices: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._vertices = vertices + new._repetition = repetition + new._annotations = annotations + return new + def __deepcopy__(self, memo: dict | None = None) -> 'Polygon': memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/shapes/rect_collection.py b/masque/shapes/rect_collection.py new file mode 100644 index 0000000..eaf028f --- /dev/null +++ b/masque/shapes/rect_collection.py @@ -0,0 +1,249 @@ +from typing import Any, cast, Self +from collections.abc import Iterator +import copy +import functools + +import numpy +from numpy import pi +from numpy.typing import NDArray, ArrayLike + +from . import Shape, normalized_shape_tuple +from .polygon import Polygon +from ..error import PatternError +from ..repetition import Repetition +from ..utils import annotations_lt, annotations_eq, rep2key, annotations_t + + +def _normalize_rects(rects: ArrayLike) -> NDArray[numpy.float64]: + arr = numpy.asarray(rects, dtype=float) + if arr.ndim != 2 or arr.shape[1] != 4: + raise PatternError('Rectangles must be an Nx4 array of [xmin, ymin, xmax, ymax]') + if numpy.any(arr[:, 0] > arr[:, 2]) or numpy.any(arr[:, 1] > arr[:, 3]): + raise PatternError('Rectangles must satisfy xmin <= xmax and ymin <= ymax') + if arr.shape[0] <= 1: + return arr + order = numpy.lexsort((arr[:, 3], arr[:, 2], arr[:, 1], arr[:, 0])) + return arr[order] + + +def _renormalize_rects_in_place(rects: NDArray[numpy.float64]) -> None: + x0 = numpy.minimum(rects[:, 0], rects[:, 2]) + x1 = numpy.maximum(rects[:, 0], rects[:, 2]) + y0 = numpy.minimum(rects[:, 1], rects[:, 3]) + y1 = numpy.maximum(rects[:, 1], rects[:, 3]) + rects[:, 0] = x0 + rects[:, 1] = y0 + rects[:, 2] = x1 + rects[:, 3] = y1 + + +@functools.total_ordering +class RectCollection(Shape): + """ + A collection of axis-aligned rectangles, stored as an Nx4 array of + `[xmin, ymin, xmax, ymax]` rows. + """ + __slots__ = ( + '_rects', + '_repetition', '_annotations', + ) + + _rects: NDArray[numpy.float64] + + @property + def rects(self) -> NDArray[numpy.float64]: + return self._rects + + @rects.setter + def rects(self, val: ArrayLike) -> None: + self._rects = _normalize_rects(val) + + @property + def offset(self) -> NDArray[numpy.float64]: + return numpy.zeros(2) + + @offset.setter + def offset(self, val: ArrayLike) -> None: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + + def set_offset(self, val: ArrayLike) -> Self: + if numpy.any(val): + raise PatternError('RectCollection offset is forced to (0, 0)') + return self + + def translate(self, offset: ArrayLike) -> Self: + delta = numpy.asarray(offset, dtype=float).reshape(2) + self._rects[:, [0, 2]] += delta[0] + self._rects[:, [1, 3]] += delta[1] + return self + + def __init__( + self, + rects: ArrayLike, + *, + offset: ArrayLike = (0.0, 0.0), + rotation: float = 0.0, + repetition: Repetition | None = None, + annotations: annotations_t = None, + ) -> None: + self.rects = rects + self.repetition = repetition + self.annotations = annotations + if rotation: + self.rotate(rotation) + if numpy.any(offset): + self.translate(offset) + + @classmethod + def _from_raw( + cls, + *, + rects: NDArray[numpy.float64], + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._rects = rects + new._repetition = repetition + new._annotations = annotations + return new + + @property + def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: + for rect in self._rects: + xmin, ymin, xmax, ymax = rect + yield numpy.array([ + [xmin, ymin], + [xmin, ymax], + [xmax, ymax], + [xmax, ymin], + ], dtype=float) + + def __deepcopy__(self, memo: dict | None = None) -> Self: + memo = {} if memo is None else memo + new = copy.copy(self) + new._rects = self._rects.copy() + new._repetition = copy.deepcopy(self._repetition, memo) + new._annotations = copy.deepcopy(self._annotations) + return new + + def _sorted_rects(self) -> NDArray[numpy.float64]: + if self._rects.shape[0] <= 1: + return self._rects + order = numpy.lexsort((self._rects[:, 3], self._rects[:, 2], self._rects[:, 1], self._rects[:, 0])) + return self._rects[order] + + def __eq__(self, other: Any) -> bool: + return ( + type(self) is type(other) + and numpy.array_equal(self._sorted_rects(), other._sorted_rects()) + and self.repetition == other.repetition + and annotations_eq(self.annotations, other.annotations) + ) + + def __lt__(self, other: Shape) -> bool: + if type(self) is not type(other): + if repr(type(self)) != repr(type(other)): + return repr(type(self)) < repr(type(other)) + return id(type(self)) < id(type(other)) + + other = cast('RectCollection', other) + self_rects = self._sorted_rects() + other_rects = other._sorted_rects() + if not numpy.array_equal(self_rects, other_rects): + min_len = min(self_rects.shape[0], other_rects.shape[0]) + eq_mask = self_rects[:min_len] != other_rects[:min_len] + eq_lt = self_rects[:min_len] < other_rects[:min_len] + eq_lt_masked = eq_lt[eq_mask] + if eq_lt_masked.size > 0: + return bool(eq_lt_masked.flat[0]) + return self_rects.shape[0] < other_rects.shape[0] + if self.repetition != other.repetition: + return rep2key(self.repetition) < rep2key(other.repetition) + return annotations_lt(self.annotations, other.annotations) + + def to_polygons( + self, + num_vertices: int | None = None, # unused # noqa: ARG002 + max_arclen: float | None = None, # unused # noqa: ARG002 + ) -> list[Polygon]: + return [ + Polygon( + vertices=vertices, + repetition=copy.deepcopy(self.repetition), + annotations=copy.deepcopy(self.annotations), + ) + for vertices in self.polygon_vertices + ] + + def get_bounds_single(self) -> NDArray[numpy.float64] | None: + if self._rects.size == 0: + return None + mins = self._rects[:, :2].min(axis=0) + maxs = self._rects[:, 2:].max(axis=0) + return numpy.vstack((mins, maxs)) + + def rotate(self, theta: float) -> Self: + quarter_turns = int(numpy.rint(theta / (pi / 2))) + if not numpy.isclose(theta, quarter_turns * (pi / 2)): + raise PatternError('RectCollection only supports Manhattan rotations') + turns = quarter_turns % 4 + if turns == 0 or self._rects.size == 0: + return self + + corners = numpy.stack(( + self._rects[:, [0, 1]], + self._rects[:, [0, 3]], + self._rects[:, [2, 3]], + self._rects[:, [2, 1]], + ), axis=1) + flat = corners.reshape(-1, 2) + if turns == 1: + rotated = numpy.column_stack((-flat[:, 1], flat[:, 0])) + elif turns == 2: + rotated = -flat + else: + rotated = numpy.column_stack((flat[:, 1], -flat[:, 0])) + corners = rotated.reshape(corners.shape) + self._rects[:, 0] = corners[:, :, 0].min(axis=1) + self._rects[:, 1] = corners[:, :, 1].min(axis=1) + self._rects[:, 2] = corners[:, :, 0].max(axis=1) + self._rects[:, 3] = corners[:, :, 1].max(axis=1) + return self + + def mirror(self, axis: int = 0) -> Self: + if axis not in (0, 1): + raise PatternError('Axis must be 0 or 1') + if axis == 0: + self._rects[:, [1, 3]] *= -1 + else: + self._rects[:, [0, 2]] *= -1 + _renormalize_rects_in_place(self._rects) + return self + + def scale_by(self, c: float) -> Self: + self._rects *= c + _renormalize_rects_in_place(self._rects) + return self + + def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + rects = self._sorted_rects() + centers = 0.5 * (rects[:, :2] + rects[:, 2:]) + offset = centers.mean(axis=0) + zeroed = rects.copy() + zeroed[:, [0, 2]] -= offset[0] + zeroed[:, [1, 3]] -= offset[1] + normed = zeroed / norm_value + return ( + (type(self), normed.data.tobytes()), + (offset, 1.0, 0.0, False), + lambda: RectCollection(rects=normed * norm_value), + ) + + def __repr__(self) -> str: + if self._rects.size == 0: + return '' + centers = 0.5 * (self._rects[:, :2] + self._rects[:, 2:]) + centroid = centers.mean(axis=0) + return f'' diff --git a/masque/shapes/text.py b/masque/shapes/text.py index 5047dc4..c078879 100644 --- a/masque/shapes/text.py +++ b/masque/shapes/text.py @@ -73,27 +73,40 @@ class Text(PositionableImpl, RotatableImpl, Shape): mirrored: bool = False, repetition: Repetition | None = None, annotations: annotations_t = None, - raw: bool = False, ) -> None: - if raw: - assert isinstance(offset, numpy.ndarray) - self._offset = offset - self._string = string - self._height = height - self._rotation = rotation - self._mirrored = mirrored - self._repetition = repetition - self._annotations = annotations - else: - self.offset = offset - self.string = string - self.height = height - self.rotation = rotation - self.mirrored = mirrored - self.repetition = repetition - self.annotations = annotations + self.offset = offset + self.string = string + self.height = height + self.rotation = rotation + self.mirrored = mirrored + self.repetition = repetition + self.annotations = annotations self.font_path = font_path + @classmethod + def _from_raw( + cls, + *, + string: str, + height: float, + font_path: str, + offset: NDArray[numpy.float64], + rotation: float, + mirrored: bool, + annotations: annotations_t = None, + repetition: Repetition | None = None, + ) -> Self: + new = cls.__new__(cls) + new._offset = offset + new._string = string + new._height = height + new._rotation = rotation % (2 * pi) + new._mirrored = mirrored + new._repetition = repetition + new._annotations = annotations + new.font_path = font_path + return new + def __deepcopy__(self, memo: dict | None = None) -> Self: memo = {} if memo is None else memo new = copy.copy(self) diff --git a/masque/test/test_file_roundtrip.py b/masque/test/test_file_roundtrip.py index 2cfb0d1..283a863 100644 --- a/masque/test/test_file_roundtrip.py +++ b/masque/test/test_file_roundtrip.py @@ -5,7 +5,7 @@ from numpy.testing import assert_allclose from ..pattern import Pattern from ..library import Library -from ..shapes import Path as MPath, Circle, Polygon +from ..shapes import Path as MPath, Circle, Polygon, RectCollection from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: @@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: assert poly.repetition is not None assert isinstance(poly.repetition, Grid) assert poly.repetition.a_count == 5 + + +def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None: + from ..file import gdsii + + lib = Library() + pat = Pattern() + pat.shapes[(5, 0)].append( + RectCollection( + rects=[[0, 0, 10, 5], [20, -5, 30, 10]], + annotations={'1': ['rects']}, + ) + ) + lib['rects'] = pat + + gds_file = tmp_path / 'rect_collection.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + read_lib, _ = gdsii.readfile(gds_file) + polys = read_lib['rects'].shapes[(5, 0)] + + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]]) + assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]]) + assert polys[0].annotations == {'1': ['rects']} + assert polys[1].annotations == {'1': ['rects']} diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py new file mode 100644 index 0000000..b2962f8 --- /dev/null +++ b/masque/test/test_gdsii_arrow.py @@ -0,0 +1,603 @@ +from pathlib import Path +import subprocess +import sys +import textwrap + +import klamath +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from .. import Ref, Label, PatternError +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection +from ..file import gdsii, gdsii_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: + if not annotations: + return None + return tuple(sorted((key, tuple(values)) for key, values in annotations.items())) + + +def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(values, dtype=float)).astype(int) + if arr.ndim == 1: + return tuple(arr.tolist()) + return tuple(tuple(row.tolist()) for row in arr) + + +def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int) + rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])] + rev = arr[::-1] + rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0])) + return min(rows) + + +def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: + if isinstance(shape, MPath): + cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) + return [( + 'path', + layer, + _coord_key(shape.vertices), + _coord_key(shape.offset), + int(round(float(shape.width))), + shape.cap.name, + cap_extensions, + _annotations_key(shape.annotations), + )] + + keys = [] + for poly in shape.to_polygons(): + keys.append(( + 'polygon', + layer, + _canonical_polygon_key(poly.vertices), + _coord_key(poly.offset), + _annotations_key(poly.annotations), + )) + return keys + + +def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]: + keys = [] + for transform in ref.as_transforms(): + keys.append(( + target, + _coord_key(transform[:2]), + round(float(transform[2]), 8), + round(float(transform[4]), 8), + bool(int(round(float(transform[3])))), + _annotations_key(ref.annotations), + )) + return keys + + +def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: + return ( + layer, + label.string, + _coord_key(label.offset), + _annotations_key(label.annotations), + ) + + +def _pattern_summary(pattern: Pattern) -> dict[str, object]: + shape_keys: list[tuple[object, ...]] = [] + for layer, shapes in pattern.shapes.items(): + for shape in shapes: + shape_keys.extend(_shape_key(shape, layer)) + + ref_keys: list[tuple[object, ...]] = [] + for target, refs in pattern.refs.items(): + for ref in refs: + ref_keys.extend(_ref_keys(target, ref)) + + label_keys = [ + _label_key(layer, label) + for layer, labels in pattern.labels.items() + for label in labels + ] + + return { + 'shapes': sorted(shape_keys), + 'refs': sorted(ref_keys), + 'labels': sorted(label_keys), + } + + +def _library_summary(lib: Library) -> dict[str, dict[str, object]]: + return {name: _pattern_summary(pattern) for name, pattern in lib.items()} + + +def _make_arrow_test_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]], annotations={'1': ['leaf-poly']}) + leaf.polygon((2, 0), vertices=[[40, 0], [50, 0], [50, 10], [40, 10]]) + leaf.polygon((1, 0), vertices=[[20, 0], [30, 0], [30, 10], [20, 10]]) + leaf.polygon((1, 0), vertices=[[80, 0], [90, 0], [90, 10], [80, 10]]) + leaf.polygon((2, 0), vertices=[[60, 0], [70, 0], [70, 10], [60, 10]], annotations={'18': ['leaf-poly-2']}) + leaf.label((10, 0), string='LEAF', offset=(3, 4), annotations={'10': ['leaf-label']}) + lib['leaf'] = leaf + + child = Pattern() + child.path( + (2, 0), + vertices=[[0, 0], [15, 5], [30, 5]], + width=6, + cap=MPath.Cap.SquareCustom, + cap_extensions=(2, 4), + annotations={'2': ['child-path']}, + ) + child.label((11, 0), string='CHILD', offset=(7, 8), annotations={'11': ['child-label']}) + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25, annotations={'12': ['child-ref']}) + lib['child'] = child + + sibling = Pattern() + sibling.polygon((3, 0), vertices=[[0, 0], [5, 0], [5, 6], [0, 6]]) + sibling.label((12, 0), string='SIB', offset=(1, 2), annotations={'13': ['sib-label']}) + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + annotations={'14': ['sib-ref']}, + ) + lib['sibling'] = sibling + + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(20, 0)) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2)) + fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2)) + fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']}) + fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2), + annotations={'20': ['fanout-aref']}) + lib['fanout'] = fanout + + top = Pattern() + top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.ref('fanout', offset=(250, -75)) + top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) + lib['top'] = top + + return lib + + +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + +def test_gdsii_arrow_matches_gdsii_readfile(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + +def test_gdsii_arrow_matches_gdsii_readfile_for_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_roundtrip.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, canonical_info = gdsii.readfile(gds_file) + arrow_lib, arrow_info = gdsii_arrow.readfile(gds_file) + + assert canonical_info == arrow_info + assert _library_summary(canonical_lib) == _library_summary(arrow_lib) + + +def test_gdsii_arrow_readfile_arrow_returns_native_payload(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_native.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == manifest.library_name + assert libarr['lib_name'].as_py() == manifest.library_name + assert len(libarr['cells']) == manifest.cells + assert 0 < len(libarr['layers']) <= manifest.layers + + +def test_gdsii_arrow_readfile_arrow_reads_gzipped_file(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'native_payload.gds.gz' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, info = gdsii_arrow.readfile_arrow(gds_file) + + assert info['name'] == 'masque-klamath' + assert libarr['lib_name'].as_py() == 'masque-klamath' + assert len(libarr['cells']) == len(lib) + assert len(libarr['layers']) > 0 + + +def test_gdsii_arrow_removed_raw_mode_arg(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'removed_raw_mode.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr, _ = gdsii_arrow.readfile_arrow(gds_file) + + with pytest.raises(TypeError): + gdsii_arrow.readfile(gds_file, raw_mode=False) + + with pytest.raises(TypeError): + gdsii_arrow.read_arrow(libarr, raw_mode=False) + + +def test_gdsii_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_arrow + try: + gdsii_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + +def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_smoke.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert 'TOP' in lib + assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 + + +def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, _ = gdsii.readfile(gds_file) + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + assert _library_summary(arrow_lib) == _library_summary(canonical_lib) + + decoded_ref = arrow_lib['top'].refs['leaf'][0] + assert decoded_ref.repetition is None + + +def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'plain_srefs.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + fanout = arrow_lib['fanout'] + + plain_leaf_refs = [ + ref + for ref in fanout.refs['leaf'] + if ref.annotations is None and ref.repetition is None + ] + assert len(plain_leaf_refs) == 2 + assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs) + + +def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref_schema.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + srefs = cells.field('srefs')[top_index].as_py() + arefs = cells.field('arefs')[top_index].as_py() + + assert len(srefs) == 1 + assert len(arefs) == 0 + assert cell_names[srefs[0]['target']] == 'leaf' + + +def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + + leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + + rect_batches = cells.field('rect_batches')[leaf_index].as_py() + boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() + boundary_props = cells.field('boundary_props')[leaf_index].as_py() + + assert len(rect_batches) == 2 + assert len(boundary_batches) == 0 + assert len(boundary_props) == 2 + + rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches} + assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10] + assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10] + + props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} + assert sorted(props_by_layer) == [(1, 0), (2, 0)] + assert props_by_layer[(1, 0)]['properties'][0]['value'] == 'leaf-poly' + assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' + + +def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None: + lib = Library() + top = Pattern() + top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]])) + top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]]) + top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]]) + lib['top'] = top + + gds_file = tmp_path / 'arrow_rect_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + rect_batches = cells.field('rect_batches')[top_index].as_py() + boundary_batches = cells.field('boundary_batches')[top_index].as_py() + + assert len(rect_batches) == 1 + assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0) + assert rect_batches[0]['rects'] == [ + 0, 0, 10, 10, + 20, 0, 30, 10, + 40, 0, 50, 10, + 60, 0, 70, 10, + ] + + assert len(boundary_batches) == 1 + assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0) + assert boundary_batches[0]['vertex_offsets'] == [0, 3] + + +def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_ref_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout') + + srefs = cells.field('srefs')[fanout_index].as_py() + arefs = cells.field('arefs')[fanout_index].as_py() + sref_props = cells.field('sref_props')[fanout_index].as_py() + aref_props = cells.field('aref_props')[fanout_index].as_py() + + sref_target_ids = [entry['target'] for entry in srefs] + sref_targets = [cell_names[target] for target in sref_target_ids] + assert sorted(sref_targets) == ['child', 'leaf', 'leaf'] + assert sref_target_ids == sorted(sref_target_ids) + sref_by_target = {} + for entry in srefs: + sref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in sref_by_target['child']] == [True] + assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1]) + assert len(sref_by_target['leaf']) == 2 + + aref_target_ids = [entry['target'] for entry in arefs] + aref_targets = [cell_names[target] for target in aref_target_ids] + assert sorted(aref_targets) == ['child', 'leaf', 'leaf'] + assert aref_target_ids == sorted(aref_target_ids) + aref_by_target = {} + for entry in arefs: + aref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in aref_by_target['child']] == [True] + assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2]) + assert len(aref_by_target['leaf']) == 2 + + assert len(sref_props) == 1 + assert cell_names[sref_props[0]['target']] == 'leaf' + assert sref_props[0]['properties'][0]['value'] == 'fanout-sref' + + assert len(aref_props) == 1 + assert cell_names[aref_props[0]['target']] == 'child' + assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' + + +def test_gdsii_arrow_invalid_path_type_matches_gdsii(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + gdsii_arrow.readfile(gds_file) + + +def test_raw_ref_grid_label_constructors_match_public() -> None: + raw_grid = Grid._from_raw( + a_vector=numpy.array([20, 0]), + a_count=3, + b_vector=numpy.array([0, 30]), + b_count=2, + ) + public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) + assert raw_grid == public_grid + + raw_poly = Polygon._from_raw( + vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]), + annotations={'1': ['poly']}, + ) + public_poly = Polygon( + vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], + annotations={'1': ['poly']}, + ) + assert raw_poly == public_poly + + raw_poly_collection = PolyCollection._from_raw( + vertex_lists=numpy.array([ + [0.0, 0.0], [2.0, 0.0], [2.0, 2.0], + [10.0, 10.0], [12.0, 10.0], [12.0, 12.0], + ]), + vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32), + annotations={'2': ['pc']}, + ) + public_poly_collection = PolyCollection( + vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]], + vertex_offsets=[0, 3], + annotations={'2': ['pc']}, + ) + assert raw_poly_collection == public_poly_collection + assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)] + + raw_rect_collection = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'3': ['rects']}, + ) + public_rect_collection = RectCollection( + rects=[[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'3': ['rects']}, + ) + assert raw_rect_collection == public_rect_collection + + raw_ref_empty = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + public_ref_empty = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=False, + scale=1.0, + repetition=None, + annotations=None, + ) + assert raw_ref_empty.annotations is None + assert raw_ref_empty == public_ref_empty + + raw_ref = Ref._from_raw( + offset=numpy.array([100, 200]), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=raw_grid, + annotations={'12': ['child-ref']}, + ) + public_ref = Ref( + offset=(100, 200), + rotation=numpy.pi / 2, + mirrored=True, + scale=1.25, + repetition=public_grid, + annotations={'12': ['child-ref']}, + ) + assert raw_ref == public_ref + assert numpy.array_equal(raw_ref.as_transforms(), public_ref.as_transforms()) + + raw_label_empty = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations=None, + ) + public_label_empty = Label( + 'LEAF', + offset=(3, 4), + annotations=None, + ) + assert raw_label_empty.annotations is None + assert raw_label_empty == public_label_empty + + raw_label = Label._from_raw( + 'LEAF', + offset=numpy.array([3, 4]), + annotations={'10': ['leaf-label']}, + ) + public_label = Label( + 'LEAF', + offset=(3, 4), + annotations={'10': ['leaf-label']}, + ) + assert raw_label == public_label + assert numpy.array_equal(raw_label.get_bounds_single(), public_label.get_bounds_single()) diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py new file mode 100644 index 0000000..eb62721 --- /dev/null +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -0,0 +1,334 @@ +from pathlib import Path +import subprocess +import sys +import textwrap + +import klamath +import numpy +import pytest + +pytest.importorskip('pyarrow') + +from .. import PatternError +from ..library import Library +from ..pattern import Pattern +from ..repetition import Grid +from ..file import gdsii, gdsii_lazy_arrow +from ..file.gdsii_perf import write_fixture + + +if not gdsii_lazy_arrow.is_available(): + pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) + + +def _make_small_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 5], [0, 5]]) + lib['leaf'] = leaf + + mid = Pattern() + mid.ref('leaf', offset=(10, 20)) + mid.ref('leaf', offset=(40, 0), repetition=Grid(a_vector=(12, 0), a_count=2, b_vector=(0, 9), b_count=2)) + lib['mid'] = mid + + top = Pattern() + top.ref('mid', offset=(100, 200)) + lib['top'] = top + return lib + + +def _make_complex_ref_library() -> Library: + lib = Library() + + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [10, 0], [10, 10], [0, 10]]) + lib['leaf'] = leaf + + child = Pattern() + child.ref('leaf', offset=(100, 200), rotation=numpy.pi / 2, mirrored=True, scale=1.25) + lib['child'] = child + + sibling = Pattern() + sibling.ref( + 'leaf', + offset=(-50, 60), + repetition=Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2), + ) + lib['sibling'] = sibling + + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref( + 'child', + offset=(40, 0), + mirrored=True, + rotation=numpy.pi / 4, + scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2), + ) + lib['fanout'] = fanout + + top = Pattern() + top.ref('child', offset=(500, 600)) + top.ref('sibling', offset=(-100, 50), rotation=numpy.pi) + top.ref('fanout', offset=(250, -75)) + lib['top'] = top + + return lib + + +def _write_invalid_path_type_fixture(path: Path) -> None: + with path.open('wb') as stream: + header = klamath.library.FileHeader( + name=b'test', + user_units_per_db_unit=1.0, + meters_per_db_unit=1e-9, + ) + header.write(stream) + elem = klamath.elements.Path( + layer=(1, 0), + path_type=3, + width=10, + extension=(0, 0), + xy=numpy.array([[0, 0], [10, 0]], dtype=numpy.int32), + properties={}, + ) + klamath.library.write_struct(stream, name=b'top', elements=[elem]) + klamath.records.ENDLIB.write(stream, None) + + +def _transform_rows_key(values: numpy.ndarray) -> tuple[tuple[object, ...], ...]: + arr = numpy.asarray(values, dtype=float) + arr = numpy.atleast_2d(arr) + rows = [ + ( + round(float(row[0]), 8), + round(float(row[1]), 8), + round(float(row[2]), 8), + bool(int(round(float(row[3])))), + round(float(row[4]), 8), + ) + for row in arr + ] + return tuple(sorted(rows)) + + +def _local_refs_key(refs: dict[str, list[numpy.ndarray]]) -> dict[str, tuple[tuple[object, ...], ...]]: + return { + parent: _transform_rows_key(numpy.concatenate(transforms)) + for parent, transforms in refs.items() + } + + +def _global_refs_key(refs: dict[tuple[str, ...], numpy.ndarray]) -> dict[tuple[str, ...], tuple[tuple[object, ...], ...]]: + return { + path: _transform_rows_key(transforms) + for path, transforms in refs.items() + } + + +def test_gdsii_lazy_arrow_loads_perf_fixture(tmp_path: Path) -> None: + gds_file = tmp_path / 'many_cells_lazy.gds' + manifest = write_fixture(gds_file, preset='many_cells', scale=0.001) + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + assert info['name'] == manifest.library_name + assert len(lib) == manifest.cells + assert lib.top() == 'TOP' + assert 'TOP' in lib.child_graph(dangling='ignore') + + +def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: + gds_file = tmp_path / 'refs.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-refs') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + local = lib.find_refs_local('leaf') + assert set(local) == {'mid'} + assert sum(arr.shape[0] for arr in local['mid']) == 5 + + global_refs = lib.find_refs_global('leaf') + assert {path for path in global_refs} == {('top', 'mid', 'leaf')} + assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 + + +def test_gdsii_lazy_arrow_ref_queries_match_eager_reader(tmp_path: Path) -> None: + gds_file = tmp_path / 'complex_refs.gds' + src = _make_complex_ref_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='lazy-complex-refs') + + eager, _ = gdsii.readfile(gds_file) + lazy, _ = gdsii_lazy_arrow.readfile(gds_file) + + for name in ('leaf', 'child'): + assert _local_refs_key(lazy.find_refs_local(name)) == _local_refs_key(eager.find_refs_local(name)) + assert _global_refs_key(lazy.find_refs_global(name)) == _global_refs_key(eager.find_refs_global(name)) + + +def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid.gds' + gds_file.write_bytes(b'not-a-gds') + + script = textwrap.dedent(f""" + from masque.file import gdsii_lazy_arrow + try: + gdsii_lazy_arrow.readfile({str(gds_file)!r}) + except Exception as exc: + print(type(exc).__module__) + print(type(exc).__qualname__) + print(exc) + else: + raise SystemExit('expected gdsii_lazy_arrow.readfile() to fail') + """) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, check=False) + + assert result.returncode == 0, result.stderr + assert 'klamath.basic' in result.stdout + assert 'KlamathError' in result.stdout + + +def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) -> None: + gds_file = tmp_path / 'invalid_path_type.gds' + _write_invalid_path_type_fixture(gds_file) + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + + with pytest.raises(PatternError, match='Unrecognized path type: 3'): + lib['top'] + + +def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_arrow_gzipped_copy_through(tmp_path: Path) -> None: + gds_file = tmp_path / 'copy_source.gds.gz' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through-gz') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + out_file = tmp_path / 'copy_out.gds.gz' + gdsii_lazy_arrow.writefile( + lib, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: + base_a = Library() + leaf_a = Pattern() + leaf_a.polygon((1, 0), vertices=[[0, 0], [8, 0], [8, 8], [0, 8]]) + base_a['leaf'] = leaf_a + top_a = Pattern() + top_a.ref('leaf', offset=(0, 0)) + base_a['top_a'] = top_a + + base_b = Library() + leaf_b = Pattern() + leaf_b.polygon((2, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + base_b['leaf'] = leaf_b + top_b = Pattern() + top_b.ref('leaf', offset=(20, 30)) + base_b['top_b'] = top_b + + gds_a = tmp_path / 'a.gds' + gds_b = tmp_path / 'b.gds' + gdsii.writefile(base_a, gds_a, meters_per_unit=1e-9, library_name='overlay') + gdsii.writefile(base_b, gds_b, meters_per_unit=1e-9, library_name='overlay') + + lib_a, _ = gdsii_lazy_arrow.readfile(gds_a) + lib_b, _ = gdsii_lazy_arrow.readfile(gds_b) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib_a) + rename_map = overlay.add_source(lib_b, rename_theirs=lambda lib, name: lib.get_name(name)) + renamed_leaf = rename_map['leaf'] + + assert rename_map == {'leaf': renamed_leaf} + assert renamed_leaf != 'leaf' + assert len(lib_a._cache) == 0 + assert len(lib_b._cache) == 0 + + overlay.move_references('leaf', renamed_leaf) + + out_file = tmp_path / 'overlay_out.gds' + gdsii_lazy_arrow.writefile(overlay, out_file) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf', renamed_leaf, 'top_a', 'top_b'} + assert 'top_b' in roundtrip + assert list(roundtrip['top_b'].refs.keys()) == [renamed_leaf] + + +def test_gdsii_writer_accepts_overlay_library(tmp_path: Path) -> None: + gds_file = tmp_path / 'overlay_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-src') + + lib, info = gdsii_lazy_arrow.readfile(gds_file) + + overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay.add_source(lib) + overlay.rename('leaf', 'leaf_copy', move_references=True) + + out_file = tmp_path / 'overlay_via_eager_writer.gds' + gdsii.writefile( + overlay, + out_file, + meters_per_unit=info['meters_per_unit'], + logical_units_per_unit=info['logical_units_per_unit'], + library_name=info['name'], + ) + + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip.keys()) == {'leaf_copy', 'mid', 'top'} + assert list(roundtrip['mid'].refs.keys()) == ['leaf_copy'] + + +def test_svg_writer_uses_detached_materialized_copy(tmp_path: Path) -> None: + pytest.importorskip('svgwrite') + from ..file import svg + from ..shapes import Path as MPath + + gds_file = tmp_path / 'svg_source.gds' + src = _make_small_library() + src['top'].path((3, 0), vertices=[[0, 0], [0, 20]], width=4) + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='svg-src') + + lib, _ = gdsii_lazy_arrow.readfile(gds_file) + top_pat = lib['top'] + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) + + svg_path = tmp_path / 'lazy.svg' + svg.writefile(lib, 'top', str(svg_path)) + + assert svg_path.exists() + assert list(top_pat.refs.keys()) == ['mid'] + assert any(isinstance(shape, MPath) for shape in top_pat.shapes[(3, 0)]) diff --git a/masque/test/test_gdsii_perf.py b/masque/test/test_gdsii_perf.py new file mode 100644 index 0000000..a595fe8 --- /dev/null +++ b/masque/test/test_gdsii_perf.py @@ -0,0 +1,24 @@ +from dataclasses import asdict +import json +from pathlib import Path + +from ..file import gdsii +from ..file.gdsii_perf import fixture_manifest, write_fixture + + +def test_gdsii_perf_fixture_smoke(tmp_path: Path) -> None: + output = tmp_path / 'many_cells.gds' + manifest = write_fixture(output, preset='many_cells', scale=0.002) + expected = fixture_manifest(output, preset='many_cells', scale=0.002) + + assert output.exists() + assert manifest == expected + + sidecar = json.loads(output.with_suffix('.gds.json').read_text()) + assert sidecar == asdict(manifest) + + read_lib, info = gdsii.readfile(output) + assert info['name'] == manifest.library_name + assert len(read_lib) == manifest.cells + assert 'TOP' in read_lib + assert len(read_lib['TOP'].refs) > 0 diff --git a/masque/test/test_raw_constructors.py b/masque/test/test_raw_constructors.py new file mode 100644 index 0000000..2f86ba0 --- /dev/null +++ b/masque/test/test_raw_constructors.py @@ -0,0 +1,97 @@ +import numpy +from numpy import pi +from numpy.testing import assert_allclose + +from ..shapes import Arc, Circle, Ellipse, Path, Text + + +def test_circle_raw_constructor_matches_public() -> None: + raw = Circle._from_raw( + radius=5.0, + offset=numpy.array([1.0, 2.0]), + annotations={'1': ['circle']}, + ) + public = Circle( + radius=5.0, + offset=(1.0, 2.0), + annotations={'1': ['circle']}, + ) + assert raw == public + + +def test_ellipse_raw_constructor_matches_public() -> None: + raw = Ellipse._from_raw( + radii=numpy.array([3.0, 5.0]), + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + public = Ellipse( + radii=(3.0, 5.0), + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'2': ['ellipse']}, + ) + assert raw == public + + +def test_arc_raw_constructor_matches_public() -> None: + raw = Arc._from_raw( + radii=numpy.array([10.0, 6.0]), + angles=numpy.array([0.0, pi / 2]), + width=2.0, + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + public = Arc( + radii=(10.0, 6.0), + angles=(0.0, pi / 2), + width=2.0, + offset=(1.0, 2.0), + rotation=5 * pi / 2, + annotations={'3': ['arc']}, + ) + assert raw == public + + +def test_path_raw_constructor_matches_public() -> None: + raw = Path._from_raw( + vertices=numpy.array([[0.0, 0.0], [10.0, 0.0], [10.0, 5.0]]), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=numpy.array([1.0, 3.0]), + annotations={'4': ['path']}, + ) + public = Path( + vertices=((0.0, 0.0), (10.0, 0.0), (10.0, 5.0)), + width=2.0, + cap=Path.Cap.SquareCustom, + cap_extensions=(1.0, 3.0), + annotations={'4': ['path']}, + ) + assert raw == public + assert raw.cap_extensions is not None + assert_allclose(raw.cap_extensions, [1.0, 3.0]) + + +def test_text_raw_constructor_matches_public() -> None: + raw = Text._from_raw( + string='RAW', + height=12.0, + font_path='font.otf', + offset=numpy.array([1.0, 2.0]), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + public = Text( + string='RAW', + height=12.0, + font_path='font.otf', + offset=(1.0, 2.0), + rotation=5 * pi / 2, + mirrored=True, + annotations={'5': ['text']}, + ) + assert raw == public diff --git a/masque/test/test_rect_collection.py b/masque/test/test_rect_collection.py new file mode 100644 index 0000000..449f4fa --- /dev/null +++ b/masque/test/test_rect_collection.py @@ -0,0 +1,70 @@ +import copy + +import numpy +import pytest +from numpy.testing import assert_allclose, assert_equal + +from ..error import PatternError +from ..shapes import Polygon, RectCollection + + +def test_rect_collection_init_and_to_polygons() -> None: + rects = RectCollection([[10, 10, 12, 12], [0, 0, 5, 5]]) + assert_equal(rects.rects, [[0, 0, 5, 5], [10, 10, 12, 12]]) + + polys = rects.to_polygons() + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_equal(polys[0].vertices, [[0, 0], [0, 5], [5, 5], [5, 0]]) + + +def test_rect_collection_rejects_invalid_rects() -> None: + with pytest.raises(PatternError): + RectCollection([[0, 0, 1]]) + with pytest.raises(PatternError): + RectCollection([[5, 0, 1, 2]]) + with pytest.raises(PatternError): + RectCollection([[0, 5, 1, 2]]) + + +def test_rect_collection_raw_constructor_matches_public() -> None: + raw = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'1': ['rects']}, + ) + public = RectCollection( + [[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'1': ['rects']}, + ) + assert raw == public + assert_equal(raw.get_bounds_single(), [[0, 0], [12, 12]]) + + +def test_rect_collection_manhattan_transforms() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + + mirrored = copy.deepcopy(rects).mirror(1) + assert_equal(mirrored.rects, [[-2, 0, 0, 4], [-12, 20, -10, 22]]) + + scaled = copy.deepcopy(rects).scale_by(-2) + assert_equal(scaled.rects, [[-4, -8, 0, 0], [-24, -44, -20, -40]]) + + rotated = copy.deepcopy(rects).rotate(numpy.pi / 2) + assert_equal(rotated.rects, [[-4, 0, 0, 2], [-22, 10, -20, 12]]) + + +def test_rect_collection_non_manhattan_rotation_raises() -> None: + rects = RectCollection([[0, 0, 2, 4]]) + with pytest.raises(PatternError, match='Manhattan rotations'): + rects.rotate(numpy.pi / 4) + + +def test_rect_collection_normalized_form_rebuild_is_independent() -> None: + rects = RectCollection([[0, 0, 2, 4], [10, 20, 12, 22]]) + _intrinsic, extrinsic, rebuild = rects.normalized_form(2) + + clone = rebuild() + clone.rects[:] = [[1, 1, 2, 2], [3, 3, 4, 4]] + + assert_allclose(extrinsic[0], [6, 11.5]) + assert_equal(rects.rects, [[0, 0, 2, 4], [10, 20, 12, 22]]) diff --git a/masque/utils/comparisons.py b/masque/utils/comparisons.py index ffb7206..bb2dfee 100644 --- a/masque/utils/comparisons.py +++ b/masque/utils/comparisons.py @@ -9,7 +9,15 @@ def annotation2key(aaa: int | float | str) -> tuple[bool, Any]: return (isinstance(aaa, str), aaa) +def _normalized_annotations(annotations: annotations_t) -> annotations_t: + if not annotations: + return None + return annotations + + def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is not None elif bb is None: # noqa: RET505 @@ -36,6 +44,8 @@ def annotations_lt(aa: annotations_t, bb: annotations_t) -> bool: def annotations_eq(aa: annotations_t, bb: annotations_t) -> bool: + aa = _normalized_annotations(aa) + bb = _normalized_annotations(bb) if aa is None: return bb is None elif bb is None: # noqa: RET505 diff --git a/pyproject.toml b/pyproject.toml index af8802c..ad03fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ [dependency-groups] dev = [ - "pytest", + "masque[arrow]", "masque[oasis]", "masque[dxf]", "masque[svg]", @@ -52,6 +52,7 @@ dev = [ "masque[text]", "masque[manhattanize]", "masque[manhattanize_slow]", + "masque[boolean]", "matplotlib>=3.10.8", "pytest>=9.0.2", "ruff>=0.15.5", @@ -66,6 +67,7 @@ build-backend = "hatchling.build" path = "masque/__init__.py" [project.optional-dependencies] +arrow = ["pyarrow", "cffi"] oasis = ["fatamorgana~=0.11"] dxf = ["ezdxf~=1.4"] svg = ["svgwrite"] @@ -121,4 +123,3 @@ mypy_path = "stubs" python_version = "3.11" strict = false check_untyped_defs = true -