From 3f63599abe174b21ba053a9065445a26621ef6d5 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 2 Apr 2026 19:42:52 -0700 Subject: [PATCH] [gdsii_arrow] more performance work --- masque/file/gdsii_arrow.py | 356 ++++++++++++++++++++++++----- masque/shapes/poly_collection.py | 13 +- masque/test/test_file_roundtrip.py | 29 ++- masque/test/test_gdsii_arrow.py | 264 ++++++++++++++++++--- 4 files changed, 562 insertions(+), 100 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index f6a7739..c5e7681 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -45,7 +45,7 @@ from pyarrow.cffi import ffi from .utils import is_gzipped, tmpfile from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, PolyCollection +from ..shapes import Polygon, Path, PolyCollection, RectCollection from ..repetition import Grid from ..utils import layer_t, annotations_t from ..library import LazyLibrary, Library, ILibrary, ILibraryView @@ -276,6 +276,15 @@ def read_arrow( poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), ) + def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]: + batches = libarr['cells'].values.field('rect_batches') + return dict( + offsets = batches.offsets.to_numpy(), + layer_inds = batches.values.field('layer').to_numpy(), + rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_off = batches.values.field('rects').offsets.to_numpy() // 4, + ) + def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]: boundaries = libarr['cells'].values.field('boundary_props') return dict( @@ -288,22 +297,46 @@ def read_arrow( prop_val = boundaries.values.field('properties').values.field('value').to_pylist(), ) - rf = libarr['cells'].values.field('refs') - refs = dict( - offsets = rf.offsets.to_numpy(), - targets = rf.values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()), - invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), - angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()), - scale = rf.values.field('mag').fill_null(1).to_numpy(), - rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), - rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()), - rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()), - rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()), - prop_off = rf.values.field('properties').offsets.to_numpy(), - prop_key = rf.values.field('properties').values.field('key').to_numpy(), - prop_val = rf.values.field('properties').values.field('value').to_pylist(), - ) + def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem + + def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]: + refs = libarr['cells'].values.field(geom_type) + values = refs.values + elem = dict( + offsets = refs.offsets.to_numpy(), + targets = values.field('target').to_numpy(), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), + angle_rad = values.field('angle_rad').to_numpy(), + scale = values.field('scale').to_numpy(), + prop_off = values.field('properties').offsets.to_numpy(), + prop_key = values.field('properties').values.field('key').to_numpy(), + prop_val = values.field('properties').values.field('value').to_pylist(), + ) + if has_repetition: + elem.update(dict( + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), + )) + return elem txt = libarr['cells'].values.field('texts') texts = dict( @@ -317,11 +350,15 @@ def read_arrow( ) elements = dict( + srefs = get_refs(libarr, 'srefs', has_repetition=False), + arefs = get_refs(libarr, 'arefs', has_repetition=True), + sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False), + aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True), + rect_batches = get_rect_batches(libarr), boundary_batches = get_boundary_batches(libarr), boundary_props = get_boundary_props(libarr), paths = get_geom(libarr, 'paths'), texts = texts, - refs = refs, ) paths = libarr['cells'].values.field('paths') @@ -344,10 +381,14 @@ def read_arrow( for cc in range(len(libarr['cells'])): name = cell_names[cell_ids[cc]] pat = Pattern() + _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc) _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc) _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc) _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) - _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _srefs_to_mrefs(pat, global_args, elements['srefs'], cc) + _arefs_to_mrefs(pat, global_args, elements['arefs'], cc) + _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc) + _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc) _texts_to_labels(pat, global_args, elements['texts'], cc) mlib[name] = pat @@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: return library_info -def _grefs_to_mrefs( +def _srefs_to_mrefs( pat: Pattern, global_args: dict[str, Any], elem: dict[str, Any], cc: int, ) -> None: cell_names = global_args['cell_names'] - elem_off = elem['offsets'] # which elements belong to each cell - xy = elem['xy'] - prop_key = elem['prop_key'] - prop_val = elem['prop_val'] - targets = elem['targets'] - + elem_off = elem['offsets'] elem_count = elem_off[cc + 1] - elem_off[cc] - elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem - prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - elem_targets = targets[elem_slc][:elem_count] - elem_xy = xy[elem_slc][:elem_count] - elem_invert_y = elem['invert_y'][elem_slc][:elem_count] - elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count] - elem_scale = elem['scale'][elem_slc][:elem_count] - elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count] - elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count] - elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count] - rep_valid = elem['rep_valid'][elem_slc][:elem_count] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] raw_mode = global_args['raw_mode'] + _append_plain_refs_sorted( + pat=pat, + cell_names=cell_names, + elem_targets=elem_targets, + elem_xy=elem_xy, + elem_invert_y=elem_invert_y, + elem_angle_rad=elem_angle_rad, + elem_scale=elem_scale, + raw_mode=raw_mode, + ) + + +def _append_plain_refs_sorted( + *, + pat: Pattern, + cell_names: list[str], + elem_targets: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.integer[Any]], + elem_invert_y: NDArray[numpy.bool_ | numpy.bool], + elem_angle_rad: NDArray[numpy.floating[Any]], + elem_scale: NDArray[numpy.floating[Any]], + raw_mode: bool, + ) -> None: + elem_count = len(elem_targets) + if elem_count == 0: + return + + make_ref = Ref._from_raw if raw_mode else Ref + + target_start = 0 + while target_start < elem_count: + target_id = elem_targets[target_start] + target_stop = target_start + 1 + while target_stop < elem_count and elem_targets[target_stop] == target_id: + target_stop += 1 + + append_refs = pat.refs[cell_names[target_id]].extend + append_refs( + make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=None, + ) + for ee in range(target_start, target_stop) + ) + + target_start = target_stop + + +def _arefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + start = elem_off[cc] + stop = elem_off[cc + 1] + elem_targets = elem['targets'][start:stop] + elem_xy = elem['xy'][start:stop] + elem_invert_y = elem['invert_y'][start:stop] + elem_angle_rad = elem['angle_rad'][start:stop] + elem_scale = elem['scale'][start:stop] + elem_xy0 = elem['xy0'][start:stop] + elem_xy1 = elem['xy1'][start:stop] + elem_counts = elem['counts'][start:stop] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + if len(elem_targets) == 0: + return + + target = None + append_ref: Callable[[Ref], Any] | None = None + for ee in range(len(elem_targets)): + if target != elem_targets[ee]: + target = elem_targets[ee] + append_ref = pat.refs[cell_names[target]].append + assert append_ref is not None + a_count, b_count = elem_counts[ee] + append_ref(make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=None, + )) + + +def _sref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref for ee in range(elem_count): - target = cell_names[elem_targets[ee]] - offset = elem_xy[ee] - mirr = elem_invert_y[ee] - rot = elem_angle_rad[ee] - mag = elem_scale[ee] - - rep: None | Grid = None - if rep_valid[ee]: - a_vector = elem_rep_xy0[ee] - b_vector = elem_rep_xy1[ee] - a_count, b_count = elem_rep_counts[ee] - if raw_mode: - rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - else: - rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) - annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - if raw_mode: - ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - else: - ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) - pat.refs[target].append(ref) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=None, + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) + + +def _aref_props_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + if elem_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) + prop_offs = elem['prop_off'][elem_slc] + elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]] + elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]] + elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]] + elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]] + elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]] + elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]] + elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]] + elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]] + raw_mode = global_args['raw_mode'] + + make_ref = Ref._from_raw if raw_mode else Ref + make_grid = Grid._from_raw if raw_mode else Grid + + for ee in range(elem_count): + a_count, b_count = elem_counts[ee] + annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) + ref = make_ref( + offset=elem_xy[ee], + mirrored=elem_invert_y[ee], + rotation=elem_angle_rad[ee], + scale=elem_scale[ee], + repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count), + annotations=annotations, + ) + pat.refs[cell_names[elem_targets[ee]]].append(ref) def _texts_to_labels( @@ -520,16 +712,53 @@ def _boundary_batches_to_polygons( for bb in range(batch_count): layer = layer_tups[elem_layer_inds[bb]] vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]] - vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp) + vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]] if vertex_offsets.size == 1: - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=None) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(poly) else: - polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode) + if raw_mode: + polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None) + else: + polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False) pat.shapes[layer].append(polys) +def _rect_batches_to_rectcollections( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] + rect_arr = elem['rect_arr'] + rect_off = elem['rect_off'] + layer_inds = elem['layer_inds'] + layer_tups = global_args['layer_tups'] + + batch_count = elem_off[cc + 1] - elem_off[cc] + if batch_count == 0: + return + + elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) + elem_rect_off = rect_off[elem_slc] + elem_layer_inds = layer_inds[elem_slc][:batch_count] + + raw_mode = global_args['raw_mode'] + for bb in range(batch_count): + layer = layer_tups[elem_layer_inds[bb]] + rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]] + if raw_mode: + rect_collection = RectCollection._from_raw(rects=rects, annotations=None) + else: + rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False) + pat.shapes[layer].append(rect_collection) + + def _boundary_props_to_polygons( pat: Pattern, global_args: dict[str, Any], @@ -558,7 +787,10 @@ def _boundary_props_to_polygons( layer = layer_tups[elem_layer_inds[ee]] vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]] annotations = _read_annotations(prop_offs, prop_key, prop_val, ee) - poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode) + if raw_mode: + poly = Polygon._from_raw(vertices=vertices, annotations=annotations) + else: + poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False) pat.shapes[layer].append(poly) diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py index 0b5543f..f90c29a 100644 --- a/masque/shapes/poly_collection.py +++ b/masque/shapes/poly_collection.py @@ -34,7 +34,7 @@ class PolyCollection(Shape): _vertex_lists: NDArray[numpy.float64] """ 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """ - _vertex_offsets: NDArray[numpy.intp] + _vertex_offsets: NDArray[numpy.integer[Any]] """ 1D NDArray specifying the starting offset for each polygon """ @property @@ -45,7 +45,7 @@ class PolyCollection(Shape): return self._vertex_lists @property - def vertex_offsets(self) -> NDArray[numpy.intp]: + def vertex_offsets(self) -> NDArray[numpy.integer[Any]]: """ Starting offset (in `vertex_lists`) for each polygon """ @@ -63,7 +63,7 @@ class PolyCollection(Shape): chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]), strict=True, ): - yield slice(ii, ff) + yield slice(int(ii), int(ff)) @property def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]: @@ -105,6 +105,7 @@ class PolyCollection(Shape): if raw: assert isinstance(vertex_lists, numpy.ndarray) assert isinstance(vertex_offsets, numpy.ndarray) + assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer) self._vertex_lists = vertex_lists self._vertex_offsets = vertex_offsets self._repetition = repetition @@ -148,7 +149,7 @@ class PolyCollection(Shape): return ( type(self) is type(other) and numpy.array_equal(self._vertex_lists, other._vertex_lists) - and numpy.array_equal(self._vertex_offsets, other._vertex_offsets) + and numpy.array_equal(self.vertex_offsets, other.vertex_offsets) and self.repetition == other.repetition and annotations_eq(self.annotations, other.annotations) ) @@ -231,11 +232,11 @@ class PolyCollection(Shape): # TODO: normalize mirroring? - return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()), + return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()), (offset, scale / norm_value, rotation, False), lambda: PolyCollection( vertex_lists=rotated_vertices * norm_value, - vertex_offsets=self._vertex_offsets.copy(), + vertex_offsets=self.vertex_offsets.copy(), ), ) diff --git a/masque/test/test_file_roundtrip.py b/masque/test/test_file_roundtrip.py index 2cfb0d1..283a863 100644 --- a/masque/test/test_file_roundtrip.py +++ b/masque/test/test_file_roundtrip.py @@ -5,7 +5,7 @@ from numpy.testing import assert_allclose from ..pattern import Pattern from ..library import Library -from ..shapes import Path as MPath, Circle, Polygon +from ..shapes import Path as MPath, Circle, Polygon, RectCollection from ..repetition import Grid, Arbitrary def create_test_library(for_gds: bool = False) -> Library: @@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None: assert poly.repetition is not None assert isinstance(poly.repetition, Grid) assert poly.repetition.a_count == 5 + + +def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None: + from ..file import gdsii + + lib = Library() + pat = Pattern() + pat.shapes[(5, 0)].append( + RectCollection( + rects=[[0, 0, 10, 5], [20, -5, 30, 10]], + annotations={'1': ['rects']}, + ) + ) + lib['rects'] = pat + + gds_file = tmp_path / 'rect_collection.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + read_lib, _ = gdsii.readfile(gds_file) + polys = read_lib['rects'].shapes[(5, 0)] + + assert len(polys) == 2 + assert all(isinstance(poly, Polygon) for poly in polys) + assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]]) + assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]]) + assert polys[0].annotations == {'1': ['rects']} + assert polys[1].annotations == {'1': ['rects']} diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index cec48c8..2ef4d83 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -9,7 +9,7 @@ from .. import Ref, Label from ..library import Library from ..pattern import Pattern from ..repetition import Grid -from ..shapes import Path as MPath +from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection from ..file import gdsii, gdsii_arrow from ..file.gdsii_perf import write_fixture @@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]: return tuple(tuple(row.tolist()) for row in arr) +def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]: + arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int) + rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])] + rev = arr[::-1] + rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0])) + return min(rows) + + def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]: if isinstance(shape, MPath): cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions) @@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...] keys.append(( 'polygon', layer, - _coord_key(poly.vertices), + _canonical_polygon_key(poly.vertices), _coord_key(poly.offset), _annotations_key(poly.annotations), )) return keys -def _ref_key(target: str, ref: object) -> tuple[object, ...]: - repetition = None - if ref.repetition is not None: - repetition = ( - _coord_key(ref.repetition.a_vector), - int(ref.repetition.a_count), - _coord_key(ref.repetition.b_vector), - int(ref.repetition.b_count), - ) - return ( - target, - _coord_key(ref.offset), - round(float(ref.rotation), 8), - round(float(ref.scale), 8), - bool(ref.mirrored), - repetition, - _annotations_key(ref.annotations), - ) +def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]: + keys = [] + for transform in ref.as_transforms(): + keys.append(( + target, + _coord_key(transform[:2]), + round(float(transform[2]), 8), + round(float(transform[4]), 8), + bool(int(round(float(transform[3])))), + _annotations_key(ref.annotations), + )) + return keys def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]: @@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]: for shape in shapes: shape_keys.extend(_shape_key(shape, layer)) - ref_keys = [ - _ref_key(target, ref) - for target, refs in pattern.refs.items() - for ref in refs - ] + ref_keys: list[tuple[object, ...]] = [] + for target, refs in pattern.refs.items(): + for ref in refs: + ref_keys.extend(_ref_keys(target, ref)) label_keys = [ _label_key(layer, label) @@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library: ) lib['sibling'] = sibling + fanout = Pattern() + fanout.ref('leaf', offset=(0, 0)) + fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1) + fanout.ref('leaf', offset=(20, 0)) + fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3)) + fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2, + repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2)) + fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2)) + fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']}) + fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2), + annotations={'20': ['fanout-aref']}) + lib['fanout'] = fanout + top = Pattern() top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']}) top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']}) + top.ref('fanout', offset=(250, -75)) top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']}) lib['top'] = top @@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None: assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0 +def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + canonical_lib, _ = gdsii.readfile(gds_file) + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + assert _library_summary(arrow_lib) == _library_summary(canonical_lib) + + decoded_ref = arrow_lib['top'].refs['leaf'][0] + assert decoded_ref.repetition is None + + +def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'plain_srefs.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + arrow_lib, _ = gdsii_arrow.readfile(gds_file) + fanout = arrow_lib['fanout'] + + plain_leaf_refs = [ + ref + for ref in fanout.refs['leaf'] + if ref.annotations is None and ref.repetition is None + ] + assert len(plain_leaf_refs) == 2 + assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs) + + +def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None: + lib = Library() + leaf = Pattern() + leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]]) + lib['leaf'] = leaf + + top = Pattern() + top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1)) + lib['top'] = top + + gds_file = tmp_path / 'degenerate_aref_schema.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + srefs = cells.field('srefs')[top_index].as_py() + arefs = cells.field('arefs')[top_index].as_py() + + assert len(srefs) == 1 + assert len(arefs) == 0 + assert cell_names[srefs[0]['target']] == 'leaf' + + def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: lib = _make_arrow_test_library() gds_file = tmp_path / 'arrow_batches.gds' @@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf') + rect_batches = cells.field('rect_batches')[leaf_index].as_py() boundary_batches = cells.field('boundary_batches')[leaf_index].as_py() boundary_props = cells.field('boundary_props')[leaf_index].as_py() - assert len(boundary_batches) == 2 + assert len(rect_batches) == 2 + assert len(boundary_batches) == 0 assert len(boundary_props) == 2 - batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches} - assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4] - assert len(batch_by_layer[(1, 0)]['vertices']) == 16 - assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0] - assert len(batch_by_layer[(2, 0)]['vertices']) == 8 + rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches} + assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10] + assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10] props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props} assert sorted(props_by_layer) == [(1, 0), (2, 0)] @@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None: assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2' +def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None: + lib = Library() + top = Pattern() + top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]])) + top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]]) + top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]]) + lib['top'] = top + + gds_file = tmp_path / 'arrow_rect_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + layer_table = [ + ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF) + for layer in libarr['layers'].values.to_numpy() + ] + top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top') + + rect_batches = cells.field('rect_batches')[top_index].as_py() + boundary_batches = cells.field('boundary_batches')[top_index].as_py() + + assert len(rect_batches) == 1 + assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0) + assert rect_batches[0]['rects'] == [ + 0, 0, 10, 10, + 20, 0, 30, 10, + 40, 0, 50, 10, + 60, 0, 70, 10, + ] + + assert len(boundary_batches) == 1 + assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0) + assert boundary_batches[0]['vertex_offsets'] == [0, 3] + + +def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None: + lib = _make_arrow_test_library() + gds_file = tmp_path / 'arrow_ref_batches.gds' + gdsii.writefile(lib, gds_file, meters_per_unit=1e-9) + + libarr = gdsii_arrow._read_to_arrow(gds_file)[0] + cells = libarr['cells'].values + cell_ids = cells.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout') + + srefs = cells.field('srefs')[fanout_index].as_py() + arefs = cells.field('arefs')[fanout_index].as_py() + sref_props = cells.field('sref_props')[fanout_index].as_py() + aref_props = cells.field('aref_props')[fanout_index].as_py() + + sref_target_ids = [entry['target'] for entry in srefs] + sref_targets = [cell_names[target] for target in sref_target_ids] + assert sorted(sref_targets) == ['child', 'leaf', 'leaf'] + assert sref_target_ids == sorted(sref_target_ids) + sref_by_target = {} + for entry in srefs: + sref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in sref_by_target['child']] == [True] + assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1]) + assert len(sref_by_target['leaf']) == 2 + + aref_target_ids = [entry['target'] for entry in arefs] + aref_targets = [cell_names[target] for target in aref_target_ids] + assert sorted(aref_targets) == ['child', 'leaf', 'leaf'] + assert aref_target_ids == sorted(aref_target_ids) + aref_by_target = {} + for entry in arefs: + aref_by_target.setdefault(cell_names[entry['target']], []).append(entry) + assert [entry['invert_y'] for entry in aref_by_target['child']] == [True] + assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2]) + assert len(aref_by_target['leaf']) == 2 + + assert len(sref_props) == 1 + assert cell_names[sref_props[0]['target']] == 'leaf' + assert sref_props[0]['properties'][0]['value'] == 'fanout-sref' + + assert len(aref_props) == 1 + assert cell_names[aref_props[0]['target']] == 'child' + assert aref_props[0]['properties'][0]['value'] == 'fanout-aref' + + def test_raw_ref_grid_label_constructors_match_public() -> None: raw_grid = Grid._from_raw( a_vector=numpy.array([20, 0]), @@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None: public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2) assert raw_grid == public_grid + raw_poly = Polygon._from_raw( + vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]), + annotations={'1': ['poly']}, + ) + public_poly = Polygon( + vertices=[[0, 0], [5, 0], [5, 5], [0, 5]], + annotations={'1': ['poly']}, + ) + assert raw_poly == public_poly + + raw_poly_collection = PolyCollection._from_raw( + vertex_lists=numpy.array([ + [0.0, 0.0], [2.0, 0.0], [2.0, 2.0], + [10.0, 10.0], [12.0, 10.0], [12.0, 12.0], + ]), + vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32), + annotations={'2': ['pc']}, + ) + public_poly_collection = PolyCollection( + vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]], + vertex_offsets=[0, 3], + annotations={'2': ['pc']}, + ) + assert raw_poly_collection == public_poly_collection + assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)] + + raw_rect_collection = RectCollection._from_raw( + rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]), + annotations={'3': ['rects']}, + ) + public_rect_collection = RectCollection( + rects=[[0, 0, 5, 5], [10, 10, 12, 12]], + annotations={'3': ['rects']}, + ) + assert raw_rect_collection == public_rect_collection + raw_ref_empty = Ref._from_raw( offset=numpy.array([100, 200]), rotation=numpy.pi / 2,