[gdsii_arrow] more performance work

masque/file/gdsii_arrow.py

@@ -45,7 +45,7 @@ from pyarrow.cffi import ffi
 
 from .utils import is_gzipped, tmpfile
 from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape
-from ..shapes import Polygon, Path, PolyCollection
+from ..shapes import Polygon, Path, PolyCollection, RectCollection
 from ..repetition import Grid
 from ..utils import layer_t, annotations_t
 from ..library import LazyLibrary, Library, ILibrary, ILibraryView
@@ -276,6 +276,15 @@ def read_arrow(
             poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
             )
 
+    def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]:
+        batches = libarr['cells'].values.field('rect_batches')
+        return dict(
+            offsets = batches.offsets.to_numpy(),
+            layer_inds = batches.values.field('layer').to_numpy(),
+            rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
+            rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
+            )
+
     def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]:
         boundaries = libarr['cells'].values.field('boundary_props')
         return dict(
@@ -288,22 +297,46 @@ def read_arrow(
             prop_val = boundaries.values.field('properties').values.field('value').to_pylist(),
             )
 
-    rf = libarr['cells'].values.field('refs')
+    def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
-    refs = dict(
+        refs = libarr['cells'].values.field(geom_type)
-        offsets = rf.offsets.to_numpy(),
+        values = refs.values
-        targets = rf.values.field('target').to_numpy(),
+        elem = dict(
-        xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()),
+            offsets = refs.offsets.to_numpy(),
-        invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False),
+            targets = values.field('target').to_numpy(),
-        angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()),
+            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
-        scale = rf.values.field('mag').fill_null(1).to_numpy(),
+            invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
-        rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False),
+            angle_rad = values.field('angle_rad').to_numpy(),
-        rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()),
+            scale = values.field('scale').to_numpy(),
-        rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()),
+            )
-        rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()),
+        if has_repetition:
-        prop_off = rf.values.field('properties').offsets.to_numpy(),
+            elem.update(dict(
-        prop_key = rf.values.field('properties').values.field('key').to_numpy(),
+                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
-        prop_val = rf.values.field('properties').values.field('value').to_pylist(),
+                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
-        )
+                counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
+                ))
+        return elem
+
+    def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
+        refs = libarr['cells'].values.field(geom_type)
+        values = refs.values
+        elem = dict(
+            offsets = refs.offsets.to_numpy(),
+            targets = values.field('target').to_numpy(),
+            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
+            invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
+            angle_rad = values.field('angle_rad').to_numpy(),
+            scale = values.field('scale').to_numpy(),
+            prop_off = values.field('properties').offsets.to_numpy(),
+            prop_key = values.field('properties').values.field('key').to_numpy(),
+            prop_val = values.field('properties').values.field('value').to_pylist(),
+            )
+        if has_repetition:
+            elem.update(dict(
+                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
+                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
+                counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
+                ))
+        return elem
 
     txt = libarr['cells'].values.field('texts')
     texts = dict(
@@ -317,11 +350,15 @@ def read_arrow(
         )
 
     elements = dict(
+        srefs = get_refs(libarr, 'srefs', has_repetition=False),
+        arefs = get_refs(libarr, 'arefs', has_repetition=True),
+        sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False),
+        aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True),
+        rect_batches = get_rect_batches(libarr),
         boundary_batches = get_boundary_batches(libarr),
         boundary_props = get_boundary_props(libarr),
         paths = get_geom(libarr, 'paths'),
         texts = texts,
-        refs = refs,
         )
 
     paths = libarr['cells'].values.field('paths')
@@ -344,10 +381,14 @@ def read_arrow(
     for cc in range(len(libarr['cells'])):
         name = cell_names[cell_ids[cc]]
         pat = Pattern()
+        _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc)
         _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc)
         _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc)
         _gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
-        _grefs_to_mrefs(pat, global_args, elements['refs'], cc)
+        _srefs_to_mrefs(pat, global_args, elements['srefs'], cc)
+        _arefs_to_mrefs(pat, global_args, elements['arefs'], cc)
+        _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc)
+        _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc)
         _texts_to_labels(pat, global_args, elements['texts'], cc)
         mlib[name] = pat
 
@@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]:
     return library_info
 
 
-def _grefs_to_mrefs(
+def _srefs_to_mrefs(
         pat: Pattern,
         global_args: dict[str, Any],
         elem: dict[str, Any],
         cc: int,
         ) -> None:
     cell_names = global_args['cell_names']
-    elem_off = elem['offsets']      # which elements belong to each cell
+    elem_off = elem['offsets']
-    xy = elem['xy']
-    prop_key = elem['prop_key']
-    prop_val = elem['prop_val']
-    targets = elem['targets']
-
     elem_count = elem_off[cc + 1] - elem_off[cc]
-    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)   # +1 to capture ending location for last elem
+    if elem_count == 0:
-    prop_offs = elem['prop_off'][elem_slc]  # which props belong to each element
+        return
-    elem_targets = targets[elem_slc][:elem_count]
+
-    elem_xy = xy[elem_slc][:elem_count]
+    start = elem_off[cc]
-    elem_invert_y = elem['invert_y'][elem_slc][:elem_count]
+    stop = elem_off[cc + 1]
-    elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count]
+    elem_targets = elem['targets'][start:stop]
-    elem_scale = elem['scale'][elem_slc][:elem_count]
+    elem_xy = elem['xy'][start:stop]
-    elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count]
+    elem_invert_y = elem['invert_y'][start:stop]
-    elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count]
+    elem_angle_rad = elem['angle_rad'][start:stop]
-    elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count]
+    elem_scale = elem['scale'][start:stop]
-    rep_valid = elem['rep_valid'][elem_slc][:elem_count]
     raw_mode = global_args['raw_mode']
 
+    _append_plain_refs_sorted(
+        pat=pat,
+        cell_names=cell_names,
+        elem_targets=elem_targets,
+        elem_xy=elem_xy,
+        elem_invert_y=elem_invert_y,
+        elem_angle_rad=elem_angle_rad,
+        elem_scale=elem_scale,
+        raw_mode=raw_mode,
+        )
+
+
+def _append_plain_refs_sorted(
+        *,
+        pat: Pattern,
+        cell_names: list[str],
+        elem_targets: NDArray[numpy.integer[Any]],
+        elem_xy: NDArray[numpy.integer[Any]],
+        elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
+        elem_angle_rad: NDArray[numpy.floating[Any]],
+        elem_scale: NDArray[numpy.floating[Any]],
+        raw_mode: bool,
+        ) -> None:
+    elem_count = len(elem_targets)
+    if elem_count == 0:
+        return
+
+    make_ref = Ref._from_raw if raw_mode else Ref
+
+    target_start = 0
+    while target_start < elem_count:
+        target_id = elem_targets[target_start]
+        target_stop = target_start + 1
+        while target_stop < elem_count and elem_targets[target_stop] == target_id:
+            target_stop += 1
+
+        append_refs = pat.refs[cell_names[target_id]].extend
+        append_refs(
+            make_ref(
+                offset=elem_xy[ee],
+                mirrored=elem_invert_y[ee],
+                rotation=elem_angle_rad[ee],
+                scale=elem_scale[ee],
+                repetition=None,
+                annotations=None,
+                )
+            for ee in range(target_start, target_stop)
+            )
+
+        target_start = target_stop
+
+
+def _arefs_to_mrefs(
+        pat: Pattern,
+        global_args: dict[str, Any],
+        elem: dict[str, Any],
+        cc: int,
+        ) -> None:
+    cell_names = global_args['cell_names']
+    elem_off = elem['offsets']
+    elem_count = elem_off[cc + 1] - elem_off[cc]
+    if elem_count == 0:
+        return
+
+    start = elem_off[cc]
+    stop = elem_off[cc + 1]
+    elem_targets = elem['targets'][start:stop]
+    elem_xy = elem['xy'][start:stop]
+    elem_invert_y = elem['invert_y'][start:stop]
+    elem_angle_rad = elem['angle_rad'][start:stop]
+    elem_scale = elem['scale'][start:stop]
+    elem_xy0 = elem['xy0'][start:stop]
+    elem_xy1 = elem['xy1'][start:stop]
+    elem_counts = elem['counts'][start:stop]
+    raw_mode = global_args['raw_mode']
+
+    make_ref = Ref._from_raw if raw_mode else Ref
+    make_grid = Grid._from_raw if raw_mode else Grid
+
+    if len(elem_targets) == 0:
+        return
+
+    target = None
+    append_ref: Callable[[Ref], Any] | None = None
+    for ee in range(len(elem_targets)):
+        if target != elem_targets[ee]:
+            target = elem_targets[ee]
+            append_ref = pat.refs[cell_names[target]].append
+        assert append_ref is not None
+        a_count, b_count = elem_counts[ee]
+        append_ref(make_ref(
+            offset=elem_xy[ee],
+            mirrored=elem_invert_y[ee],
+            rotation=elem_angle_rad[ee],
+            scale=elem_scale[ee],
+            repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
+            annotations=None,
+            ))
+
+
+def _sref_props_to_mrefs(
+        pat: Pattern,
+        global_args: dict[str, Any],
+        elem: dict[str, Any],
+        cc: int,
+        ) -> None:
+    cell_names = global_args['cell_names']
+    elem_off = elem['offsets']
+    prop_key = elem['prop_key']
+    prop_val = elem['prop_val']
+
+    elem_count = elem_off[cc + 1] - elem_off[cc]
+    if elem_count == 0:
+        return
+
+    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
+    prop_offs = elem['prop_off'][elem_slc]
+    elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
+    elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
+    elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
+    elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
+    elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
+    raw_mode = global_args['raw_mode']
+
+    make_ref = Ref._from_raw if raw_mode else Ref
 
     for ee in range(elem_count):
-        target = cell_names[elem_targets[ee]]
-        offset = elem_xy[ee]
-        mirr = elem_invert_y[ee]
-        rot = elem_angle_rad[ee]
-        mag = elem_scale[ee]
-
-        rep: None | Grid = None
-        if rep_valid[ee]:
-            a_vector = elem_rep_xy0[ee]
-            b_vector = elem_rep_xy1[ee]
-            a_count, b_count = elem_rep_counts[ee]
-            if raw_mode:
-                rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
-            else:
-                rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
-
         annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
-        if raw_mode:
+        ref = make_ref(
-            ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
+            offset=elem_xy[ee],
-        else:
+            mirrored=elem_invert_y[ee],
-            ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
+            rotation=elem_angle_rad[ee],
-        pat.refs[target].append(ref)
+            scale=elem_scale[ee],
+            repetition=None,
+            annotations=annotations,
+            )
+        pat.refs[cell_names[elem_targets[ee]]].append(ref)
+
+
+def _aref_props_to_mrefs(
+        pat: Pattern,
+        global_args: dict[str, Any],
+        elem: dict[str, Any],
+        cc: int,
+        ) -> None:
+    cell_names = global_args['cell_names']
+    elem_off = elem['offsets']
+    prop_key = elem['prop_key']
+    prop_val = elem['prop_val']
+
+    elem_count = elem_off[cc + 1] - elem_off[cc]
+    if elem_count == 0:
+        return
+
+    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
+    prop_offs = elem['prop_off'][elem_slc]
+    elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
+    elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
+    elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
+    elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
+    elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
+    elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]]
+    elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]]
+    elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]]
+    raw_mode = global_args['raw_mode']
+
+    make_ref = Ref._from_raw if raw_mode else Ref
+    make_grid = Grid._from_raw if raw_mode else Grid
+
+    for ee in range(elem_count):
+        a_count, b_count = elem_counts[ee]
+        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
+        ref = make_ref(
+            offset=elem_xy[ee],
+            mirrored=elem_invert_y[ee],
+            rotation=elem_angle_rad[ee],
+            scale=elem_scale[ee],
+            repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
+            annotations=annotations,
+            )
+        pat.refs[cell_names[elem_targets[ee]]].append(ref)
 
 
 def _texts_to_labels(
@@ -520,16 +712,53 @@ def _boundary_batches_to_polygons(
     for bb in range(batch_count):
         layer = layer_tups[elem_layer_inds[bb]]
         vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
-        vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp)
+        vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]]
 
         if vertex_offsets.size == 1:
-            poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
+            if raw_mode:
+                poly = Polygon._from_raw(vertices=vertices, annotations=None)
+            else:
+                poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False)
             pat.shapes[layer].append(poly)
         else:
-            polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
+            if raw_mode:
+                polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
+            else:
+                polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False)
             pat.shapes[layer].append(polys)
 
 
+def _rect_batches_to_rectcollections(
+        pat: Pattern,
+        global_args: dict[str, Any],
+        elem: dict[str, Any],
+        cc: int,
+        ) -> None:
+    elem_off = elem['offsets']
+    rect_arr = elem['rect_arr']
+    rect_off = elem['rect_off']
+    layer_inds = elem['layer_inds']
+    layer_tups = global_args['layer_tups']
+
+    batch_count = elem_off[cc + 1] - elem_off[cc]
+    if batch_count == 0:
+        return
+
+    elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1)
+    elem_rect_off = rect_off[elem_slc]
+    elem_layer_inds = layer_inds[elem_slc][:batch_count]
+
+    raw_mode = global_args['raw_mode']
+    for bb in range(batch_count):
+        layer = layer_tups[elem_layer_inds[bb]]
+        rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]]
+        if raw_mode:
+            rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
+        else:
+            rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False)
+        pat.shapes[layer].append(rect_collection)
+
+
 def _boundary_props_to_polygons(
         pat: Pattern,
         global_args: dict[str, Any],
@@ -558,7 +787,10 @@ def _boundary_props_to_polygons(
         layer = layer_tups[elem_layer_inds[ee]]
         vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
         annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
-        poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode)
+        if raw_mode:
+            poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
+        else:
+            poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False)
         pat.shapes[layer].append(poly)
 
 

masque/shapes/poly_collection.py

@@ -34,7 +34,7 @@ class PolyCollection(Shape):
     _vertex_lists: NDArray[numpy.float64]
     """ 2D NDArray ((N+M+...) x 2)  of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """
 
-    _vertex_offsets: NDArray[numpy.intp]
+    _vertex_offsets: NDArray[numpy.integer[Any]]
     """ 1D NDArray specifying the starting offset for each polygon """
 
     @property
@@ -45,7 +45,7 @@ class PolyCollection(Shape):
         return self._vertex_lists
 
     @property
-    def vertex_offsets(self) -> NDArray[numpy.intp]:
+    def vertex_offsets(self) -> NDArray[numpy.integer[Any]]:
         """
         Starting offset (in `vertex_lists`) for each polygon
         """
@@ -63,7 +63,7 @@ class PolyCollection(Shape):
                 chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]),
                 strict=True,
                 ):
-            yield slice(ii, ff)
+            yield slice(int(ii), int(ff))
 
     @property
     def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]:
@@ -105,6 +105,7 @@ class PolyCollection(Shape):
         if raw:
             assert isinstance(vertex_lists, numpy.ndarray)
             assert isinstance(vertex_offsets, numpy.ndarray)
+            assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer)
             self._vertex_lists = vertex_lists
             self._vertex_offsets = vertex_offsets
             self._repetition = repetition
@@ -148,7 +149,7 @@ class PolyCollection(Shape):
         return (
             type(self) is type(other)
             and numpy.array_equal(self._vertex_lists, other._vertex_lists)
-            and numpy.array_equal(self._vertex_offsets, other._vertex_offsets)
+            and numpy.array_equal(self.vertex_offsets, other.vertex_offsets)
             and self.repetition == other.repetition
             and annotations_eq(self.annotations, other.annotations)
             )
@@ -231,11 +232,11 @@ class PolyCollection(Shape):
 
         # TODO: normalize mirroring?
 
-        return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()),
+        return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()),
                 (offset, scale / norm_value, rotation, False),
                 lambda: PolyCollection(
                     vertex_lists=rotated_vertices * norm_value,
-                    vertex_offsets=self._vertex_offsets.copy(),
+                    vertex_offsets=self.vertex_offsets.copy(),
                     ),
                 )
 

masque/test/test_file_roundtrip.py

@@ -5,7 +5,7 @@ from numpy.testing import assert_allclose
 
 from ..pattern import Pattern
 from ..library import Library
-from ..shapes import Path as MPath, Circle, Polygon
+from ..shapes import Path as MPath, Circle, Polygon, RectCollection
 from ..repetition import Grid, Arbitrary
 
 def create_test_library(for_gds: bool = False) -> Library:
@@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None:
     assert poly.repetition is not None
     assert isinstance(poly.repetition, Grid)
     assert poly.repetition.a_count == 5
+
+
+def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None:
+    from ..file import gdsii
+
+    lib = Library()
+    pat = Pattern()
+    pat.shapes[(5, 0)].append(
+        RectCollection(
+            rects=[[0, 0, 10, 5], [20, -5, 30, 10]],
+            annotations={'1': ['rects']},
+            )
+        )
+    lib['rects'] = pat
+
+    gds_file = tmp_path / 'rect_collection.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    read_lib, _ = gdsii.readfile(gds_file)
+    polys = read_lib['rects'].shapes[(5, 0)]
+
+    assert len(polys) == 2
+    assert all(isinstance(poly, Polygon) for poly in polys)
+    assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]])
+    assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]])
+    assert polys[0].annotations == {'1': ['rects']}
+    assert polys[1].annotations == {'1': ['rects']}

masque/test/test_gdsii_arrow.py

@@ -9,7 +9,7 @@ from .. import Ref, Label
 from ..library import Library
 from ..pattern import Pattern
 from ..repetition import Grid
-from ..shapes import Path as MPath
+from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection
 from ..file import gdsii, gdsii_arrow
 from ..file.gdsii_perf import write_fixture
 
@@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]:
     return tuple(tuple(row.tolist()) for row in arr)
 
 
+def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]:
+    arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int)
+    rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])]
+    rev = arr[::-1]
+    rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0]))
+    return min(rows)
+
+
 def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]:
     if isinstance(shape, MPath):
         cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions)
@@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]
         keys.append((
             'polygon',
             layer,
-            _coord_key(poly.vertices),
+            _canonical_polygon_key(poly.vertices),
             _coord_key(poly.offset),
             _annotations_key(poly.annotations),
             ))
     return keys
 
 
-def _ref_key(target: str, ref: object) -> tuple[object, ...]:
+def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]:
-    repetition = None
+    keys = []
-    if ref.repetition is not None:
+    for transform in ref.as_transforms():
-        repetition = (
+        keys.append((
-            _coord_key(ref.repetition.a_vector),
+            target,
-            int(ref.repetition.a_count),
+            _coord_key(transform[:2]),
-            _coord_key(ref.repetition.b_vector),
+            round(float(transform[2]), 8),
-            int(ref.repetition.b_count),
+            round(float(transform[4]), 8),
-            )
+            bool(int(round(float(transform[3])))),
-    return (
+            _annotations_key(ref.annotations),
-        target,
+            ))
-        _coord_key(ref.offset),
+    return keys
-        round(float(ref.rotation), 8),
-        round(float(ref.scale), 8),
-        bool(ref.mirrored),
-        repetition,
-        _annotations_key(ref.annotations),
-        )
 
 
 def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]:
@@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]:
         for shape in shapes:
             shape_keys.extend(_shape_key(shape, layer))
 
-    ref_keys = [
+    ref_keys: list[tuple[object, ...]] = []
-        _ref_key(target, ref)
+    for target, refs in pattern.refs.items():
-        for target, refs in pattern.refs.items()
+        for ref in refs:
-        for ref in refs
+            ref_keys.extend(_ref_keys(target, ref))
-        ]
 
     label_keys = [
         _label_key(layer, label)
@@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library:
         )
     lib['sibling'] = sibling
 
+    fanout = Pattern()
+    fanout.ref('leaf', offset=(0, 0))
+    fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1)
+    fanout.ref('leaf', offset=(20, 0))
+    fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3))
+    fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2,
+               repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2))
+    fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2))
+    fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']})
+    fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2),
+               annotations={'20': ['fanout-aref']})
+    lib['fanout'] = fanout
+
     top = Pattern()
     top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']})
     top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']})
+    top.ref('fanout', offset=(250, -75))
     top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']})
     lib['top'] = top
 
@@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None:
     assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0
 
 
+def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None:
+    lib = Library()
+    leaf = Pattern()
+    leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]])
+    lib['leaf'] = leaf
+
+    top = Pattern()
+    top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1))
+    lib['top'] = top
+
+    gds_file = tmp_path / 'degenerate_aref.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    canonical_lib, _ = gdsii.readfile(gds_file)
+    arrow_lib, _ = gdsii_arrow.readfile(gds_file)
+    assert _library_summary(arrow_lib) == _library_summary(canonical_lib)
+
+    decoded_ref = arrow_lib['top'].refs['leaf'][0]
+    assert decoded_ref.repetition is None
+
+
+def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None:
+    lib = _make_arrow_test_library()
+    gds_file = tmp_path / 'plain_srefs.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    arrow_lib, _ = gdsii_arrow.readfile(gds_file)
+    fanout = arrow_lib['fanout']
+
+    plain_leaf_refs = [
+        ref
+        for ref in fanout.refs['leaf']
+        if ref.annotations is None and ref.repetition is None
+        ]
+    assert len(plain_leaf_refs) == 2
+    assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs)
+
+
+def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None:
+    lib = Library()
+    leaf = Pattern()
+    leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]])
+    lib['leaf'] = leaf
+
+    top = Pattern()
+    top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1))
+    lib['top'] = top
+
+    gds_file = tmp_path / 'degenerate_aref_schema.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
+    cells = libarr['cells'].values
+    cell_ids = cells.field('id').to_numpy()
+    cell_names = libarr['cell_names'].as_py()
+    top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top')
+
+    srefs = cells.field('srefs')[top_index].as_py()
+    arefs = cells.field('arefs')[top_index].as_py()
+
+    assert len(srefs) == 1
+    assert len(arefs) == 0
+    assert cell_names[srefs[0]['target']] == 'leaf'
+
+
 def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
     lib = _make_arrow_test_library()
     gds_file = tmp_path / 'arrow_batches.gds'
@@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
 
     leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf')
 
+    rect_batches = cells.field('rect_batches')[leaf_index].as_py()
     boundary_batches = cells.field('boundary_batches')[leaf_index].as_py()
     boundary_props = cells.field('boundary_props')[leaf_index].as_py()
 
-    assert len(boundary_batches) == 2
+    assert len(rect_batches) == 2
+    assert len(boundary_batches) == 0
     assert len(boundary_props) == 2
 
-    batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches}
+    rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches}
-    assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4]
+    assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10]
-    assert len(batch_by_layer[(1, 0)]['vertices']) == 16
+    assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10]
-    assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0]
-    assert len(batch_by_layer[(2, 0)]['vertices']) == 8
 
     props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props}
     assert sorted(props_by_layer) == [(1, 0), (2, 0)]
@@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
     assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2'
 
 
+def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None:
+    lib = Library()
+    top = Pattern()
+    top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]]))
+    top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]])
+    top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]])
+    lib['top'] = top
+
+    gds_file = tmp_path / 'arrow_rect_batches.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
+    cells = libarr['cells'].values
+    cell_ids = cells.field('id').to_numpy()
+    cell_names = libarr['cell_names'].as_py()
+    layer_table = [
+        ((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF)
+        for layer in libarr['layers'].values.to_numpy()
+        ]
+    top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top')
+
+    rect_batches = cells.field('rect_batches')[top_index].as_py()
+    boundary_batches = cells.field('boundary_batches')[top_index].as_py()
+
+    assert len(rect_batches) == 1
+    assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0)
+    assert rect_batches[0]['rects'] == [
+        0, 0, 10, 10,
+        20, 0, 30, 10,
+        40, 0, 50, 10,
+        60, 0, 70, 10,
+        ]
+
+    assert len(boundary_batches) == 1
+    assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0)
+    assert boundary_batches[0]['vertex_offsets'] == [0, 3]
+
+
+def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None:
+    lib = _make_arrow_test_library()
+    gds_file = tmp_path / 'arrow_ref_batches.gds'
+    gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
+
+    libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
+    cells = libarr['cells'].values
+    cell_ids = cells.field('id').to_numpy()
+    cell_names = libarr['cell_names'].as_py()
+
+    fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout')
+
+    srefs = cells.field('srefs')[fanout_index].as_py()
+    arefs = cells.field('arefs')[fanout_index].as_py()
+    sref_props = cells.field('sref_props')[fanout_index].as_py()
+    aref_props = cells.field('aref_props')[fanout_index].as_py()
+
+    sref_target_ids = [entry['target'] for entry in srefs]
+    sref_targets = [cell_names[target] for target in sref_target_ids]
+    assert sorted(sref_targets) == ['child', 'leaf', 'leaf']
+    assert sref_target_ids == sorted(sref_target_ids)
+    sref_by_target = {}
+    for entry in srefs:
+        sref_by_target.setdefault(cell_names[entry['target']], []).append(entry)
+    assert [entry['invert_y'] for entry in sref_by_target['child']] == [True]
+    assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1])
+    assert len(sref_by_target['leaf']) == 2
+
+    aref_target_ids = [entry['target'] for entry in arefs]
+    aref_targets = [cell_names[target] for target in aref_target_ids]
+    assert sorted(aref_targets) == ['child', 'leaf', 'leaf']
+    assert aref_target_ids == sorted(aref_target_ids)
+    aref_by_target = {}
+    for entry in arefs:
+        aref_by_target.setdefault(cell_names[entry['target']], []).append(entry)
+    assert [entry['invert_y'] for entry in aref_by_target['child']] == [True]
+    assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2])
+    assert len(aref_by_target['leaf']) == 2
+
+    assert len(sref_props) == 1
+    assert cell_names[sref_props[0]['target']] == 'leaf'
+    assert sref_props[0]['properties'][0]['value'] == 'fanout-sref'
+
+    assert len(aref_props) == 1
+    assert cell_names[aref_props[0]['target']] == 'child'
+    assert aref_props[0]['properties'][0]['value'] == 'fanout-aref'
+
+
 def test_raw_ref_grid_label_constructors_match_public() -> None:
     raw_grid = Grid._from_raw(
         a_vector=numpy.array([20, 0]),
@@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None:
     public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2)
     assert raw_grid == public_grid
 
+    raw_poly = Polygon._from_raw(
+        vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]),
+        annotations={'1': ['poly']},
+        )
+    public_poly = Polygon(
+        vertices=[[0, 0], [5, 0], [5, 5], [0, 5]],
+        annotations={'1': ['poly']},
+        )
+    assert raw_poly == public_poly
+
+    raw_poly_collection = PolyCollection._from_raw(
+        vertex_lists=numpy.array([
+            [0.0, 0.0], [2.0, 0.0], [2.0, 2.0],
+            [10.0, 10.0], [12.0, 10.0], [12.0, 12.0],
+            ]),
+        vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32),
+        annotations={'2': ['pc']},
+        )
+    public_poly_collection = PolyCollection(
+        vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]],
+        vertex_offsets=[0, 3],
+        annotations={'2': ['pc']},
+        )
+    assert raw_poly_collection == public_poly_collection
+    assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)]
+
+    raw_rect_collection = RectCollection._from_raw(
+        rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]),
+        annotations={'3': ['rects']},
+        )
+    public_rect_collection = RectCollection(
+        rects=[[0, 0, 5, 5], [10, 10, 12, 12]],
+        annotations={'3': ['rects']},
+        )
+    assert raw_rect_collection == public_rect_collection
+
     raw_ref_empty = Ref._from_raw(
         offset=numpy.array([100, 200]),
         rotation=numpy.pi / 2,