jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[gdsii_arrow] more performance work

Browse source
This commit is contained in:
Jan Petykiewicz 2026-04-02 19:42:52 -07:00
commit 3f63599abe
4 changed files with 558 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

356
masque/file/gdsii_arrow.py
View file

@ -45,7 +45,7 @@ from pyarrow.cffi import ffi
from .utils import is_gzipped, tmpfile
from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape
from ..shapes import Polygon, Path, PolyCollection
from ..shapes import Polygon, Path, PolyCollection, RectCollection
from ..repetition import Grid
from ..utils import layer_t, annotations_t
from ..library import LazyLibrary, Library, ILibrary, ILibraryView
@ -276,6 +276,15 @@ def read_arrow(
poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
)
def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]:
batches = libarr['cells'].values.field('rect_batches')
return dict(
offsets = batches.offsets.to_numpy(),
layer_inds = batches.values.field('layer').to_numpy(),
rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
)
def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]:
boundaries = libarr['cells'].values.field('boundary_props')
return dict(
@ -288,22 +297,46 @@ def read_arrow(
prop_val = boundaries.values.field('properties').values.field('value').to_pylist(),
)
rf = libarr['cells'].values.field('refs')
refs = dict(
offsets = rf.offsets.to_numpy(),
targets = rf.values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()),
invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False),
angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()),
scale = rf.values.field('mag').fill_null(1).to_numpy(),
rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False),
rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()),
rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()),
rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()),
prop_off = rf.values.field('properties').offsets.to_numpy(),
prop_key = rf.values.field('properties').values.field('key').to_numpy(),
prop_val = rf.values.field('properties').values.field('value').to_pylist(),
)
def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
refs = libarr['cells'].values.field(geom_type)
values = refs.values
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
refs = libarr['cells'].values.field(geom_type)
values = refs.values
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
prop_off = values.field('properties').offsets.to_numpy(),
prop_key = values.field('properties').values.field('key').to_numpy(),
prop_val = values.field('properties').values.field('value').to_pylist(),
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
txt = libarr['cells'].values.field('texts')
texts = dict(
@ -317,11 +350,15 @@ def read_arrow(
)
elements = dict(
srefs = get_refs(libarr, 'srefs', has_repetition=False),
arefs = get_refs(libarr, 'arefs', has_repetition=True),
sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False),
aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True),
rect_batches = get_rect_batches(libarr),
boundary_batches = get_boundary_batches(libarr),
boundary_props = get_boundary_props(libarr),
paths = get_geom(libarr, 'paths'),
texts = texts,
refs = refs,
)
paths = libarr['cells'].values.field('paths')
@ -344,10 +381,14 @@ def read_arrow(
for cc in range(len(libarr['cells'])):
name = cell_names[cell_ids[cc]]
pat = Pattern()
_rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc)
_boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc)
_boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc)
_gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
_grefs_to_mrefs(pat, global_args, elements['refs'], cc)
_srefs_to_mrefs(pat, global_args, elements['srefs'], cc)
_arefs_to_mrefs(pat, global_args, elements['arefs'], cc)
_sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc)
_aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc)
_texts_to_labels(pat, global_args, elements['texts'], cc)
mlib[name] = pat
@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]:
return library_info
def _grefs_to_mrefs(
def _srefs_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets'] # which elements belong to each cell
xy = elem['xy']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
targets = elem['targets']
elem_off = elem['offsets']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
elem_targets = targets[elem_slc][:elem_count]
elem_xy = xy[elem_slc][:elem_count]
elem_invert_y = elem['invert_y'][elem_slc][:elem_count]
elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count]
elem_scale = elem['scale'][elem_slc][:elem_count]
elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count]
elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count]
elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count]
rep_valid = elem['rep_valid'][elem_slc][:elem_count]
if elem_count == 0:
return
start = elem_off[cc]
stop = elem_off[cc + 1]
elem_targets = elem['targets'][start:stop]
elem_xy = elem['xy'][start:stop]
elem_invert_y = elem['invert_y'][start:stop]
elem_angle_rad = elem['angle_rad'][start:stop]
elem_scale = elem['scale'][start:stop]
raw_mode = global_args['raw_mode']
_append_plain_refs_sorted(
pat=pat,
cell_names=cell_names,
elem_targets=elem_targets,
elem_xy=elem_xy,
elem_invert_y=elem_invert_y,
elem_angle_rad=elem_angle_rad,
elem_scale=elem_scale,
raw_mode=raw_mode,
)
def _append_plain_refs_sorted(
*,
pat: Pattern,
cell_names: list[str],
elem_targets: NDArray[numpy.integer[Any]],
elem_xy: NDArray[numpy.integer[Any]],
elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
elem_angle_rad: NDArray[numpy.floating[Any]],
elem_scale: NDArray[numpy.floating[Any]],
raw_mode: bool,
) -> None:
elem_count = len(elem_targets)
if elem_count == 0:
return
make_ref = Ref._from_raw if raw_mode else Ref
target_start = 0
while target_start < elem_count:
target_id = elem_targets[target_start]
target_stop = target_start + 1
while target_stop < elem_count and elem_targets[target_stop] == target_id:
target_stop += 1
append_refs = pat.refs[cell_names[target_id]].extend
append_refs(
make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=None,
annotations=None,
)
for ee in range(target_start, target_stop)
)
target_start = target_stop
def _arefs_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
start = elem_off[cc]
stop = elem_off[cc + 1]
elem_targets = elem['targets'][start:stop]
elem_xy = elem['xy'][start:stop]
elem_invert_y = elem['invert_y'][start:stop]
elem_angle_rad = elem['angle_rad'][start:stop]
elem_scale = elem['scale'][start:stop]
elem_xy0 = elem['xy0'][start:stop]
elem_xy1 = elem['xy1'][start:stop]
elem_counts = elem['counts'][start:stop]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
make_grid = Grid._from_raw if raw_mode else Grid
if len(elem_targets) == 0:
return
target = None
append_ref: Callable[[Ref], Any] | None = None
for ee in range(len(elem_targets)):
if target != elem_targets[ee]:
target = elem_targets[ee]
append_ref = pat.refs[cell_names[target]].append
assert append_ref is not None
a_count, b_count = elem_counts[ee]
append_ref(make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
annotations=None,
))
def _sref_props_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
prop_offs = elem['prop_off'][elem_slc]
elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
for ee in range(elem_count):
target = cell_names[elem_targets[ee]]
offset = elem_xy[ee]
mirr = elem_invert_y[ee]
rot = elem_angle_rad[ee]
mag = elem_scale[ee]
rep: None | Grid = None
if rep_valid[ee]:
a_vector = elem_rep_xy0[ee]
b_vector = elem_rep_xy1[ee]
a_count, b_count = elem_rep_counts[ee]
if raw_mode:
rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
else:
rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
if raw_mode:
ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
else:
ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
pat.refs[target].append(ref)
ref = make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=None,
annotations=annotations,
)
pat.refs[cell_names[elem_targets[ee]]].append(ref)
def _aref_props_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
prop_offs = elem['prop_off'][elem_slc]
elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]]
elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]]
elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
make_grid = Grid._from_raw if raw_mode else Grid
for ee in range(elem_count):
a_count, b_count = elem_counts[ee]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
ref = make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
annotations=annotations,
)
pat.refs[cell_names[elem_targets[ee]]].append(ref)
def _texts_to_labels(
@ -520,16 +712,53 @@ def _boundary_batches_to_polygons(
for bb in range(batch_count):
layer = layer_tups[elem_layer_inds[bb]]
vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp)
vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]]
if vertex_offsets.size == 1:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
if raw_mode:
poly = Polygon._from_raw(vertices=vertices, annotations=None)
else:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(poly)
else:
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
if raw_mode:
polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
else:
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(polys)
def _rect_batches_to_rectcollections(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets']
rect_arr = elem['rect_arr']
rect_off = elem['rect_off']
layer_inds = elem['layer_inds']
layer_tups = global_args['layer_tups']
batch_count = elem_off[cc + 1] - elem_off[cc]
if batch_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1)
elem_rect_off = rect_off[elem_slc]
elem_layer_inds = layer_inds[elem_slc][:batch_count]
raw_mode = global_args['raw_mode']
for bb in range(batch_count):
layer = layer_tups[elem_layer_inds[bb]]
rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]]
if raw_mode:
rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
else:
rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(rect_collection)
def _boundary_props_to_polygons(
pat: Pattern,
global_args: dict[str, Any],
@ -558,7 +787,10 @@ def _boundary_props_to_polygons(
layer = layer_tups[elem_layer_inds[ee]]
vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode)
if raw_mode:
poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
else:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False)
pat.shapes[layer].append(poly)

13
masque/shapes/poly_collection.py
View file

@ -34,7 +34,7 @@ class PolyCollection(Shape):
_vertex_lists: NDArray[numpy.float64]
""" 2D NDArray ((N+M+...) x 2) of vertices `[[xa0, ya0], [xa1, ya1], ..., [xb0, yb0], [xb1, yb1], ... ]` """
_vertex_offsets: NDArray[numpy.intp]
_vertex_offsets: NDArray[numpy.integer[Any]]
""" 1D NDArray specifying the starting offset for each polygon """
@property
@ -45,7 +45,7 @@ class PolyCollection(Shape):
return self._vertex_lists
@property
def vertex_offsets(self) -> NDArray[numpy.intp]:
def vertex_offsets(self) -> NDArray[numpy.integer[Any]]:
"""
Starting offset (in `vertex_lists`) for each polygon
"""
@ -63,7 +63,7 @@ class PolyCollection(Shape):
chain(self._vertex_offsets[1:], [self._vertex_lists.shape[0]]),
strict=True,
):
yield slice(ii, ff)
yield slice(int(ii), int(ff))
@property
def polygon_vertices(self) -> Iterator[NDArray[numpy.float64]]:
@ -105,6 +105,7 @@ class PolyCollection(Shape):
if raw:
assert isinstance(vertex_lists, numpy.ndarray)
assert isinstance(vertex_offsets, numpy.ndarray)
assert numpy.issubdtype(vertex_offsets.dtype, numpy.integer)
self._vertex_lists = vertex_lists
self._vertex_offsets = vertex_offsets
self._repetition = repetition
@ -148,7 +149,7 @@ class PolyCollection(Shape):
return (
type(self) is type(other)
and numpy.array_equal(self._vertex_lists, other._vertex_lists)
and numpy.array_equal(self._vertex_offsets, other._vertex_offsets)
and numpy.array_equal(self.vertex_offsets, other.vertex_offsets)
and self.repetition == other.repetition
and annotations_eq(self.annotations, other.annotations)
)
@ -231,11 +232,11 @@ class PolyCollection(Shape):
# TODO: normalize mirroring?
return ((type(self), rotated_vertices.data.tobytes() + self._vertex_offsets.tobytes()),
return ((type(self), rotated_vertices.data.tobytes() + self.vertex_offsets.tobytes()),
(offset, scale / norm_value, rotation, False),
lambda: PolyCollection(
vertex_lists=rotated_vertices * norm_value,
vertex_offsets=self._vertex_offsets.copy(),
vertex_offsets=self.vertex_offsets.copy(),
),
)

29
masque/test/test_file_roundtrip.py
View file

@ -5,7 +5,7 @@ from numpy.testing import assert_allclose
from ..pattern import Pattern
from ..library import Library
from ..shapes import Path as MPath, Circle, Polygon
from ..shapes import Path as MPath, Circle, Polygon, RectCollection
from ..repetition import Grid, Arbitrary
def create_test_library(for_gds: bool = False) -> Library:
@ -150,3 +150,30 @@ def test_oasis_full_roundtrip(tmp_path: Path) -> None:
assert poly.repetition is not None
assert isinstance(poly.repetition, Grid)
assert poly.repetition.a_count == 5
def test_gdsii_rect_collection_roundtrip(tmp_path: Path) -> None:
from ..file import gdsii
lib = Library()
pat = Pattern()
pat.shapes[(5, 0)].append(
RectCollection(
rects=[[0, 0, 10, 5], [20, -5, 30, 10]],
annotations={'1': ['rects']},
)
)
lib['rects'] = pat
gds_file = tmp_path / 'rect_collection.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
read_lib, _ = gdsii.readfile(gds_file)
polys = read_lib['rects'].shapes[(5, 0)]
assert len(polys) == 2
assert all(isinstance(poly, Polygon) for poly in polys)
assert_allclose(polys[0].vertices, [[0, 0], [0, 5], [10, 5], [10, 0]])
assert_allclose(polys[1].vertices, [[20, -5], [20, 10], [30, 10], [30, -5]])
assert polys[0].annotations == {'1': ['rects']}
assert polys[1].annotations == {'1': ['rects']}

264
masque/test/test_gdsii_arrow.py
View file

@ -9,7 +9,7 @@ from .. import Ref, Label
from ..library import Library
from ..pattern import Pattern
from ..repetition import Grid
from ..shapes import Path as MPath
from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection
from ..file import gdsii, gdsii_arrow
from ..file.gdsii_perf import write_fixture
@ -31,6 +31,14 @@ def _coord_key(values: object) -> tuple[int, ...] | tuple[tuple[int, int], ...]:
return tuple(tuple(row.tolist()) for row in arr)
def _canonical_polygon_key(vertices: object) -> tuple[tuple[int, int], ...]:
arr = numpy.rint(numpy.asarray(vertices, dtype=float)).astype(int)
rows = [tuple(tuple(row.tolist()) for row in numpy.roll(arr, -shift, axis=0)) for shift in range(arr.shape[0])]
rev = arr[::-1]
rows.extend(tuple(tuple(row.tolist()) for row in numpy.roll(rev, -shift, axis=0)) for shift in range(rev.shape[0]))
return min(rows)
def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]]:
if isinstance(shape, MPath):
cap_extensions = None if shape.cap_extensions is None else _coord_key(shape.cap_extensions)
@ -50,31 +58,25 @@ def _shape_key(shape: object, layer: tuple[int, int]) -> list[tuple[object, ...]
keys.append((
'polygon',
layer,
_coord_key(poly.vertices),
_canonical_polygon_key(poly.vertices),
_coord_key(poly.offset),
_annotations_key(poly.annotations),
))
return keys
def _ref_key(target: str, ref: object) -> tuple[object, ...]:
repetition = None
if ref.repetition is not None:
repetition = (
_coord_key(ref.repetition.a_vector),
int(ref.repetition.a_count),
_coord_key(ref.repetition.b_vector),
int(ref.repetition.b_count),
)
return (
target,
_coord_key(ref.offset),
round(float(ref.rotation), 8),
round(float(ref.scale), 8),
bool(ref.mirrored),
repetition,
_annotations_key(ref.annotations),
)
def _ref_keys(target: str, ref: object) -> list[tuple[object, ...]]:
keys = []
for transform in ref.as_transforms():
keys.append((
target,
_coord_key(transform[:2]),
round(float(transform[2]), 8),
round(float(transform[4]), 8),
bool(int(round(float(transform[3])))),
_annotations_key(ref.annotations),
))
return keys
def _label_key(layer: tuple[int, int], label: object) -> tuple[object, ...]:
@ -92,11 +94,10 @@ def _pattern_summary(pattern: Pattern) -> dict[str, object]:
for shape in shapes:
shape_keys.extend(_shape_key(shape, layer))
ref_keys = [
_ref_key(target, ref)
for target, refs in pattern.refs.items()
for ref in refs
]
ref_keys: list[tuple[object, ...]] = []
for target, refs in pattern.refs.items():
for ref in refs:
ref_keys.extend(_ref_keys(target, ref))
label_keys = [
_label_key(layer, label)
@ -151,9 +152,23 @@ def _make_arrow_test_library() -> Library:
)
lib['sibling'] = sibling
fanout = Pattern()
fanout.ref('leaf', offset=(0, 0))
fanout.ref('child', offset=(10, 0), mirrored=True, rotation=numpy.pi / 6, scale=1.1)
fanout.ref('leaf', offset=(20, 0))
fanout.ref('leaf', offset=(30, 0), repetition=Grid(a_vector=(5, 0), a_count=2, b_vector=(0, 7), b_count=3))
fanout.ref('child', offset=(40, 0), mirrored=True, rotation=numpy.pi / 4, scale=1.2,
repetition=Grid(a_vector=(9, 0), a_count=2, b_vector=(0, 11), b_count=2))
fanout.ref('leaf', offset=(50, 0), repetition=Grid(a_vector=(6, 0), a_count=3, b_vector=(0, 8), b_count=2))
fanout.ref('leaf', offset=(60, 0), annotations={'19': ['fanout-sref']})
fanout.ref('child', offset=(70, 0), repetition=Grid(a_vector=(4, 0), a_count=2, b_vector=(0, 5), b_count=2),
annotations={'20': ['fanout-aref']})
lib['fanout'] = fanout
top = Pattern()
top.ref('child', offset=(500, 600), annotations={'15': ['top-child-ref']})
top.ref('sibling', offset=(-100, 50), rotation=numpy.pi, annotations={'16': ['top-sibling-ref']})
top.ref('fanout', offset=(250, -75))
top.label((13, 0), string='TOP', offset=(0, 0), annotations={'17': ['top-label']})
lib['top'] = top
@ -184,6 +199,71 @@ def test_gdsii_arrow_reads_small_perf_fixture(tmp_path: Path) -> None:
assert sum(len(refs) for refs in lib['TOP'].refs.values()) > 0
def test_gdsii_arrow_degenerate_aref_decodes_as_single_transform(tmp_path: Path) -> None:
lib = Library()
leaf = Pattern()
leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]])
lib['leaf'] = leaf
top = Pattern()
top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1))
lib['top'] = top
gds_file = tmp_path / 'degenerate_aref.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
canonical_lib, _ = gdsii.readfile(gds_file)
arrow_lib, _ = gdsii_arrow.readfile(gds_file)
assert _library_summary(arrow_lib) == _library_summary(canonical_lib)
decoded_ref = arrow_lib['top'].refs['leaf'][0]
assert decoded_ref.repetition is None
def test_gdsii_arrow_plain_srefs_decode_without_arbitrary(tmp_path: Path) -> None:
lib = _make_arrow_test_library()
gds_file = tmp_path / 'plain_srefs.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
arrow_lib, _ = gdsii_arrow.readfile(gds_file)
fanout = arrow_lib['fanout']
plain_leaf_refs = [
ref
for ref in fanout.refs['leaf']
if ref.annotations is None and ref.repetition is None
]
assert len(plain_leaf_refs) == 2
assert all(type(ref.repetition) is not Grid for ref in plain_leaf_refs)
def test_gdsii_arrow_degenerate_aref_schema_normalizes_to_sref(tmp_path: Path) -> None:
lib = Library()
leaf = Pattern()
leaf.polygon((1, 0), vertices=[[0, 0], [5, 0], [5, 5], [0, 5]])
lib['leaf'] = leaf
top = Pattern()
top.ref('leaf', offset=(100, 200), repetition=Grid(a_vector=(7, 0), a_count=1, b_vector=(0, 9), b_count=1))
lib['top'] = top
gds_file = tmp_path / 'degenerate_aref_schema.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
cells = libarr['cells'].values
cell_ids = cells.field('id').to_numpy()
cell_names = libarr['cell_names'].as_py()
top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top')
srefs = cells.field('srefs')[top_index].as_py()
arefs = cells.field('arefs')[top_index].as_py()
assert len(srefs) == 1
assert len(arefs) == 0
assert cell_names[srefs[0]['target']] == 'leaf'
def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
lib = _make_arrow_test_library()
gds_file = tmp_path / 'arrow_batches.gds'
@ -200,17 +280,17 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
leaf_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'leaf')
rect_batches = cells.field('rect_batches')[leaf_index].as_py()
boundary_batches = cells.field('boundary_batches')[leaf_index].as_py()
boundary_props = cells.field('boundary_props')[leaf_index].as_py()
assert len(boundary_batches) == 2
assert len(rect_batches) == 2
assert len(boundary_batches) == 0
assert len(boundary_props) == 2
batch_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_batches}
assert batch_by_layer[(1, 0)]['vertex_offsets'] == [0, 4]
assert len(batch_by_layer[(1, 0)]['vertices']) == 16
assert batch_by_layer[(2, 0)]['vertex_offsets'] == [0]
assert len(batch_by_layer[(2, 0)]['vertices']) == 8
rects_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in rect_batches}
assert rects_by_layer[(1, 0)]['rects'] == [20, 0, 30, 10, 80, 0, 90, 10]
assert rects_by_layer[(2, 0)]['rects'] == [40, 0, 50, 10]
props_by_layer = {tuple(layer_table[entry['layer']]): entry for entry in boundary_props}
assert sorted(props_by_layer) == [(1, 0), (2, 0)]
@ -218,6 +298,92 @@ def test_gdsii_arrow_boundary_batch_schema(tmp_path: Path) -> None:
assert props_by_layer[(2, 0)]['properties'][0]['value'] == 'leaf-poly-2'
def test_gdsii_arrow_rect_batch_schema_for_mixed_layer(tmp_path: Path) -> None:
lib = Library()
top = Pattern()
top.shapes[(1, 0)].append(RectCollection(rects=[[0, 0, 10, 10], [20, 0, 30, 10], [40, 0, 50, 10], [60, 0, 70, 10]]))
top.polygon((1, 0), vertices=[[80, 0], [85, 10], [90, 0]])
top.polygon((1, 0), vertices=[[100, 0], [105, 10], [110, 0]])
lib['top'] = top
gds_file = tmp_path / 'arrow_rect_batches.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
cells = libarr['cells'].values
cell_ids = cells.field('id').to_numpy()
cell_names = libarr['cell_names'].as_py()
layer_table = [
((int(layer) >> 16) & 0xFFFF, int(layer) & 0xFFFF)
for layer in libarr['layers'].values.to_numpy()
]
top_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'top')
rect_batches = cells.field('rect_batches')[top_index].as_py()
boundary_batches = cells.field('boundary_batches')[top_index].as_py()
assert len(rect_batches) == 1
assert tuple(layer_table[rect_batches[0]['layer']]) == (1, 0)
assert rect_batches[0]['rects'] == [
0, 0, 10, 10,
20, 0, 30, 10,
40, 0, 50, 10,
60, 0, 70, 10,
]
assert len(boundary_batches) == 1
assert tuple(layer_table[boundary_batches[0]['layer']]) == (1, 0)
assert boundary_batches[0]['vertex_offsets'] == [0, 3]
def test_gdsii_arrow_ref_schema(tmp_path: Path) -> None:
lib = _make_arrow_test_library()
gds_file = tmp_path / 'arrow_ref_batches.gds'
gdsii.writefile(lib, gds_file, meters_per_unit=1e-9)
libarr = gdsii_arrow._read_to_arrow(gds_file)[0]
cells = libarr['cells'].values
cell_ids = cells.field('id').to_numpy()
cell_names = libarr['cell_names'].as_py()
fanout_index = next(ii for ii, cell_id in enumerate(cell_ids) if cell_names[cell_id] == 'fanout')
srefs = cells.field('srefs')[fanout_index].as_py()
arefs = cells.field('arefs')[fanout_index].as_py()
sref_props = cells.field('sref_props')[fanout_index].as_py()
aref_props = cells.field('aref_props')[fanout_index].as_py()
sref_target_ids = [entry['target'] for entry in srefs]
sref_targets = [cell_names[target] for target in sref_target_ids]
assert sorted(sref_targets) == ['child', 'leaf', 'leaf']
assert sref_target_ids == sorted(sref_target_ids)
sref_by_target = {}
for entry in srefs:
sref_by_target.setdefault(cell_names[entry['target']], []).append(entry)
assert [entry['invert_y'] for entry in sref_by_target['child']] == [True]
assert [entry['scale'] for entry in sref_by_target['child']] == pytest.approx([1.1])
assert len(sref_by_target['leaf']) == 2
aref_target_ids = [entry['target'] for entry in arefs]
aref_targets = [cell_names[target] for target in aref_target_ids]
assert sorted(aref_targets) == ['child', 'leaf', 'leaf']
assert aref_target_ids == sorted(aref_target_ids)
aref_by_target = {}
for entry in arefs:
aref_by_target.setdefault(cell_names[entry['target']], []).append(entry)
assert [entry['invert_y'] for entry in aref_by_target['child']] == [True]
assert [entry['scale'] for entry in aref_by_target['child']] == pytest.approx([1.2])
assert len(aref_by_target['leaf']) == 2
assert len(sref_props) == 1
assert cell_names[sref_props[0]['target']] == 'leaf'
assert sref_props[0]['properties'][0]['value'] == 'fanout-sref'
assert len(aref_props) == 1
assert cell_names[aref_props[0]['target']] == 'child'
assert aref_props[0]['properties'][0]['value'] == 'fanout-aref'
def test_raw_ref_grid_label_constructors_match_public() -> None:
raw_grid = Grid._from_raw(
a_vector=numpy.array([20, 0]),
@ -228,6 +394,42 @@ def test_raw_ref_grid_label_constructors_match_public() -> None:
public_grid = Grid(a_vector=(20, 0), a_count=3, b_vector=(0, 30), b_count=2)
assert raw_grid == public_grid
raw_poly = Polygon._from_raw(
vertices=numpy.array([[0.0, 0.0], [5.0, 0.0], [5.0, 5.0], [0.0, 5.0]]),
annotations={'1': ['poly']},
)
public_poly = Polygon(
vertices=[[0, 0], [5, 0], [5, 5], [0, 5]],
annotations={'1': ['poly']},
)
assert raw_poly == public_poly
raw_poly_collection = PolyCollection._from_raw(
vertex_lists=numpy.array([
[0.0, 0.0], [2.0, 0.0], [2.0, 2.0],
[10.0, 10.0], [12.0, 10.0], [12.0, 12.0],
]),
vertex_offsets=numpy.array([0, 3], dtype=numpy.uint32),
annotations={'2': ['pc']},
)
public_poly_collection = PolyCollection(
vertex_lists=[[0, 0], [2, 0], [2, 2], [10, 10], [12, 10], [12, 12]],
vertex_offsets=[0, 3],
annotations={'2': ['pc']},
)
assert raw_poly_collection == public_poly_collection
assert [tuple(s.indices(len(raw_poly_collection.vertex_lists))) for s in raw_poly_collection.vertex_slices] == [(0, 3, 1), (3, 6, 1)]
raw_rect_collection = RectCollection._from_raw(
rects=numpy.array([[10.0, 10.0, 12.0, 12.0], [0.0, 0.0, 5.0, 5.0]]),
annotations={'3': ['rects']},
)
public_rect_collection = RectCollection(
rects=[[0, 0, 5, 5], [10, 10, 12, 12]],
annotations={'3': ['rects']},
)
assert raw_rect_collection == public_rect_collection
raw_ref_empty = Ref._from_raw(
offset=numpy.array([100, 200]),
rotation=numpy.pi / 2,