jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[arrow] make sure loaded values are mutable

Browse source
This commit is contained in:
Jan Petykiewicz 2026-09-14 21:51:50 -07:00
commit 70c4cb3589
4 changed files with 137 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

29
masque/file/gdsii/arrow.py
View file

@ -327,11 +327,14 @@ def read_arrow(
cell_ids = libarr['cells'].values.field('id').to_numpy()
cell_names = libarr['cell_names'].as_py()
# Masque geometry is mutable and supports fractional transforms. Convert
# coordinates in bulk before slicing them into objects; scan-only and raw
# GDS copy-through workflows never enter this materialization path.
def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]:
el = libarr['cells'].values.field(geom_type)
elem = dict(
offsets = el.offsets.to_numpy(),
xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)),
xy_arr = el.values.field('xy').values.to_numpy().astype(float).reshape((-1, 2)),
xy_off = el.values.field('xy').offsets.to_numpy() // 2,
layer_inds = el.values.field('layer').to_numpy(),
prop_off = el.values.field('properties').offsets.to_numpy(),
@ -345,7 +348,7 @@ def read_arrow(
return dict(
offsets = batches.offsets.to_numpy(),
layer_inds = batches.values.field('layer').to_numpy(),
vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)),
vert_arr = batches.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)),
vert_off = batches.values.field('vertices').offsets.to_numpy() // 2,
poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(),
poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
@ -356,7 +359,7 @@ def read_arrow(
return dict(
offsets = batches.offsets.to_numpy(),
layer_inds = batches.values.field('layer').to_numpy(),
rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
rect_arr = batches.values.field('rects').values.to_numpy().astype(float).reshape((-1, 4)),
rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
)
@ -365,7 +368,7 @@ def read_arrow(
return dict(
offsets = boundaries.offsets.to_numpy(),
layer_inds = boundaries.values.field('layer').to_numpy(),
vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)),
vert_arr = boundaries.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)),
vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2,
prop_off = boundaries.values.field('properties').offsets.to_numpy(),
prop_key = boundaries.values.field('properties').values.field('key').to_numpy(),
@ -378,15 +381,15 @@ def read_arrow(
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
@ -397,7 +400,7 @@ def read_arrow(
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
@ -407,8 +410,8 @@ def read_arrow(
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
@ -417,7 +420,7 @@ def read_arrow(
texts = dict(
offsets = txt.offsets.to_numpy(),
layer_inds = txt.values.field('layer').to_numpy(),
xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()),
xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()).astype(float),
string = txt.values.field('string').to_pylist(),
prop_off = txt.values.field('properties').offsets.to_numpy(),
prop_key = txt.values.field('properties').values.field('key').to_numpy(),
@ -443,7 +446,7 @@ def read_arrow(
extensions = numpy.stack((
paths.values.field('extension_start').fill_null(0).to_numpy(),
paths.values.field('extension_end').fill_null(0).to_numpy(),
), axis=-1),
), axis=-1, dtype=float),
))
global_args = dict(
@ -517,7 +520,7 @@ def _append_plain_refs_sorted(
pat: Pattern,
cell_names: list[str],
elem_targets: NDArray[numpy.integer[Any]],
elem_xy: NDArray[numpy.integer[Any]],
elem_xy: NDArray[numpy.float64],
elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
elem_angle_rad: NDArray[numpy.floating[Any]],
elem_scale: NDArray[numpy.floating[Any]],

7
masque/file/gdsii/lazy_arrow.py
View file

@ -209,6 +209,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
self.path = path
self.library_info = payload.library_info
self._payload = payload
self._name_to_id = {name: cell_id for cell_id, name in enumerate(payload.cell_names)}
self._source = source
self._cache: dict[str, Pattern] = {}
@ -368,10 +369,10 @@ class ArrowLibrary(ILibraryView, IMaterializable):
) -> list[NDArray[numpy.float64]]:
if parent in self._cache:
return super()._raw_ref_transforms(parent, target)
target_cell = self._payload.cells.get(target)
if target_cell is None or parent not in self._payload.cells:
target_id = self._name_to_id.get(target)
if target_id is None or parent not in self._payload.cells:
return []
return self._collect_raw_transforms(self._payload.cells[parent], target_cell.cell_id)
return self._collect_raw_transforms(self._payload.cells[parent], target_id)
def readfile(

61
masque/test/test_gdsii_arrow.py
View file

@ -23,6 +23,67 @@ if not gdsii_arrow.is_available():
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None:
original = _make_arrow_test_library()
for annotations, layer in [(None, (30, 0)), ({'1': ['prop']}, (31, 0))]:
for xx in (0, 10):
original['leaf'].polygon(layer, [(xx, 0), (xx + 4, 0), (xx, 3)], annotations=annotations)
filename = tmp_path / 'mutable.gds'
gdsii.writefile(original, filename, meters_per_unit=1e-9)
lib, _ = gdsii_arrow.readfile(filename)
arrays = []
types = set()
for pattern in lib.values():
for shapes in pattern.shapes.values():
for shape in shapes:
types.add(type(shape))
if isinstance(shape, RectCollection):
coordinates = shape.rects
elif isinstance(shape, PolyCollection):
coordinates = shape.vertex_lists
else:
coordinates = shape.vertices
arrays.append(coordinates)
before = coordinates.copy()
shape.translate((0.25, 0.5)).scale_by(1.5)
shift = (0.25, 0.5, 0.25, 0.5) if isinstance(shape, RectCollection) else (0.25, 0.5)
numpy.testing.assert_allclose(coordinates, (before + shift) * 1.5)
if isinstance(shape, MPath) and shape.cap_extensions is not None:
arrays.append(shape.cap_extensions)
for labels in pattern.labels.values():
arrays.extend(label.offset for label in labels)
for refs in pattern.refs.values():
for ref in refs:
arrays.append(ref.offset)
ref.translate((0.25, 0.5))
if ref.repetition is not None:
arrays.extend((ref.repetition.a_vector, ref.repetition.b_vector))
ref.repetition.scale_by(1.5)
assert {Polygon, PolyCollection, RectCollection, MPath} <= types
for array in arrays:
assert array.dtype == numpy.float64
assert array.flags.writeable
def test_arrow_path_extensions_are_quantized_for_oasis(tmp_path: Path) -> None:
pytest.importorskip('fatamorgana')
from ..file import oasis # noqa: PLC0415
source = Library({'top': Pattern().path((1, 0), [(0, 0), (20, 0)], width=4,
cap=MPath.Cap.SquareCustom, cap_extensions=(1, 5))})
gds_path = tmp_path / 'path.gds'
gdsii.writefile(source, gds_path, meters_per_unit=1e-9)
loaded, _ = gdsii_arrow.readfile(gds_path)
path = loaded['top'].shapes[(1, 0)][0]
path.scale_by(1.25)
exported = oasis.build(loaded, units_per_micron=1000).cells[0].geometry[0]
assert exported.extension_start[1] == 1
assert exported.extension_end[1] == 6
assert isinstance(exported.extension_start[1], int | numpy.integer)
assert isinstance(exported.extension_end[1], int | numpy.integer)
numpy.testing.assert_allclose(path.cap_extensions, (1.25, 6.25))
def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None:
if not annotations:
return None

58
masque/test/test_gdsii_lazy_arrow.py
View file

@ -9,7 +9,7 @@ import pytest
pytest.importorskip('pyarrow')
from .. import PatternError
from .. import PatternError, LibraryError
from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView
from ..pattern import Pattern
from ..repetition import Grid
@ -25,6 +25,55 @@ if not gdsii_arrow.is_available():
pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
@pytest.mark.parametrize('cached', [False, True])
def test_arrow_detached_mutation_is_isolated(tmp_path: Path, cached: bool) -> None:
filename = tmp_path / 'detached.gds'
gdsii.writefile(_make_small_library(), filename, meters_per_unit=1e-9)
with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
if cached:
lib['leaf']
detached = lib.materialize_detached('leaf')
detached.translate_elements((0.25, 0.5))
numpy.testing.assert_allclose(detached.get_bounds(), [[0.25, 0.5], [10.25, 5.5]])
numpy.testing.assert_allclose(lib.materialize_detached('leaf').get_bounds(), [[0, 0], [10, 5]])
assert lib.can_copy_raw_struct('leaf') == (not cached)
top = lib.materialize_detached('top').flatten(lib)
assert top.get_bounds() is not None
def test_arrow_rect_hierarchy_requires_explicit_polygonization(tmp_path: Path) -> None:
original = _make_small_library()
original['mid'].refs['leaf'][0].rotation = numpy.pi / 4
filename = tmp_path / 'rotated.gds'
gdsii.writefile(original, filename, meters_per_unit=1e-9)
with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
for action in ('bounds', 'flatten'):
top = lib.materialize_detached('top')
with pytest.raises(PatternError, match='Pattern.polygonize'):
top.get_bounds(lib) if action == 'bounds' else top.flatten(lib)
lib['leaf'].polygonize()
numpy.testing.assert_allclose(lib['top'].get_bounds(lib), original['top'].get_bounds(original))
assert lib.materialize_detached('top').flatten(lib).get_bounds() is not None
def test_arrow_dangling_ref_queries_are_cache_independent(tmp_path: Path) -> None:
original = Library({'parent': Pattern()})
original['parent'].ref('missing', offset=(3, 4), rotation=numpy.pi / 3, mirrored=True, scale=2)
original['parent'].ref('missing', offset=(10, 20), repetition=Grid(a_vector=(7, 0), a_count=3))
filename = tmp_path / 'dangling.gds'
gdsii.writefile(original, filename, meters_per_unit=1e-9)
with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
expected = _local_refs_key(original.find_refs_local('missing', dangling='include'))
assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected
assert lib.can_copy_raw_struct('parent')
assert not lib.find_refs_local('missing', dangling='ignore')
with pytest.raises(LibraryError, match='missing'):
lib.find_refs_local('missing', dangling='error')
assert not lib.find_refs_local('unknown', dangling='include')
lib['parent']
assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected
def test_gdsii_lazy_arrow_has_reader_only_surface() -> None:
assert not hasattr(gdsii_lazy_arrow, 'is_available')
assert not hasattr(gdsii_lazy_arrow, 'write')
@ -251,12 +300,17 @@ def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path)
lib['top']
def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None:
def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
gds_file = tmp_path / 'copy_source.gds'
src = _make_small_library()
gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through')
lib, info = gdsii_lazy_arrow.readfile(gds_file)
def forbid_materialization(*_args, **_kwargs) -> None:
pytest.fail('Untouched GDS writes must not materialize Arrow coordinates')
monkeypatch.setattr(gdsii_arrow, 'read_arrow', forbid_materialization)
out_file = tmp_path / 'copy_out.gds'
gdsii.writefile(
lib,