[arrow] make sure loaded values are mutable

masque/file/gdsii/arrow.py

@@ -327,11 +327,14 @@ def read_arrow(
     cell_ids = libarr['cells'].values.field('id').to_numpy()
     cell_names = libarr['cell_names'].as_py()
 
+    # Masque geometry is mutable and supports fractional transforms. Convert
+    # coordinates in bulk before slicing them into objects; scan-only and raw
+    # GDS copy-through workflows never enter this materialization path.
     def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]:
         el = libarr['cells'].values.field(geom_type)
         elem = dict(
             offsets = el.offsets.to_numpy(),
-            xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)),
+            xy_arr = el.values.field('xy').values.to_numpy().astype(float).reshape((-1, 2)),
             xy_off = el.values.field('xy').offsets.to_numpy() // 2,
             layer_inds = el.values.field('layer').to_numpy(),
             prop_off = el.values.field('properties').offsets.to_numpy(),
@@ -345,7 +348,7 @@ def read_arrow(
         return dict(
             offsets = batches.offsets.to_numpy(),
             layer_inds = batches.values.field('layer').to_numpy(),
-            vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)),
+            vert_arr = batches.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)),
             vert_off = batches.values.field('vertices').offsets.to_numpy() // 2,
             poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(),
             poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
@@ -356,7 +359,7 @@ def read_arrow(
         return dict(
             offsets = batches.offsets.to_numpy(),
             layer_inds = batches.values.field('layer').to_numpy(),
-            rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
+            rect_arr = batches.values.field('rects').values.to_numpy().astype(float).reshape((-1, 4)),
             rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
             )
 
@@ -365,7 +368,7 @@ def read_arrow(
         return dict(
             offsets = boundaries.offsets.to_numpy(),
             layer_inds = boundaries.values.field('layer').to_numpy(),
-            vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)),
+            vert_arr = boundaries.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)),
             vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2,
             prop_off = boundaries.values.field('properties').offsets.to_numpy(),
             prop_key = boundaries.values.field('properties').values.field('key').to_numpy(),
@@ -378,15 +381,15 @@ def read_arrow(
         elem = dict(
             offsets = refs.offsets.to_numpy(),
             targets = values.field('target').to_numpy(),
-            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
+            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float),
             invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
             angle_rad = values.field('angle_rad').to_numpy(),
             scale = values.field('scale').to_numpy(),
             )
         if has_repetition:
             elem.update(dict(
-                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
+                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float),
-                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
+                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float),
                 counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
                 ))
         return elem
@@ -397,7 +400,7 @@ def read_arrow(
         elem = dict(
             offsets = refs.offsets.to_numpy(),
             targets = values.field('target').to_numpy(),
-            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
+            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float),
             invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
             angle_rad = values.field('angle_rad').to_numpy(),
             scale = values.field('scale').to_numpy(),
@@ -407,8 +410,8 @@ def read_arrow(
             )
         if has_repetition:
             elem.update(dict(
-                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
+                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float),
-                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
+                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float),
                 counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
                 ))
         return elem
@@ -417,7 +420,7 @@ def read_arrow(
     texts = dict(
         offsets = txt.offsets.to_numpy(),
         layer_inds = txt.values.field('layer').to_numpy(),
-        xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()),
+        xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()).astype(float),
         string = txt.values.field('string').to_pylist(),
         prop_off = txt.values.field('properties').offsets.to_numpy(),
         prop_key = txt.values.field('properties').values.field('key').to_numpy(),
@@ -443,7 +446,7 @@ def read_arrow(
         extensions = numpy.stack((
             paths.values.field('extension_start').fill_null(0).to_numpy(),
             paths.values.field('extension_end').fill_null(0).to_numpy(),
-            ), axis=-1),
+            ), axis=-1, dtype=float),
         ))
 
     global_args = dict(
@@ -517,7 +520,7 @@ def _append_plain_refs_sorted(
         pat: Pattern,
         cell_names: list[str],
         elem_targets: NDArray[numpy.integer[Any]],
-        elem_xy: NDArray[numpy.integer[Any]],
+        elem_xy: NDArray[numpy.float64],
         elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
         elem_angle_rad: NDArray[numpy.floating[Any]],
         elem_scale: NDArray[numpy.floating[Any]],

masque/file/gdsii/lazy_arrow.py

@@ -209,6 +209,7 @@ class ArrowLibrary(ILibraryView, IMaterializable):
         self.path = path
         self.library_info = payload.library_info
         self._payload = payload
+        self._name_to_id = {name: cell_id for cell_id, name in enumerate(payload.cell_names)}
         self._source = source
         self._cache: dict[str, Pattern] = {}
 
@@ -368,10 +369,10 @@ class ArrowLibrary(ILibraryView, IMaterializable):
             ) -> list[NDArray[numpy.float64]]:
         if parent in self._cache:
             return super()._raw_ref_transforms(parent, target)
-        target_cell = self._payload.cells.get(target)
+        target_id = self._name_to_id.get(target)
-        if target_cell is None or parent not in self._payload.cells:
+        if target_id is None or parent not in self._payload.cells:
             return []
-        return self._collect_raw_transforms(self._payload.cells[parent], target_cell.cell_id)
+        return self._collect_raw_transforms(self._payload.cells[parent], target_id)
 
 
 def readfile(

masque/test/test_gdsii_arrow.py

@@ -23,6 +23,67 @@ if not gdsii_arrow.is_available():
     pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
 
 
+def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None:
+    original = _make_arrow_test_library()
+    for annotations, layer in [(None, (30, 0)), ({'1': ['prop']}, (31, 0))]:
+        for xx in (0, 10):
+            original['leaf'].polygon(layer, [(xx, 0), (xx + 4, 0), (xx, 3)], annotations=annotations)
+    filename = tmp_path / 'mutable.gds'
+    gdsii.writefile(original, filename, meters_per_unit=1e-9)
+    lib, _ = gdsii_arrow.readfile(filename)
+    arrays = []
+    types = set()
+    for pattern in lib.values():
+        for shapes in pattern.shapes.values():
+            for shape in shapes:
+                types.add(type(shape))
+                if isinstance(shape, RectCollection):
+                    coordinates = shape.rects
+                elif isinstance(shape, PolyCollection):
+                    coordinates = shape.vertex_lists
+                else:
+                    coordinates = shape.vertices
+                arrays.append(coordinates)
+                before = coordinates.copy()
+                shape.translate((0.25, 0.5)).scale_by(1.5)
+                shift = (0.25, 0.5, 0.25, 0.5) if isinstance(shape, RectCollection) else (0.25, 0.5)
+                numpy.testing.assert_allclose(coordinates, (before + shift) * 1.5)
+                if isinstance(shape, MPath) and shape.cap_extensions is not None:
+                    arrays.append(shape.cap_extensions)
+        for labels in pattern.labels.values():
+            arrays.extend(label.offset for label in labels)
+        for refs in pattern.refs.values():
+            for ref in refs:
+                arrays.append(ref.offset)
+                ref.translate((0.25, 0.5))
+                if ref.repetition is not None:
+                    arrays.extend((ref.repetition.a_vector, ref.repetition.b_vector))
+                    ref.repetition.scale_by(1.5)
+    assert {Polygon, PolyCollection, RectCollection, MPath} <= types
+    for array in arrays:
+        assert array.dtype == numpy.float64
+        assert array.flags.writeable
+
+
+def test_arrow_path_extensions_are_quantized_for_oasis(tmp_path: Path) -> None:
+    pytest.importorskip('fatamorgana')
+    from ..file import oasis        # noqa: PLC0415
+
+    source = Library({'top': Pattern().path((1, 0), [(0, 0), (20, 0)], width=4,
+                                           cap=MPath.Cap.SquareCustom, cap_extensions=(1, 5))})
+    gds_path = tmp_path / 'path.gds'
+    gdsii.writefile(source, gds_path, meters_per_unit=1e-9)
+    loaded, _ = gdsii_arrow.readfile(gds_path)
+    path = loaded['top'].shapes[(1, 0)][0]
+    path.scale_by(1.25)
+    exported = oasis.build(loaded, units_per_micron=1000).cells[0].geometry[0]
+    assert exported.extension_start[1] == 1
+    assert exported.extension_end[1] == 6
+    assert isinstance(exported.extension_start[1], int | numpy.integer)
+    assert isinstance(exported.extension_end[1], int | numpy.integer)
+    numpy.testing.assert_allclose(path.cap_extensions, (1.25, 6.25))
+
+
 def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None:
     if not annotations:
         return None

masque/test/test_gdsii_lazy_arrow.py

@@ -9,7 +9,7 @@ import pytest
 
 pytest.importorskip('pyarrow')
 
-from .. import PatternError
+from .. import PatternError, LibraryError
 from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView
 from ..pattern import Pattern
 from ..repetition import Grid
@@ -25,6 +25,55 @@ if not gdsii_arrow.is_available():
     pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True)
 
 
+@pytest.mark.parametrize('cached', [False, True])
+def test_arrow_detached_mutation_is_isolated(tmp_path: Path, cached: bool) -> None:
+    filename = tmp_path / 'detached.gds'
+    gdsii.writefile(_make_small_library(), filename, meters_per_unit=1e-9)
+    with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
+        if cached:
+            lib['leaf']
+        detached = lib.materialize_detached('leaf')
+        detached.translate_elements((0.25, 0.5))
+        numpy.testing.assert_allclose(detached.get_bounds(), [[0.25, 0.5], [10.25, 5.5]])
+        numpy.testing.assert_allclose(lib.materialize_detached('leaf').get_bounds(), [[0, 0], [10, 5]])
+        assert lib.can_copy_raw_struct('leaf') == (not cached)
+        top = lib.materialize_detached('top').flatten(lib)
+        assert top.get_bounds() is not None
+
+
+def test_arrow_rect_hierarchy_requires_explicit_polygonization(tmp_path: Path) -> None:
+    original = _make_small_library()
+    original['mid'].refs['leaf'][0].rotation = numpy.pi / 4
+    filename = tmp_path / 'rotated.gds'
+    gdsii.writefile(original, filename, meters_per_unit=1e-9)
+    with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
+        for action in ('bounds', 'flatten'):
+            top = lib.materialize_detached('top')
+            with pytest.raises(PatternError, match='Pattern.polygonize'):
+                top.get_bounds(lib) if action == 'bounds' else top.flatten(lib)
+        lib['leaf'].polygonize()
+        numpy.testing.assert_allclose(lib['top'].get_bounds(lib), original['top'].get_bounds(original))
+        assert lib.materialize_detached('top').flatten(lib).get_bounds() is not None
+
+
+def test_arrow_dangling_ref_queries_are_cache_independent(tmp_path: Path) -> None:
+    original = Library({'parent': Pattern()})
+    original['parent'].ref('missing', offset=(3, 4), rotation=numpy.pi / 3, mirrored=True, scale=2)
+    original['parent'].ref('missing', offset=(10, 20), repetition=Grid(a_vector=(7, 0), a_count=3))
+    filename = tmp_path / 'dangling.gds'
+    gdsii.writefile(original, filename, meters_per_unit=1e-9)
+    with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib:
+        expected = _local_refs_key(original.find_refs_local('missing', dangling='include'))
+        assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected
+        assert lib.can_copy_raw_struct('parent')
+        assert not lib.find_refs_local('missing', dangling='ignore')
+        with pytest.raises(LibraryError, match='missing'):
+            lib.find_refs_local('missing', dangling='error')
+        assert not lib.find_refs_local('unknown', dangling='include')
+        lib['parent']
+        assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected
+
+
 def test_gdsii_lazy_arrow_has_reader_only_surface() -> None:
     assert not hasattr(gdsii_lazy_arrow, 'is_available')
     assert not hasattr(gdsii_lazy_arrow, 'write')
@@ -251,12 +300,17 @@ def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path)
         lib['top']
 
 
-def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None:
+def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     gds_file = tmp_path / 'copy_source.gds'
     src = _make_small_library()
     gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through')
 
     lib, info = gdsii_lazy_arrow.readfile(gds_file)
+
+    def forbid_materialization(*_args, **_kwargs) -> None:
+        pytest.fail('Untouched GDS writes must not materialize Arrow coordinates')
+
+    monkeypatch.setattr(gdsii_arrow, 'read_arrow', forbid_materialization)
     out_file = tmp_path / 'copy_out.gds'
     gdsii.writefile(
         lib,