diff --git a/masque/file/gdsii/arrow.py b/masque/file/gdsii/arrow.py index 030bdbb..6a2a42c 100644 --- a/masque/file/gdsii/arrow.py +++ b/masque/file/gdsii/arrow.py @@ -327,11 +327,14 @@ def read_arrow( cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() + # Masque geometry is mutable and supports fractional transforms. Convert + # coordinates in bulk before slicing them into objects; scan-only and raw + # GDS copy-through workflows never enter this materialization path. def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: el = libarr['cells'].values.field(geom_type) elem = dict( offsets = el.offsets.to_numpy(), - xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_arr = el.values.field('xy').values.to_numpy().astype(float).reshape((-1, 2)), xy_off = el.values.field('xy').offsets.to_numpy() // 2, layer_inds = el.values.field('layer').to_numpy(), prop_off = el.values.field('properties').offsets.to_numpy(), @@ -345,7 +348,7 @@ def read_arrow( return dict( offsets = batches.offsets.to_numpy(), layer_inds = batches.values.field('layer').to_numpy(), - vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_arr = batches.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)), vert_off = batches.values.field('vertices').offsets.to_numpy() // 2, poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(), poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(), @@ -356,7 +359,7 @@ def read_arrow( return dict( offsets = batches.offsets.to_numpy(), layer_inds = batches.values.field('layer').to_numpy(), - rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)), + rect_arr = batches.values.field('rects').values.to_numpy().astype(float).reshape((-1, 4)), rect_off = batches.values.field('rects').offsets.to_numpy() // 4, ) @@ -365,7 +368,7 @@ def read_arrow( return dict( offsets = boundaries.offsets.to_numpy(), layer_inds = boundaries.values.field('layer').to_numpy(), - vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)), + vert_arr = boundaries.values.field('vertices').values.to_numpy().astype(float).reshape((-1, 2)), vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2, prop_off = boundaries.values.field('properties').offsets.to_numpy(), prop_key = boundaries.values.field('properties').values.field('key').to_numpy(), @@ -378,15 +381,15 @@ def read_arrow( elem = dict( offsets = refs.offsets.to_numpy(), targets = values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float), invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), angle_rad = values.field('angle_rad').to_numpy(), scale = values.field('scale').to_numpy(), ) if has_repetition: elem.update(dict( - xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), - xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float), counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), )) return elem @@ -397,7 +400,7 @@ def read_arrow( elem = dict( offsets = refs.offsets.to_numpy(), targets = values.field('target').to_numpy(), - xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()), + xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()).astype(float), invert_y = values.field('invert_y').to_numpy(zero_copy_only=False), angle_rad = values.field('angle_rad').to_numpy(), scale = values.field('scale').to_numpy(), @@ -407,8 +410,8 @@ def read_arrow( ) if has_repetition: elem.update(dict( - xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()), - xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()), + xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()).astype(float), + xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()).astype(float), counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()), )) return elem @@ -417,7 +420,7 @@ def read_arrow( texts = dict( offsets = txt.offsets.to_numpy(), layer_inds = txt.values.field('layer').to_numpy(), - xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()), + xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()).astype(float), string = txt.values.field('string').to_pylist(), prop_off = txt.values.field('properties').offsets.to_numpy(), prop_key = txt.values.field('properties').values.field('key').to_numpy(), @@ -443,7 +446,7 @@ def read_arrow( extensions = numpy.stack(( paths.values.field('extension_start').fill_null(0).to_numpy(), paths.values.field('extension_end').fill_null(0).to_numpy(), - ), axis=-1), + ), axis=-1, dtype=float), )) global_args = dict( @@ -517,7 +520,7 @@ def _append_plain_refs_sorted( pat: Pattern, cell_names: list[str], elem_targets: NDArray[numpy.integer[Any]], - elem_xy: NDArray[numpy.integer[Any]], + elem_xy: NDArray[numpy.float64], elem_invert_y: NDArray[numpy.bool_ | numpy.bool], elem_angle_rad: NDArray[numpy.floating[Any]], elem_scale: NDArray[numpy.floating[Any]], diff --git a/masque/file/gdsii/lazy_arrow.py b/masque/file/gdsii/lazy_arrow.py index b20edad..d686bfa 100644 --- a/masque/file/gdsii/lazy_arrow.py +++ b/masque/file/gdsii/lazy_arrow.py @@ -209,6 +209,7 @@ class ArrowLibrary(ILibraryView, IMaterializable): self.path = path self.library_info = payload.library_info self._payload = payload + self._name_to_id = {name: cell_id for cell_id, name in enumerate(payload.cell_names)} self._source = source self._cache: dict[str, Pattern] = {} @@ -368,10 +369,10 @@ class ArrowLibrary(ILibraryView, IMaterializable): ) -> list[NDArray[numpy.float64]]: if parent in self._cache: return super()._raw_ref_transforms(parent, target) - target_cell = self._payload.cells.get(target) - if target_cell is None or parent not in self._payload.cells: + target_id = self._name_to_id.get(target) + if target_id is None or parent not in self._payload.cells: return [] - return self._collect_raw_transforms(self._payload.cells[parent], target_cell.cell_id) + return self._collect_raw_transforms(self._payload.cells[parent], target_id) def readfile( diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index 8f0f39f..0a33453 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -23,6 +23,67 @@ if not gdsii_arrow.is_available(): pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) +def test_arrow_materialized_coordinates_are_writable_floats(tmp_path: Path) -> None: + original = _make_arrow_test_library() + for annotations, layer in [(None, (30, 0)), ({'1': ['prop']}, (31, 0))]: + for xx in (0, 10): + original['leaf'].polygon(layer, [(xx, 0), (xx + 4, 0), (xx, 3)], annotations=annotations) + filename = tmp_path / 'mutable.gds' + gdsii.writefile(original, filename, meters_per_unit=1e-9) + lib, _ = gdsii_arrow.readfile(filename) + arrays = [] + types = set() + for pattern in lib.values(): + for shapes in pattern.shapes.values(): + for shape in shapes: + types.add(type(shape)) + if isinstance(shape, RectCollection): + coordinates = shape.rects + elif isinstance(shape, PolyCollection): + coordinates = shape.vertex_lists + else: + coordinates = shape.vertices + arrays.append(coordinates) + before = coordinates.copy() + shape.translate((0.25, 0.5)).scale_by(1.5) + shift = (0.25, 0.5, 0.25, 0.5) if isinstance(shape, RectCollection) else (0.25, 0.5) + numpy.testing.assert_allclose(coordinates, (before + shift) * 1.5) + if isinstance(shape, MPath) and shape.cap_extensions is not None: + arrays.append(shape.cap_extensions) + for labels in pattern.labels.values(): + arrays.extend(label.offset for label in labels) + for refs in pattern.refs.values(): + for ref in refs: + arrays.append(ref.offset) + ref.translate((0.25, 0.5)) + if ref.repetition is not None: + arrays.extend((ref.repetition.a_vector, ref.repetition.b_vector)) + ref.repetition.scale_by(1.5) + assert {Polygon, PolyCollection, RectCollection, MPath} <= types + for array in arrays: + assert array.dtype == numpy.float64 + assert array.flags.writeable + + +def test_arrow_path_extensions_are_quantized_for_oasis(tmp_path: Path) -> None: + pytest.importorskip('fatamorgana') + from ..file import oasis # noqa: PLC0415 + + source = Library({'top': Pattern().path((1, 0), [(0, 0), (20, 0)], width=4, + cap=MPath.Cap.SquareCustom, cap_extensions=(1, 5))}) + gds_path = tmp_path / 'path.gds' + gdsii.writefile(source, gds_path, meters_per_unit=1e-9) + loaded, _ = gdsii_arrow.readfile(gds_path) + path = loaded['top'].shapes[(1, 0)][0] + path.scale_by(1.25) + exported = oasis.build(loaded, units_per_micron=1000).cells[0].geometry[0] + assert exported.extension_start[1] == 1 + assert exported.extension_end[1] == 6 + assert isinstance(exported.extension_start[1], int | numpy.integer) + assert isinstance(exported.extension_end[1], int | numpy.integer) + numpy.testing.assert_allclose(path.cap_extensions, (1.25, 6.25)) + + def _annotations_key(annotations: dict[str, list[object]] | None) -> tuple[tuple[str, tuple[object, ...]], ...] | None: if not annotations: return None diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 7d840ab..927f35b 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -9,7 +9,7 @@ import pytest pytest.importorskip('pyarrow') -from .. import PatternError +from .. import PatternError, LibraryError from ..library import IBorrowing, IMaterializable, LayerMappedView, Library, OverlayLibrary, PortLoadView from ..pattern import Pattern from ..repetition import Grid @@ -25,6 +25,55 @@ if not gdsii_arrow.is_available(): pytest.skip('klamath_rs_ext shared library is not available', allow_module_level=True) +@pytest.mark.parametrize('cached', [False, True]) +def test_arrow_detached_mutation_is_isolated(tmp_path: Path, cached: bool) -> None: + filename = tmp_path / 'detached.gds' + gdsii.writefile(_make_small_library(), filename, meters_per_unit=1e-9) + with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib: + if cached: + lib['leaf'] + detached = lib.materialize_detached('leaf') + detached.translate_elements((0.25, 0.5)) + numpy.testing.assert_allclose(detached.get_bounds(), [[0.25, 0.5], [10.25, 5.5]]) + numpy.testing.assert_allclose(lib.materialize_detached('leaf').get_bounds(), [[0, 0], [10, 5]]) + assert lib.can_copy_raw_struct('leaf') == (not cached) + top = lib.materialize_detached('top').flatten(lib) + assert top.get_bounds() is not None + + +def test_arrow_rect_hierarchy_requires_explicit_polygonization(tmp_path: Path) -> None: + original = _make_small_library() + original['mid'].refs['leaf'][0].rotation = numpy.pi / 4 + filename = tmp_path / 'rotated.gds' + gdsii.writefile(original, filename, meters_per_unit=1e-9) + with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib: + for action in ('bounds', 'flatten'): + top = lib.materialize_detached('top') + with pytest.raises(PatternError, match='Pattern.polygonize'): + top.get_bounds(lib) if action == 'bounds' else top.flatten(lib) + lib['leaf'].polygonize() + numpy.testing.assert_allclose(lib['top'].get_bounds(lib), original['top'].get_bounds(original)) + assert lib.materialize_detached('top').flatten(lib).get_bounds() is not None + + +def test_arrow_dangling_ref_queries_are_cache_independent(tmp_path: Path) -> None: + original = Library({'parent': Pattern()}) + original['parent'].ref('missing', offset=(3, 4), rotation=numpy.pi / 3, mirrored=True, scale=2) + original['parent'].ref('missing', offset=(10, 20), repetition=Grid(a_vector=(7, 0), a_count=3)) + filename = tmp_path / 'dangling.gds' + gdsii.writefile(original, filename, meters_per_unit=1e-9) + with gdsii_lazy_arrow.ArrowLibrary.from_file(filename) as lib: + expected = _local_refs_key(original.find_refs_local('missing', dangling='include')) + assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected + assert lib.can_copy_raw_struct('parent') + assert not lib.find_refs_local('missing', dangling='ignore') + with pytest.raises(LibraryError, match='missing'): + lib.find_refs_local('missing', dangling='error') + assert not lib.find_refs_local('unknown', dangling='include') + lib['parent'] + assert _local_refs_key(lib.find_refs_local('missing', dangling='include')) == expected + + def test_gdsii_lazy_arrow_has_reader_only_surface() -> None: assert not hasattr(gdsii_lazy_arrow, 'is_available') assert not hasattr(gdsii_lazy_arrow, 'write') @@ -251,12 +300,17 @@ def test_gdsii_lazy_arrow_invalid_path_type_raises_pattern_error(tmp_path: Path) lib['top'] -def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> None: +def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: gds_file = tmp_path / 'copy_source.gds' src = _make_small_library() gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='copy-through') lib, info = gdsii_lazy_arrow.readfile(gds_file) + + def forbid_materialization(*_args, **_kwargs) -> None: + pytest.fail('Untouched GDS writes must not materialize Arrow coordinates') + + monkeypatch.setattr(gdsii_arrow, 'read_arrow', forbid_materialization) out_file = tmp_path / 'copy_out.gds' gdsii.writefile( lib,