diff --git a/masque/file/gdsii/lazy.py b/masque/file/gdsii/lazy.py index 63401d8..7fe1cc4 100644 --- a/masque/file/gdsii/lazy.py +++ b/masque/file/gdsii/lazy.py @@ -7,6 +7,8 @@ This module provides the non-Arrow half of Masque's lazy GDS architecture: struct order, and child edges without materializing every cell. - cells are materialized on demand through the classic `gdsii` decoder whenever a caller indexes the lazy view +- untouched cells can be copied directly to another GDS file without + materializing them - the source can be wrapped in `PortLoadView` or merged through `OverlayLibrary` @@ -66,6 +68,8 @@ class _SourceHandle: class _CellScan: """ Scan-time metadata for one cell in the source stream. """ offset: int + struct_start: int + struct_end: int children: set[str] @@ -78,19 +82,19 @@ def _open_source_stream( if is_gzipped(path): if use_mmap: logger.info('Asked to mmap a gzipped file, reading into memory instead...') - with gzip.open(path, mode='rb') as stream: - data = stream.read() + with gzip.open(path, mode='rb') as gzip_stream: + data = gzip_stream.read() return _SourceHandle(path=path, stream=io.BytesIO(data)) - stream = cast('IO[bytes]', gzip.open(path, mode='rb')) # noqa: SIM115 - return _SourceHandle(path=path, stream=stream) + source_stream = cast('IO[bytes]', gzip.open(path, mode='rb')) # noqa: SIM115 + return _SourceHandle(path=path, stream=source_stream) if use_mmap: handle = path.open(mode='rb', buffering=0) mapped = cast('IO[bytes]', mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ)) return _SourceHandle(path=path, stream=mapped, handle=handle) - stream = path.open(mode='rb') - return _SourceHandle(path=path, stream=stream) + source_stream = path.open(mode='rb') + return _SourceHandle(path=path, stream=source_stream) def _scan_library( @@ -100,19 +104,26 @@ def _scan_library( order: list[str] = [] cells: dict[str, _CellScan] = {} - found_struct = records.BGNSTR.skip_past(stream) - while found_struct: + while True: + struct_start = stream.tell() + if not records.BGNSTR.skip_past(stream): + break name = records.STRNAME.skip_and_read(stream).decode('ASCII') offset = stream.tell() elements = klamath.library.read_elements(stream) + struct_end = stream.tell() children = { element.struct_name.decode('ASCII') for element in elements if isinstance(element, klamath.elements.Reference) } order.append(name) - cells[name] = _CellScan(offset=offset, children=children) - found_struct = records.BGNSTR.skip_past(stream) + cells[name] = _CellScan( + offset=offset, + struct_start=struct_start, + struct_end=struct_end, + children=children, + ) return library_info, order, cells @@ -121,8 +132,9 @@ class GdsLibrarySource(ILibraryView, IMaterializable): """ Read-only library backed by a seekable GDS stream. - Cells are scanned once up front to discover order and child edges, then - materialized one at a time through the classic `gdsii.read_elements` path. + Cells are scanned once up front to discover order, byte ranges, and child + edges. Untouched structures can be copied directly, while accessed cells + are materialized through the classic `gdsii.read_elements` path. The source owns the stream lifetime, preserves on-disk ordering through `source_order()`, and answers graph queries from scan metadata whenever @@ -172,6 +184,20 @@ class GdsLibrarySource(ILibraryView, IMaterializable): def source_order(self) -> tuple[str, ...]: return self._cell_order + def can_copy_raw_struct(self, name: str) -> bool: + """Return whether `name` still matches its original GDS structure.""" + return name in self._cells and name not in self._cache + + def raw_struct_bytes(self, name: str) -> bytes: + """Read the original complete GDS structure for `name`.""" + cell = self._cells[name] + stream = self._source.stream + stream.seek(cell.struct_start) + data = stream.read(cell.struct_end - cell.struct_start) + if len(data) != cell.struct_end - cell.struct_start: + raise LibraryError(f'Unexpected end of GDS source while copying structure {name!r}') + return data + def materialize(self, name: str, *, persist: bool = True) -> Pattern: if name in self._cache: return self._cache[name] diff --git a/masque/test/test_gdsii_lazy.py b/masque/test/test_gdsii_lazy.py index 3f12928..2b7b424 100644 --- a/masque/test/test_gdsii_lazy.py +++ b/masque/test/test_gdsii_lazy.py @@ -1,4 +1,5 @@ from pathlib import Path +import gzip import io import numpy @@ -8,6 +9,7 @@ from numpy.testing import assert_allclose from ..file import gdsii from ..file.gdsii import lazy as gdsii_lazy from ..file.gdsii import lazy_write as gdsii_lazy_write +from ..file.utils import preflight_source_aware from ..error import LibraryError from ..pattern import Pattern from ..ports import Port @@ -404,6 +406,130 @@ def test_gdsii_lazy_processed_write_roundtrips_without_explicit_units(tmp_path: assert out_file.read_bytes() == gds_file.read_bytes() +def test_gdsii_lazy_source_aware_write_copies_untouched_structures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + gds_file = tmp_path / 'classic_copy_source.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-copy') + + raw, _ = gdsii_lazy.readfile(gds_file) + ports = PortLoadView(raw, layers=[(10, 0)], max_depth=2) + mapped = LayerMappedView(ports, lambda layer: (20, 0) if layer == (10, 0) else layer, copy_through=True) + prepared = preflight_source_aware(mapped) + copied: list[str] = [] + raw_reader = raw.raw_struct_bytes + + def record_raw_read(name: str) -> bytes: + copied.append(name) + return raw_reader(name) + + def fail_materialize(_name: str, *, persist: bool = True) -> Pattern: # noqa: ARG001 + raise AssertionError('untouched cells must not be materialized') + + monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read) + monkeypatch.setattr(raw, 'materialize', fail_materialize) + out_file = tmp_path / 'classic_copy_out.gds' + gdsii_lazy.writefile(prepared, out_file) + + assert copied == ['leaf', 'child', 'top'] + assert not raw._cache + assert not ports._cache + assert not mapped._cache + assert out_file.read_bytes() == gds_file.read_bytes() + + +@pytest.mark.parametrize('use_mmap', [False, True]) +def test_gdsii_lazy_raw_copy_stream_backends(tmp_path: Path, use_mmap: bool) -> None: + gds_file = tmp_path / 'classic_stream_source.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-stream') + + raw, _ = gdsii_lazy.readfile(gds_file, use_mmap=use_mmap) + out_file = tmp_path / f'classic_stream_{use_mmap}.gds' + gdsii_lazy.writefile(raw, out_file) + + assert not raw._cache + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_raw_copy_gzipped_source(tmp_path: Path) -> None: + gds_file = tmp_path / 'classic_gzip_source.gds' + gz_file = tmp_path / 'classic_gzip_source.gds.gz' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-gzip') + with gzip.open(gz_file, 'wb') as stream: + stream.write(gds_file.read_bytes()) + + raw, _ = gdsii_lazy.readfile(gz_file, use_mmap=False) + for name in raw.source_order(): + raw.raw_struct_bytes(name) + assert raw._source.stream.tell() == raw._cells[name].struct_end + out_file = tmp_path / 'classic_gzip_out.gds' + gdsii_lazy.writefile(raw, out_file) + + assert not raw._cache + assert out_file.read_bytes() == gds_file.read_bytes() + + +def test_gdsii_lazy_cached_source_cell_disables_only_its_raw_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + gds_file = tmp_path / 'classic_cached_source.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-cached') + + raw, _ = gdsii_lazy.readfile(gds_file) + assert raw.can_copy_raw_struct('leaf') + raw['leaf'].label((30, 0), string='cached', offset=(1, 2)) + assert not raw.can_copy_raw_struct('leaf') + copied: list[str] = [] + raw_reader = raw.raw_struct_bytes + + def record_raw_read(name: str) -> bytes: + copied.append(name) + return raw_reader(name) + + monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read) + out_file = tmp_path / 'classic_cached_out.gds' + gdsii_lazy.writefile(raw, out_file) + + assert copied == ['child', 'top'] + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip['leaf'].labels) == {(10, 0), (30, 0)} + + +def test_gdsii_lazy_materialized_cell_disables_only_its_raw_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + gds_file = tmp_path / 'classic_partial_copy_source.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-partial-copy') + + raw, _ = gdsii_lazy.readfile(gds_file) + mapped = LayerMappedView(raw, lambda layer: (20, 0) if layer == (10, 0) else layer, copy_through=True) + assert set(mapped['leaf'].labels) == {(20, 0)} + prepared = preflight_source_aware(mapped) + copied: list[str] = [] + raw_reader = raw.raw_struct_bytes + + def record_raw_read(name: str) -> bytes: + copied.append(name) + return raw_reader(name) + + monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read) + out_file = tmp_path / 'classic_partial_copy_out.gds' + gdsii_lazy.writefile(prepared, out_file) + + assert copied == ['child', 'top'] + assert not raw._cache + roundtrip, _ = gdsii.readfile(out_file) + assert set(roundtrip['leaf'].labels) == {(20, 0)} + + def test_gdsii_lazy_layer_mapped_write_materializes_without_source_cache(tmp_path: Path) -> None: gds_file = tmp_path / 'lazy_layer_source.gds' src = _make_lazy_port_library()