[gdsii] enable lazy write-through

masque/file/gdsii/lazy.py

@@ -7,6 +7,8 @@ This module provides the non-Arrow half of Masque's lazy GDS architecture:
   struct order, and child edges without materializing every cell.
 - cells are materialized on demand through the classic `gdsii` decoder
   whenever a caller indexes the lazy view
+- untouched cells can be copied directly to another GDS file without
+  materializing them
 - the source can be wrapped in `PortLoadView` or merged through
   `OverlayLibrary`
 
@@ -66,6 +68,8 @@ class _SourceHandle:
 class _CellScan:
     """ Scan-time metadata for one cell in the source stream. """
     offset: int
+    struct_start: int
+    struct_end: int
     children: set[str]
 
 
@@ -78,19 +82,19 @@ def _open_source_stream(
     if is_gzipped(path):
         if use_mmap:
             logger.info('Asked to mmap a gzipped file, reading into memory instead...')
-            with gzip.open(path, mode='rb') as stream:
+            with gzip.open(path, mode='rb') as gzip_stream:
-                data = stream.read()
+                data = gzip_stream.read()
             return _SourceHandle(path=path, stream=io.BytesIO(data))
-        stream = cast('IO[bytes]', gzip.open(path, mode='rb'))      # noqa: SIM115
+        source_stream = cast('IO[bytes]', gzip.open(path, mode='rb'))      # noqa: SIM115
-        return _SourceHandle(path=path, stream=stream)
+        return _SourceHandle(path=path, stream=source_stream)
 
     if use_mmap:
         handle = path.open(mode='rb', buffering=0)
         mapped = cast('IO[bytes]', mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ))
         return _SourceHandle(path=path, stream=mapped, handle=handle)
 
-    stream = path.open(mode='rb')
+    source_stream = path.open(mode='rb')
-    return _SourceHandle(path=path, stream=stream)
+    return _SourceHandle(path=path, stream=source_stream)
 
 
 def _scan_library(
@@ -100,19 +104,26 @@ def _scan_library(
     order: list[str] = []
     cells: dict[str, _CellScan] = {}
 
-    found_struct = records.BGNSTR.skip_past(stream)
+    while True:
-    while found_struct:
+        struct_start = stream.tell()
+        if not records.BGNSTR.skip_past(stream):
+            break
         name = records.STRNAME.skip_and_read(stream).decode('ASCII')
         offset = stream.tell()
         elements = klamath.library.read_elements(stream)
+        struct_end = stream.tell()
         children = {
             element.struct_name.decode('ASCII')
             for element in elements
             if isinstance(element, klamath.elements.Reference)
         }
         order.append(name)
-        cells[name] = _CellScan(offset=offset, children=children)
+        cells[name] = _CellScan(
-        found_struct = records.BGNSTR.skip_past(stream)
+            offset=offset,
+            struct_start=struct_start,
+            struct_end=struct_end,
+            children=children,
+            )
 
     return library_info, order, cells
 
@@ -121,8 +132,9 @@ class GdsLibrarySource(ILibraryView, IMaterializable):
     """
     Read-only library backed by a seekable GDS stream.
 
-    Cells are scanned once up front to discover order and child edges, then
+    Cells are scanned once up front to discover order, byte ranges, and child
-    materialized one at a time through the classic `gdsii.read_elements` path.
+    edges. Untouched structures can be copied directly, while accessed cells
+    are materialized through the classic `gdsii.read_elements` path.
 
     The source owns the stream lifetime, preserves on-disk ordering through
     `source_order()`, and answers graph queries from scan metadata whenever
@@ -172,6 +184,20 @@ class GdsLibrarySource(ILibraryView, IMaterializable):
     def source_order(self) -> tuple[str, ...]:
         return self._cell_order
 
+    def can_copy_raw_struct(self, name: str) -> bool:
+        """Return whether `name` still matches its original GDS structure."""
+        return name in self._cells and name not in self._cache
+
+    def raw_struct_bytes(self, name: str) -> bytes:
+        """Read the original complete GDS structure for `name`."""
+        cell = self._cells[name]
+        stream = self._source.stream
+        stream.seek(cell.struct_start)
+        data = stream.read(cell.struct_end - cell.struct_start)
+        if len(data) != cell.struct_end - cell.struct_start:
+            raise LibraryError(f'Unexpected end of GDS source while copying structure {name!r}')
+        return data
+
     def materialize(self, name: str, *, persist: bool = True) -> Pattern:
         if name in self._cache:
             return self._cache[name]

masque/test/test_gdsii_lazy.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import gzip
 import io
 
 import numpy
@@ -8,6 +9,7 @@ from numpy.testing import assert_allclose
 from ..file import gdsii
 from ..file.gdsii import lazy as gdsii_lazy
 from ..file.gdsii import lazy_write as gdsii_lazy_write
+from ..file.utils import preflight_source_aware
 from ..error import LibraryError
 from ..pattern import Pattern
 from ..ports import Port
@@ -404,6 +406,130 @@ def test_gdsii_lazy_processed_write_roundtrips_without_explicit_units(tmp_path:
     assert out_file.read_bytes() == gds_file.read_bytes()
 
 
+def test_gdsii_lazy_source_aware_write_copies_untouched_structures(
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
+        ) -> None:
+    gds_file = tmp_path / 'classic_copy_source.gds'
+    src = _make_lazy_port_library()
+    gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-copy')
+
+    raw, _ = gdsii_lazy.readfile(gds_file)
+    ports = PortLoadView(raw, layers=[(10, 0)], max_depth=2)
+    mapped = LayerMappedView(ports, lambda layer: (20, 0) if layer == (10, 0) else layer, copy_through=True)
+    prepared = preflight_source_aware(mapped)
+    copied: list[str] = []
+    raw_reader = raw.raw_struct_bytes
+
+    def record_raw_read(name: str) -> bytes:
+        copied.append(name)
+        return raw_reader(name)
+
+    def fail_materialize(_name: str, *, persist: bool = True) -> Pattern:     # noqa: ARG001
+        raise AssertionError('untouched cells must not be materialized')
+
+    monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read)
+    monkeypatch.setattr(raw, 'materialize', fail_materialize)
+    out_file = tmp_path / 'classic_copy_out.gds'
+    gdsii_lazy.writefile(prepared, out_file)
+
+    assert copied == ['leaf', 'child', 'top']
+    assert not raw._cache
+    assert not ports._cache
+    assert not mapped._cache
+    assert out_file.read_bytes() == gds_file.read_bytes()
+
+
+@pytest.mark.parametrize('use_mmap', [False, True])
+def test_gdsii_lazy_raw_copy_stream_backends(tmp_path: Path, use_mmap: bool) -> None:
+    gds_file = tmp_path / 'classic_stream_source.gds'
+    src = _make_lazy_port_library()
+    gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-stream')
+
+    raw, _ = gdsii_lazy.readfile(gds_file, use_mmap=use_mmap)
+    out_file = tmp_path / f'classic_stream_{use_mmap}.gds'
+    gdsii_lazy.writefile(raw, out_file)
+
+    assert not raw._cache
+    assert out_file.read_bytes() == gds_file.read_bytes()
+
+
+def test_gdsii_lazy_raw_copy_gzipped_source(tmp_path: Path) -> None:
+    gds_file = tmp_path / 'classic_gzip_source.gds'
+    gz_file = tmp_path / 'classic_gzip_source.gds.gz'
+    src = _make_lazy_port_library()
+    gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-gzip')
+    with gzip.open(gz_file, 'wb') as stream:
+        stream.write(gds_file.read_bytes())
+
+    raw, _ = gdsii_lazy.readfile(gz_file, use_mmap=False)
+    for name in raw.source_order():
+        raw.raw_struct_bytes(name)
+        assert raw._source.stream.tell() == raw._cells[name].struct_end
+    out_file = tmp_path / 'classic_gzip_out.gds'
+    gdsii_lazy.writefile(raw, out_file)
+
+    assert not raw._cache
+    assert out_file.read_bytes() == gds_file.read_bytes()
+
+
+def test_gdsii_lazy_cached_source_cell_disables_only_its_raw_copy(
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
+        ) -> None:
+    gds_file = tmp_path / 'classic_cached_source.gds'
+    src = _make_lazy_port_library()
+    gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-cached')
+
+    raw, _ = gdsii_lazy.readfile(gds_file)
+    assert raw.can_copy_raw_struct('leaf')
+    raw['leaf'].label((30, 0), string='cached', offset=(1, 2))
+    assert not raw.can_copy_raw_struct('leaf')
+    copied: list[str] = []
+    raw_reader = raw.raw_struct_bytes
+
+    def record_raw_read(name: str) -> bytes:
+        copied.append(name)
+        return raw_reader(name)
+
+    monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read)
+    out_file = tmp_path / 'classic_cached_out.gds'
+    gdsii_lazy.writefile(raw, out_file)
+
+    assert copied == ['child', 'top']
+    roundtrip, _ = gdsii.readfile(out_file)
+    assert set(roundtrip['leaf'].labels) == {(10, 0), (30, 0)}
+
+
+def test_gdsii_lazy_materialized_cell_disables_only_its_raw_copy(
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
+        ) -> None:
+    gds_file = tmp_path / 'classic_partial_copy_source.gds'
+    src = _make_lazy_port_library()
+    gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-partial-copy')
+
+    raw, _ = gdsii_lazy.readfile(gds_file)
+    mapped = LayerMappedView(raw, lambda layer: (20, 0) if layer == (10, 0) else layer, copy_through=True)
+    assert set(mapped['leaf'].labels) == {(20, 0)}
+    prepared = preflight_source_aware(mapped)
+    copied: list[str] = []
+    raw_reader = raw.raw_struct_bytes
+
+    def record_raw_read(name: str) -> bytes:
+        copied.append(name)
+        return raw_reader(name)
+
+    monkeypatch.setattr(raw, 'raw_struct_bytes', record_raw_read)
+    out_file = tmp_path / 'classic_partial_copy_out.gds'
+    gdsii_lazy.writefile(prepared, out_file)
+
+    assert copied == ['child', 'top']
+    assert not raw._cache
+    roundtrip, _ = gdsii.readfile(out_file)
+    assert set(roundtrip['leaf'].labels) == {(20, 0)}
+
+
 def test_gdsii_lazy_layer_mapped_write_materializes_without_source_cache(tmp_path: Path) -> None:
     gds_file = tmp_path / 'lazy_layer_source.gds'
     src = _make_lazy_port_library()