From 0edd735a115b4c64dd7467e741b2213ad4d1ed01 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 13 Jul 2026 12:24:33 -0700 Subject: [PATCH] [library] More library rework --- MIGRATION.md | 334 +++++++++++++- examples/tutorial/README.md | 2 +- examples/tutorial/library.py | 20 +- masque/__init__.py | 5 +- masque/file/gdsii/lazy.py | 107 +---- masque/file/gdsii/lazy_arrow.py | 91 +--- masque/file/gdsii/lazy_write.py | 74 ++-- masque/library/__init__.py | 7 +- masque/library/base.py | 325 +++++++------- masque/library/build.py | 334 +++++++------- masque/library/capabilities.py | 42 ++ masque/library/lazy.py | 20 +- masque/library/mapping.py | 107 ++++- masque/library/overlay.py | 241 ++++------ masque/library/utils.py | 137 +++++- masque/pattern.py | 83 +++- masque/test/test_autotool_planning.py | 106 ++++- masque/test/test_build_library.py | 465 ++++++++++++++++---- masque/test/test_gdsii_lazy.py | 95 +++- masque/test/test_gdsii_lazy_arrow.py | 68 ++- masque/test/test_library.py | 450 ++++++++++++++++++- masque/test/test_pather_constraints.py | 2 +- masque/test/test_pather_primitive_offers.py | 101 ++++- masque/test/test_ports2data.py | 9 + masque/utils/ports2data.py | 4 +- 25 files changed, 2358 insertions(+), 871 deletions(-) create mode 100644 masque/library/capabilities.py diff --git a/MIGRATION.md b/MIGRATION.md index fdda9e5..1a54793 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,7 +1,8 @@ # Migration Guide -This guide covers changes between the git tag `release` and the current tree. -At `release`, `masque.__version__` was `3.3`; the current tree reports `3.4`. +This guide covers changes between the `master` branch and the current tree. +Both `master` and the current tree report `masque.__version__ == '3.4'`; the +version string has not yet been bumped for these changes. Most downstream changes are in `masque/builder/*`, but there are a few other API changes that may require code updates. @@ -126,28 +127,40 @@ builder = Pather(...) deferred = Pather(..., render='deferred') ``` -Top-level imports from `masque` also continue to work. +The new `Pather` remains importable from both `masque` and `masque.builder`. +The removed `Builder` and `RenderPather` names are no longer exported from +either location. `Pather` now defaults to `render='auto'`, so plain construction replaces the old `Builder` behavior. Use `Pather(..., render='deferred')` where you previously used `RenderPather`. -## `BasicTool` was replaced +## `SimpleTool` was removed and `AutoTool` registration changed -`BasicTool` is no longer exported. Use `AutoTool` for reusable straight, bend, -transition, and S-bend primitives. +`SimpleTool` is no longer exported. `AutoTool` remains, but its old public +descriptor classes and constructor-oriented configuration were replaced by +registration methods. Use it for generated straights and S-bends and reusable +bends, U-turns, and transitions. -### Old `BasicTool` +### Old `AutoTool` ```python -from masque.builder.tools import BasicTool +from masque.builder import AutoTool -tool = BasicTool( - straight=(make_straight, 'input', 'output'), - bend=(lib.abstract('bend'), 'input', 'output'), +tool = AutoTool( + straights=[ + AutoTool.Straight('m1wire', make_straight, 'input', 'output'), + ], + bends=[ + AutoTool.Bend(lib.abstract('bend'), 'input', 'output'), + ], + sbends=[], transitions={ - 'm2wire': (lib.abstract('via'), 'top', 'bottom'), + ('m2wire', 'm1wire'): AutoTool.Transition( + lib.abstract('via'), 'top', 'bottom' + ), }, + default_out_ptype='m1wire', ) ``` @@ -166,12 +179,44 @@ tool = ( The key differences are: -- `BasicTool` -> `AutoTool` -- `straight=(fn, in_name, out_name)` -> `add_straight(fn, ptype, in_name)` -- `bend=(abstract, in_name, out_name)` -> `add_bend(abstract, in_name, out_name)` +- `SimpleTool` was removed; use `AutoTool` or implement the new `Tool` + primitive-offer interface +- `AutoTool.Straight(...)` -> `add_straight(fn, ptype, in_name)` +- `AutoTool.Bend(...)` -> `add_bend(abstract, in_name, out_name)` +- `AutoTool.SBend(...)` -> `add_sbend(fn, ptype, in_name, out_name)` +- reusable native U-turns can be registered with `add_uturn(...)` - transitions are registered with `add_transition(abstract, external_port, internal_port)` - transitions are bidirectional by default; pass `one_way=True` to inhibit the reverse adapter +Primitive costs are now explicit and independent of `AutoTool` registration +order. Every `add_*()` method accepts `cost=`, as do the concrete offer +factories. A numeric value scales the default geometric cost; for example, +`cost=2` makes a primitive twice as expensive. A callable receives the +canonical primitive parameter and local endpoint and returns the complete +cost: + +```python +tool.add_straight(make_straight, 'm1wire', 'input', cost=1.5) +tool.add_sbend( + make_sbend, + 'm1wire', + 'input', + 'output', + cost=lambda jog, endpoint: abs(jog) + 2 * abs(endpoint.x), +) +``` + +`PrimitiveOffer.priority_bias` was removed; custom offers should use `cost` +instead. Exact equal-cost candidates still use deterministic discovery order +as the final tie-break, but registration order no longer changes their +reported cost. + +For two-port primitives, `AutoTool` can infer omitted port names and, for +generated straight/S-bend primitives, omitted ptype metadata by sampling an +in-domain example. Supply those values explicitly when generation requires +route-specific keyword arguments, because metadata inference does not receive +`tool_options`. + ## Custom `Tool` subclasses If you maintain your own `Tool` subclass, the interface changed: @@ -412,25 +457,268 @@ Check code that calls: - `Ref.rotate(...)` - `Ref.mirror(...)` - `Label.rotate_around(...)` / `Label.mirror(...)` +- `Abstract.mirror_port_offsets(...)` / `Abstract.mirror_ports(...)` If that code expected offsets or repetition grids to move automatically, it needs updating. For whole-pattern transforms, prefer calling `Pattern.mirror()` or `Pattern.rotate_around(...)` at the container level. +`Abstract.mirror_port_offsets()` and `Abstract.mirror_ports()` were removed. +Use `Abstract.mirror(axis)` to mirror both port locations and orientations. If +you intentionally need only one half of that operation, update the individual +ports explicitly with `Port.flip_across(...)` or `Port.mirror(...)`. + +## Library hierarchy and graph behavior + +`masque/library.py` was split into the `masque.library` package. Imports from +the public module remain stable: + +```python +from masque import Library, LazyLibrary +# or +from masque.library import Library, LazyLibrary +``` + +Code importing the implementation file itself must move to the public package; +do not depend on the new internal `base`, `mapping`, or `lazy` module paths. + +Hierarchy helpers now handle dangling references explicitly. The following +methods accept `dangling='error' | 'ignore' | 'include'` and default to +`'error'`: + +- `child_graph()` +- `parent_graph()` +- `child_order()` +- `find_refs_local()` +- `find_refs_global()` +- `prune_empty()` + +On `master`, graph construction could expose missing targets implicitly or +fail later with a `KeyError`. If dangling refs are intentional, choose the +behavior explicitly, for example: + +```python +graph = library.child_graph(dangling='include') +order = library.child_order(dangling='ignore') +``` + +Graph cycles and invalid hierarchy states are now reported as `LibraryError` +with context. Audit code that caught `KeyError` or `graphlib.CycleError` from +these helpers. + +Invalid `dangling=` strings now raise `ValueError`; they no longer fall through +to the `include` behavior. Empty lists stored in `Pattern.refs` are consistently +treated as absent references by hierarchy and geometry traversal. Code that +uses a `defaultdict` lookup such as `pattern.refs[name]` without appending a +`Ref` will therefore not create an edge or force that target to be loaded. + +`Library.add()` now resolves the full name plan and remaps references before it +starts inserting cells. Name-resolution and preparation failures no longer +leave partially-added cells behind. A `rename_theirs` callback now receives an +`INameView` containing both existing names and names reserved earlier in the +same addition. Use membership, iteration, `len()`, or `get_name()`; the callback +argument is not the destination object and does not support pattern lookup or +mapping helpers such as `keys()` and `items()`. Update callback annotations from +`ILibraryView` to `INameView`. Custom `_merge()` implementations remain +responsible for their own rollback if they fail during the final commit. + +Reference transforms returned by `find_refs_local()` and +`find_refs_global()` are now Nx5 arrays. The fifth column is the cumulative +scale, so rows have the form `(x, y, rotation, mirrored, scale)` rather than the +old Nx4 form. + +`BuildReport` mapping fields are now defensively copied and read-only. Copy a +field to a new `dict` before adding or removing report entries. + +`PortsLibraryView`, `OverlayLibrary`, and source-backed outputs returned by +`LibraryBuilder.build()` borrow their sources. They do not close source resources, and +`PortsLibraryView` is not a context manager. Keep each lazy source open and +unchanged until every borrowing view or overlay is finished, then close the +owning source explicitly. An eager `build(output='library')` result is detached +and does not need its sources afterward. + +Read-only `subtree()` results are now borrowed lazy views rather than eager +`LibraryView` snapshots. Creating one no longer loads its reachable patterns, +and the view preserves source ordering, hierarchy metadata, and lazy GDS +copy-through. Mutable `Library`, `LazyLibrary`, and `OverlayLibrary` subtrees +return the same writable type as their source. Overlay subtrees retain their +source layers and lazy GDS capabilities. Mutable subtree containers are +structurally independent, but already-materialized patterns remain shared. +Keep borrowed sources open and structurally unchanged for the subtree's +lifetime. + +`ILibrary` no longer inherits `collections.abc.MutableMapping`. It remains a +readable `Mapping` with explicit insert-only item assignment and deletion, but +generic mutation helpers such as `update()`, `setdefault()`, `pop()`, +`popitem()`, and `clear()` are no longer supplied. Use `add()`, `rename()`, +`delete()`, item insertion, and item deletion so library name and reference +invariants remain explicit. + +Library-level `referenced_patterns()` and `dangling_refs()` now report only +named cell targets and return `set[str]`. Populated refs whose target is `None` +are ignored, matching `child_graph()` and recursive geometry behavior; +`Pattern.referenced_patterns()` continues to report `None` locally. + +Port-importing views now always process detached patterns, including when the +raw source cell was already cached. Code can safely retain and compare raw and +processed views without port overrides leaking back into the raw pattern. +Once a processed cell is persistently materialized, lazy GDS writers no longer +copy the raw source structure for that cell, so later mutations to the returned +`Pattern` are serialized. Non-persistent materialization does not mark the cell +as changed. + +Underscore-prefixed declarations work through the attribute authoring surface: +`builder.cells._helper = pattern` now declares `_helper`. Only the view's exact +internal `_library` attribute is reserved. + +Recursive geometry operations now reject cyclic reference hierarchies with a +contextual `PatternError` instead of eventually leaking `RecursionError`. This +applies to bounds calculation, flattened layer polygon extraction, and +visualization as well as the existing flattening checks. + +`LibraryBuilder.validate(names=...)` now accepts either one string or a sequence +of strings. Non-string roots raise `TypeError`, and duplicate roots are reduced +to their first occurrence. A recipe may build a different `LibraryBuilder`, but +calling `build()` or `validate()` recursively on its own active builder now +raises `BuildError` before starting another session. + +`LibraryBuilder` is an authoring registry, not an `ILibraryView` or mapping. +Use membership, iteration, `keys()`, `get_name()`, assignment, and deletion to +manage declarations, then use the library returned by `build()` for reads and +hierarchy operations. Recipes that need an active library must receive the +builder-owned placeholder as a direct argument: + +```python +def make_top(lib: ILibrary) -> Pattern: + return Pather(library=lib, ports='device').pattern + +builder.cells.top = cell(make_top)(builder.library) +builder.cells.device = cell(factory)(hole_lib=builder.library) +``` + +The builder-owned `builder.library` placeholder is read-only. Only direct +positional and keyword values equal to it are substituted; placeholders nested +inside containers are not interpreted. A placeholder from another builder is +rejected when the recipe is assigned. + +`IMaterializable` now identifies libraries which support explicit +`materialize()` and `materialize_many()` operations. `LibraryBuilder.add()` +borrows these marked inputs, while ordinary mappings and `ILibraryView` +instances are copied eagerly. Use `add_source()` to force borrowing of an +unmarked view. Wrapping a materializable library in `LibraryView` intentionally +erases the marker. + +`IBorrowing` separately identifies composite views which retain direct source +libraries and expose them through `borrowed_sources()`. Keep those sources open +and unchanged for the lifetime of the borrowing view. Owner libraries such as +`LazyLibrary` and the lazy GDS readers are materializable but do not implement +`IBorrowing`. GDS raw-structure copy-through remains a separate, +format-specific capability. + +`LibraryBuilder`, `OverlayLibrary`, and `PortsLibraryView` are new additive +library implementations. `LibraryBuilder` supports declarative `@cell` recipes +and dependency-aware builds; `OverlayLibrary` composes source libraries without +eagerly copying all patterns; `PortsLibraryView` overlays port metadata on a +read-only source. + +Flattening with `flatten_ports=True` now rejects repeated refs whose target has +ports, because expanding them would create duplicate port names. Resolve the +repetition and assign unique port names before flattening, or use +`flatten_ports=False`. + +## GDSII module and lazy-loading changes + +`masque.file.gdsii` changed from a module into a package. The eager klamath +API remains available at the old import path, so ordinary `read`, `readfile`, +`write`, and `writefile` calls do not need to change: + +```python +from masque.file import gdsii +library, info = gdsii.readfile('layout.gds') +``` + +The old `gdsii.load_library()` and `gdsii.load_libraryfile()` entry points were +removed. Use the source-backed lazy reader instead: + +```python +# old +library, info = gdsii.load_libraryfile('layout.gds') + +# new +from masque.file.gdsii import lazy + +library, info = lazy.readfile('layout.gds') +try: + pattern = library['TOP'] +finally: + library.close() +``` + +`lazy.read(stream)` and `lazy.readfile(path, use_mmap=...)` return a read-only +`GdsLibrarySource`. It owns file resources when it opens them and also supports +the context-manager protocol. The old `full_load` and `postprocess` arguments +are gone; materialize/copy the desired cells and post-process them explicitly. + +An optional Arrow/native backend is available through +`masque.file.gdsii.arrow` and `masque.file.gdsii.lazy_arrow`; install the new +`arrow` extra to use it. These modules are additive and are not a transparent +replacement unless their additional dependencies and native library are +available. + +## Shape construction and geometry additions + +The public `raw=True` constructor shortcut was removed from `Arc`, `Circle`, +`Ellipse`, `Path`, `Polygon`, `PolyCollection`, and `Text`. Call their normal +constructors without `raw`; `_from_raw()` is an internal fast path and is not a +compatibility API. + +```python +# old +polygon = Polygon(vertices, raw=True) + +# new +polygon = Polygon(vertices) +``` + +`Arc` radii must now be strictly positive rather than merely non-negative. +`Arc.angle_ref` is additive and defaults to `Arc.AngleRef.Center`, preserving +the previous center-referenced angle interpretation. + +`RectCollection` is a new shape for batches of axis-aligned rectangles and is +exported from both `masque` and `masque.shapes`. `Polygon.boolean()` and the +top-level `masque.boolean()` helper are also new; install the `boolean` extra +for their `pyclipper` dependency. + ## Other user-facing changes +### File writers + +SVG writing no longer polygonizes or flattens caller-owned patterns in place; +it works from detached copies. `svg.writefile(..., annotate_ports=True)` can +add port arrows. DXF writing now expands shape repetitions into individual DXF +entities, so callers no longer need to wrap repeated shapes solely for DXF +output. + ### DXF environments If you install the DXF extra, the supported `ezdxf` baseline moved from `~=1.0.2` to `~=1.4`. Any pinned environments should be updated accordingly. +### Optional dependency names + +The misspelled `manhatanize_slow` extra was corrected to +`manhattanize_slow`. A separate `manhattanize` extra now installs the +scikit-image implementation. The `arrow` and `boolean` extras are also new. + ### New exports These are additive, but available now from `masque` and `masque.builder`: -- `PortPather` -- `AutoTool` -- `boolean` +- from `masque`: `RectCollection`, `boolean`, `OverlayLibrary`, + `PortsLibraryView`, `IMaterializable`, `IBorrowing`, `LibraryBuilder`, + `BuildReport`, `CellProvenance`, and `cell` +- from `masque.builder`: `CostCallable`, the concrete primitive-offer classes, + and structured route error/status types ## Minimal migration checklist @@ -438,11 +726,15 @@ If your code uses the routing stack, do these first: 1. Replace `path`/`path_to`/`mpath`/`path_into` calls with `trace`/`trace_to`/multi-port `trace`/`trace_into`. -2. Replace `BasicTool` with `AutoTool`. +2. Replace `SimpleTool` and old `AutoTool` descriptor construction with the + new `AutoTool.add_*()` methods. 3. Fix imports that still reference `masque.builder.builder` or `masque.builder.renderpather`. 4. Audit any low-level `mirror()` usage, especially on `Port` and `Ref`. +5. Move lazy GDS calls from `gdsii.load_library*()` to + `masque.file.gdsii.lazy`. +6. Remove `raw=True` from public shape constructors. If your code only uses `Pattern`, `Library`, `place()`, and `plug()` without the -routing helpers, you may not need any changes beyond the transform audit and any -stale imports. +routing helpers, audit transforms, dangling-reference graph calls, raw shape +construction, and any stale imports. diff --git a/examples/tutorial/README.md b/examples/tutorial/README.md index aab8ad2..7aee3f7 100644 --- a/examples/tutorial/README.md +++ b/examples/tutorial/README.md @@ -20,7 +20,7 @@ Contents * Use `Pather` to snap ports together into a circuit * Check for dangling references - [library](library.py) - * Continue from `devices.py` by declaring a mixed library with `BuildLibrary` + * Continue from `devices.py` by declaring a mixed library with `LibraryBuilder` * Import source-backed GDS cells and register python-generated recipes together * Call `build()` to produce a normal library and report for downstream `Pather` usage and writing * Explore alternate ways of specifying a pattern for `.plug()` and `.place()` diff --git a/examples/tutorial/library.py b/examples/tutorial/library.py index d7a2615..da582fe 100644 --- a/examples/tutorial/library.py +++ b/examples/tutorial/library.py @@ -1,5 +1,5 @@ """ -Tutorial: authoring a mixed library with `BuildLibrary`. +Tutorial: authoring a mixed library with `LibraryBuilder`. This example assumes you have already read `devices.py` and generated the `circuit.gds` file it writes. The goal here is not the photonic-crystal geometry @@ -11,7 +11,7 @@ from typing import Any from pprint import pformat -from masque import BuildLibrary, Pather, Pattern, cell +from masque import ILibrary, LibraryBuilder, Pather, Pattern, cell from masque.file.gdsii import writefile from masque.file.gdsii.lazy import readfile @@ -20,7 +20,7 @@ import devices from basic_shapes import GDS_OPTS -def make_mixed_waveguide(lib: BuildLibrary) -> Pattern: +def make_mixed_waveguide(lib: ILibrary) -> Pattern: """ Recipe which assembles imported and generated cells behind the builder API. """ @@ -42,7 +42,7 @@ def make_mixed_waveguide(lib: BuildLibrary) -> Pattern: def main() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() cells = builder.cells # @@ -72,8 +72,12 @@ def main() -> None: cells.tri_wg28 = cell(devices.waveguide)(length=28, mirror_periods=5, **opts) cells.tri_bend0 = cell(devices.bend)(mirror_periods=5, **opts) cells.tri_ysplit = cell(devices.y_splitter)(mirror_periods=5, **opts) - cells.tri_l3cav = cell(devices.perturbed_l3)(xy_size=(4, 10), **opts, hole_lib=builder) - cells.mixed_wg_cav = cell(make_mixed_waveguide)(builder) + cells.tri_l3cav = cell(devices.perturbed_l3)( + xy_size=(4, 10), + **opts, + hole_lib=builder.library, + ) + cells.mixed_wg_cav = cell(make_mixed_waveguide)(builder.library) print('Declared cells waiting to be built:\n' + pformat(list(builder.keys()))) @@ -110,6 +114,10 @@ def main() -> None: print('Writing library to file...') writefile(built, 'library.gds', **GDS_OPTS) + # The default build output is an overlay which borrows its lazy sources. + # Close the owning GDS source only after the overlay is no longer needed. + gds_lib.close() + if __name__ == '__main__': main() diff --git a/masque/__init__.py b/masque/__init__.py index defd1ef..89b45b0 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -59,13 +59,16 @@ from .pattern import ( from .utils.boolean import boolean as boolean from .library import ( + INameView as INameView, ILibraryView as ILibraryView, ILibrary as ILibrary, + IBorrowing as IBorrowing, + IMaterializable as IMaterializable, LibraryView as LibraryView, Library as Library, OverlayLibrary as OverlayLibrary, PortsLibraryView as PortsLibraryView, - BuildLibrary as BuildLibrary, + LibraryBuilder as LibraryBuilder, BuildReport as BuildReport, CellProvenance as CellProvenance, LazyLibrary as LazyLibrary, diff --git a/masque/file/gdsii/lazy.py b/masque/file/gdsii/lazy.py index 1439306..75092ad 100644 --- a/masque/file/gdsii/lazy.py +++ b/masque/file/gdsii/lazy.py @@ -26,7 +26,6 @@ import mmap import pathlib import klamath -import numpy from klamath import records from . import klamath as gdsii @@ -35,15 +34,16 @@ from ..utils import is_gzipped from ...error import LibraryError from ...library import ( ILibraryView, - LibraryView, + IMaterializable, PortsLibraryView, dangling_mode_t, ) -from ...utils import apply_transforms +from ...library.utils import _validate_dangling_mode if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence + import numpy from numpy.typing import NDArray from ...pattern import Pattern @@ -122,7 +122,7 @@ def _scan_library( return library_info, order, cells -class GdsLibrarySource(ILibraryView): +class GdsLibrarySource(ILibraryView, IMaterializable): """ Read-only library backed by a seekable GDS stream. @@ -163,7 +163,7 @@ class GdsLibrarySource(ILibraryView): return cls(source=source, library_info=library_info, cell_order=cell_order, cells=cells) def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) + return self.materialize(key, persist=True) def __iter__(self) -> Iterator[str]: return iter(self._cell_order) @@ -177,19 +177,7 @@ class GdsLibrarySource(ILibraryView): def source_order(self) -> tuple[str, ...]: return self._cell_order - def materialize_many( - self, - names: Sequence[str], - *, - persist: bool = True, - ) -> LibraryView: - mats = { - name: self._materialize_pattern(name, persist=persist) - for name in dict.fromkeys(names) - } - return LibraryView(mats) - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + def materialize(self, name: str, *, persist: bool = True) -> Pattern: if name in self._cache: return self._cache[name] @@ -223,6 +211,7 @@ class GdsLibrarySource(ILibraryView): self, dangling: dangling_mode_t = 'error', ) -> dict[str, set[str]]: + _validate_dangling_mode(dangling) graph: dict[str, set[str]] = {} for name in self._cell_order: if name in self._cache: @@ -243,41 +232,11 @@ class GdsLibrarySource(ILibraryView): graph.setdefault(cast('str', child), set()) return graph - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - def subtree( self, tops: str | Sequence[str], ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - return self.materialize_many(tuple(keep), persist=True) - - def tops(self) -> list[str]: - graph = self.child_graph(dangling='ignore') - names = set(graph) - not_toplevel: set[str] = set() - for children in graph.values(): - not_toplevel |= children - return list(names - not_toplevel) + return super().subtree(tops) def with_ports_from_data( self, @@ -315,6 +274,7 @@ class GdsLibrarySource(ILibraryView): parent_graph: dict[str, set[str]] | None = None, dangling: dangling_mode_t = 'error', ) -> dict[str, list[NDArray[numpy.float64]]]: + _validate_dangling_mode(dangling) instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) if parent_graph is None: graph_mode = 'ignore' if dangling == 'ignore' else 'include' @@ -333,58 +293,11 @@ class GdsLibrarySource(ILibraryView): for ref in self._cache[parent].refs.get(name, []): instances[parent].append(ref.as_transforms()) continue - pat = self._materialize_pattern(parent, persist=False) + pat = self.materialize(parent, persist=False) for ref in pat.refs.get(name, []): instances[parent].append(ref.as_transforms()) return instances - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - result[(parent,) + path] = instances - return result - def close(self) -> None: self._source.close() diff --git a/masque/file/gdsii/lazy_arrow.py b/masque/file/gdsii/lazy_arrow.py index acfd454..98b741f 100644 --- a/masque/file/gdsii/lazy_arrow.py +++ b/masque/file/gdsii/lazy_arrow.py @@ -21,11 +21,12 @@ from .lazy_write import write as write, writefile as writefile from ..utils import is_gzipped from ...library import ( ILibraryView, + IMaterializable, LibraryView, PortsLibraryView, dangling_mode_t, ) -from ...utils import apply_transforms +from ...library.utils import _validate_dangling_mode if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence @@ -197,7 +198,7 @@ def _expand_aref_row( return rows -class ArrowLibrary(ILibraryView): +class ArrowLibrary(ILibraryView, IMaterializable): """ Read-only library backed by the native lazy Arrow scan schema. @@ -231,7 +232,7 @@ class ArrowLibrary(ILibraryView): return cls(path=path, payload=payload, source=source) def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) + return self.materialize(key, persist=True) def __iter__(self) -> Iterator[str]: return iter(self._payload.cell_order) @@ -299,7 +300,7 @@ class ArrowLibrary(ILibraryView): materialized[name] = self._cache[name] return materialized - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + def materialize(self, name: str, *, persist: bool = True) -> Pattern: return self._materialize_patterns((name,), persist=persist)[name] def _raw_children(self, name: str) -> set[str]: @@ -346,6 +347,7 @@ class ArrowLibrary(ILibraryView): self, dangling: dangling_mode_t = 'error', ) -> dict[str, set[str]]: + _validate_dangling_mode(dangling) graph: dict[str, set[str]] = {} for name in self._payload.cell_order: if name in self._cache: @@ -366,41 +368,11 @@ class ArrowLibrary(ILibraryView): graph.setdefault(cast('str', child), set()) return graph - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - def subtree( self, tops: str | Sequence[str], ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - return self.materialize_many(tuple(keep), persist=True) - - def tops(self) -> list[str]: - graph = self.child_graph(dangling='ignore') - names = set(graph) - not_toplevel: set[str] = set() - for children in graph.values(): - not_toplevel |= children - return list(names - not_toplevel) + return super().subtree(tops) def with_ports_from_data( self, @@ -452,6 +424,7 @@ class ArrowLibrary(ILibraryView): parent_graph: dict[str, set[str]] | None = None, dangling: dangling_mode_t = 'error', ) -> dict[str, list[NDArray[numpy.float64]]]: + _validate_dangling_mode(dangling) instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) if parent_graph is None: graph_mode = 'ignore' if dangling == 'ignore' else 'include' @@ -479,54 +452,6 @@ class ArrowLibrary(ILibraryView): instances[parent].extend(rows) return instances - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - full_path = (parent,) + path - result[full_path] = instances - return result - def readfile( filename: str | pathlib.Path, diff --git a/masque/file/gdsii/lazy_write.py b/masque/file/gdsii/lazy_write.py index 225787a..a9e40bd 100644 --- a/masque/file/gdsii/lazy_write.py +++ b/masque/file/gdsii/lazy_write.py @@ -8,7 +8,7 @@ or remapped. """ from __future__ import annotations -from typing import IO, TYPE_CHECKING, Any, cast +from typing import IO, TYPE_CHECKING, Any, Protocol, cast, runtime_checkable import gzip import logging import pathlib @@ -18,8 +18,8 @@ import klamath from . import klamath as gdsii from ..utils import tmpfile from ...error import LibraryError -from ...library import ILibraryView, OverlayLibrary -from ...library.overlay import _SourceEntry, _materialize_detached_pattern +from ...library import IBorrowing, ILibraryView +from ...library.overlay import _materialize_detached_pattern if TYPE_CHECKING: from collections.abc import Mapping @@ -30,6 +30,23 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +@runtime_checkable +class _GdsInfoSource(Protocol): + """Structural capability for propagating GDS header metadata.""" + library_info: dict[str, Any] + + +@runtime_checkable +class _GdsRawCellSource(Protocol): + """GDS-specific raw-structure copy-through capability.""" + + def source_order(self) -> tuple[str, ...]: ... + + def can_copy_raw_struct(self, name: str) -> bool: ... + + def raw_struct_bytes(self, name: str) -> bytes: ... + + def _get_write_info( library: Mapping[str, Pattern] | ILibraryView, *, @@ -42,13 +59,16 @@ def _get_write_info( infos: list[dict[str, Any]] = [] stack: list[Mapping[str, Pattern] | ILibraryView] = [library] + seen: set[int] = set() while stack: current = stack.pop() - info = getattr(current, 'library_info', None) - if isinstance(info, dict): - infos.append(info) - if isinstance(current, OverlayLibrary): - stack.extend(reversed([layer.library for layer in current._layers])) + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, _GdsInfoSource) and isinstance(current.library_info, dict): + infos.append(current.library_info) + if isinstance(current, IBorrowing): + stack.extend(reversed(current.borrowed_sources())) if infos: unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} @@ -65,20 +85,6 @@ def _get_write_info( return meters_per_unit, logical_units_per_unit, library_name -def _can_copy_raw_cell(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bool: - can_copy = getattr(library, 'can_copy_raw_struct', None) - if not callable(can_copy): - return False - return bool(can_copy(name)) - - -def _raw_struct_bytes(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bytes: - reader = getattr(library, 'raw_struct_bytes', None) - if not callable(reader): - raise TypeError('raw_struct_bytes') - return cast('bytes', reader(name)) - - def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: elements: list[klamath.elements.Element] = [] elements += gdsii._shapes_to_elements(pat.shapes) @@ -109,28 +115,10 @@ def write( ) header.write(stream) - if isinstance(library, OverlayLibrary): + if isinstance(library, _GdsRawCellSource): for name in library.source_order(): - entry = library._entries[name] - can_copy_overlay = False - if isinstance(entry, _SourceEntry) and name == entry.source_name: - layer = library._layers[entry.layer_index] - children = layer.child_graph.get(entry.source_name, set()) - can_copy_overlay = ( - _can_copy_raw_cell(layer.library, entry.source_name) - and all(library._effective_target(layer, child) == child for child in children) - ) - if can_copy_overlay: - stream.write(_raw_struct_bytes(layer.library, entry.source_name)) - else: - _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) - klamath.records.ENDLIB.write(stream, None) - return - - if hasattr(library, 'raw_struct_bytes'): - for name in library.source_order(): - if _can_copy_raw_cell(library, name): - stream.write(_raw_struct_bytes(library, name)) + if library.can_copy_raw_struct(name): + stream.write(library.raw_struct_bytes(name)) else: _write_pattern_struct(stream, name, _materialize_detached_pattern(cast('ILibraryView', library), name)) klamath.records.ENDLIB.write(stream, None) diff --git a/masque/library/__init__.py b/masque/library/__init__.py index b05b020..5e206a3 100644 --- a/masque/library/__init__.py +++ b/masque/library/__init__.py @@ -1,5 +1,6 @@ """Library classes for managing name-to-pattern mappings.""" from .utils import ( + INameView as INameView, SINGLE_USE_PREFIX as SINGLE_USE_PREFIX, Tree as Tree, TreeView as TreeView, @@ -12,6 +13,10 @@ from .base import ( ILibrary as ILibrary, ILibraryView as ILibraryView, ) +from .capabilities import ( + IBorrowing as IBorrowing, + IMaterializable as IMaterializable, +) from .mapping import ( Library as Library, LibraryView as LibraryView, @@ -21,7 +26,7 @@ from .overlay import ( PortsLibraryView as PortsLibraryView, ) from .build import ( - BuildLibrary as BuildLibrary, + LibraryBuilder as LibraryBuilder, BuildReport as BuildReport, CellProvenance as CellProvenance, cell as cell, diff --git a/masque/library/base.py b/masque/library/base.py index 283ca6d..7e8be6c 100644 --- a/masque/library/base.py +++ b/masque/library/base.py @@ -8,8 +8,6 @@ from graphlib import CycleError, TopologicalSorter from pprint import pformat from typing import TYPE_CHECKING, Self, cast import copy -import logging -import re import numpy @@ -18,7 +16,18 @@ from ..error import LibraryError, PatternError from ..pattern import Pattern, map_layers from ..shapes import Polygon, Shape from ..utils import apply_transforms, layer_t -from .utils import SINGLE_USE_PREFIX, TreeView, b64suffix, dangling_mode_t, _rename_patterns, visitor_function_t +from .utils import ( + SINGLE_USE_PREFIX, + INameView, + TreeView, + b64suffix, + dangling_mode_t, + _plan_source_names, + _rename_patterns, + _source_rename_map, + _validate_dangling_mode, + visitor_function_t, +) if TYPE_CHECKING: from collections.abc import Iterator @@ -28,10 +37,7 @@ if TYPE_CHECKING: from ..label import Label -logger = logging.getLogger(__name__) - - -class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): +class ILibraryView(Mapping[str, 'Pattern'], INameView, metaclass=ABCMeta): """ Interface for a read-only library. @@ -78,7 +84,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): def dangling_refs( self, tops: str | Sequence[str] | None = None, - ) -> set[str | None]: + ) -> set[str]: """ Get the set of all pattern names not present in the library but referenced by `tops`, recursively traversing any refs. @@ -101,8 +107,8 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): def referenced_patterns( self, tops: str | Sequence[str] | None = None, - skip: set[str | None] | None = None, - ) -> set[str | None]: + skip: set[str] | None = None, + ) -> set[str]: """ Get the set of all pattern names referenced by `tops`. Recursively traverses into any refs. @@ -116,29 +122,78 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: Set of all referenced pattern names """ + graph = self.child_graph(dangling='include') + return self._referenced_patterns_from_graph(graph, tops=tops, skip=skip) + + def _referenced_patterns_from_graph( + self, + graph: Mapping[str, set[str]], + *, + tops: str | Sequence[str] | None, + skip: set[str] | None = None, + ) -> set[str]: + """Traverse named references in an already-computed child graph.""" + existing = set(self.keys()) if tops is None: - tops = tuple(self.keys()) + roots = tuple(existing) + elif isinstance(tops, str): + roots = (tops,) + else: + roots = tuple(tops) + + for root in roots: + if root not in existing: + raise KeyError(root) if skip is None: - skip = {None} + skip = set() + root_set = set(roots) + skip |= root_set - if isinstance(tops, str): - tops = (tops,) - tops = set(tops) - skip |= tops # don't re-visit tops + targets: set[str] = set() + stack = list(root_set) + while stack: + name = stack.pop() + children = graph.get(name, set()) + targets |= children + for child in children - skip: + skip.add(child) + if child in existing: + stack.append(child) + return targets - # Get referenced patterns for all tops - targets = set() - for top in set(tops): - targets |= self[top].referenced_patterns() + def _referenced_patterns_by_lookup( + self, + tops: str | Sequence[str] | None = None, + skip: set[str] | None = None, + ) -> set[str]: + """Traverse refs by loading only reachable patterns.""" + if tops is None: + roots = tuple(self.keys()) + elif isinstance(tops, str): + roots = (tops,) + else: + roots = tuple(tops) - # Perform recursive lookups, but only once for each name - for target in targets - skip: - assert target is not None - skip.add(target) - if target in self: - targets |= self.referenced_patterns(target, skip=skip) + if skip is None: + skip = set() + root_set = set(roots) + skip |= root_set + targets: set[str] = set() + stack = list(root_set) + while stack: + name = stack.pop() + children = { + target + for target in self[name].referenced_patterns() + if target is not None + } + targets |= children + for child in children - skip: + skip.add(child) + if child in self: + stack.append(child) return targets def subtree( @@ -154,19 +209,22 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): tops: Name(s) of patterns to keep Returns: - A `LibraryView` containing only `tops` and the patterns they reference. + A borrowed view containing only `tops` and the patterns they reference. + + The returned view snapshots membership and hierarchy, but borrows patterns + and source capabilities from this library. Keep this library open and do + not structurally mutate it for the lifetime of the returned view. """ if isinstance(tops, str): tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + graph = self.child_graph(dangling='include') + keep = self._referenced_patterns_from_graph(graph, tops=tops) + keep &= set(self.keys()) keep |= set(tops) - from .mapping import LibraryView # noqa: PLC0415 - - filtered = {kk: vv for kk, vv in self.items() if kk in keep} - new = LibraryView(filtered) - return new + from .mapping import _SubtreeLibraryView # noqa: PLC0415 + return _SubtreeLibraryView(self, names=keep, child_graph=graph) def polygonize( self, @@ -242,6 +300,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): def flatten_single(name: str) -> None: flattened[name] = None pat = self[name].deepcopy() + pat.prune_refs() refs_by_target = tuple((target, tuple(refs)) for target, refs in pat.refs.items()) for target, refs in refs_by_target: @@ -281,59 +340,6 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): assert None not in flattened.values() return cast('dict[str, Pattern]', flattened) - def get_name( - self, - name: str = SINGLE_USE_PREFIX * 2, - sanitize: bool = True, - max_length: int = 32, - quiet: bool | None = None, - ) -> str: - """ - Find a unique name for the pattern. - - This function may be overridden in a subclass or monkey-patched to fit the caller's requirements. - - Args: - name: Preferred name for the pattern. Default is `SINGLE_USE_PREFIX * 2`. - sanitize: Allows only alphanumeric charaters and _?$. Replaces invalid characters with underscores. - max_length: Names longer than this will be truncated. - quiet: If `True`, suppress log messages. Default `None` suppresses messages only if - the name starts with `SINGLE_USE_PREFIX`. - - Returns: - Name, unique within this library. - """ - if quiet is None: - quiet = name.startswith(SINGLE_USE_PREFIX) - - if sanitize: - # Remove invalid characters - sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name) - else: - sanitized_name = name - - suffixed_name = sanitized_name - if sanitized_name in self: - ii = sum(1 for nn in self.keys() if nn.startswith(sanitized_name)) - else: - ii = 0 - while suffixed_name in self or suffixed_name == '': - suffixed_name = sanitized_name + b64suffix(ii) - ii += 1 - - if len(suffixed_name) > max_length: - if name == '': - raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}') - - cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True) - else: - cropped_name = suffixed_name - - if not quiet: - logger.info(f'Requested name "{name}" changed to "{cropped_name}"') - - return cropped_name - def tops(self) -> list[str]: """ Return the list of all patterns that are not referenced by any other pattern in the library. @@ -341,13 +347,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: A list of pattern names in which no pattern is referenced by any other pattern. """ + graph = self.child_graph(dangling='ignore') names = set(self.keys()) - not_toplevel: set[str | None] = set() - for name in names: - not_toplevel |= set(self[name].refs.keys()) - - toplevel = list(names - not_toplevel) - return toplevel + referenced = set().union(*graph.values()) if graph else set() + return list(names - referenced) def top(self) -> str: """ @@ -412,7 +415,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): tuple of all parent-and-higher pattern names. Top pattern name may be `None` if not provided in first call to .dfs() `transform`: numpy.ndarray containing cumulative - [x_offset, y_offset, rotation (rad), mirror_x (0 or 1)] + [x_offset, y_offset, rotation (rad), mirror_x (0 or 1), scale] for the instance being visited `memo`: Arbitrary dict (not altered except by `visit_before()` and `visit_after()`) @@ -450,13 +453,13 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): if visit_before is not None: pattern = visit_before(pattern, hierarchy=hierarchy, memo=memo, transform=transform) - for target in pattern.refs: - if target is None: + for target, refs in pattern.refs.items(): + if target is None or not refs: continue if target in hierarchy: raise LibraryError(f'.dfs() called on pattern with circular reference to "{target}"') - for ref in pattern.refs[target]: + for ref in refs: ref_transforms: list[bool] | NDArray[numpy.float64] if transform is not False: ref_transforms = apply_transforms(transform, ref.as_transforms()) @@ -509,6 +512,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: Mapping from pattern name to a set of all pattern names it references. """ + _validate_dangling_mode(dangling) graph, dangling_refs = self._raw_child_graph() if dangling == 'error': if dangling_refs: @@ -538,13 +542,18 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: Mapping from pattern name to a set of all patterns which reference it. """ - child_graph, dangling_refs = self._raw_child_graph() + _validate_dangling_mode(dangling) + graph_mode: dangling_mode_t = 'ignore' if dangling == 'ignore' else 'include' + child_graph = self.child_graph(dangling=graph_mode) + existing = set(self.keys()) + dangling_refs = set(child_graph) - existing if dangling == 'error' and dangling_refs: raise self._dangling_refs_error(dangling_refs, 'building parent graph') - existing = set(child_graph) - igraph: dict[str, set[str]] = {name: set() for name in existing} - for parent, children in child_graph.items(): + graph_names = set(child_graph) if dangling == 'include' else existing + igraph: dict[str, set[str]] = {name: set() for name in graph_names} + for parent in existing: + children = child_graph.get(parent, set()) for child in children: if child in existing: igraph[child].add(parent) @@ -566,6 +575,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Return: Topologically sorted list of pattern names. """ + _validate_dangling_mode(dangling) try: return cast('list[str]', list(TopologicalSorter(self.child_graph(dangling=dangling)).static_order())) except CycleError as exc: @@ -596,9 +606,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: Mapping of {parent_name: transform_list}, where transform_list - is an Nx4 ndarray with rows - `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. + is an Nx5 ndarray with rows + `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x, scale)`. """ + _validate_dangling_mode(dangling) instances = defaultdict(list) if parent_graph is None: graph_mode = 'ignore' if dangling == 'ignore' else 'include' @@ -648,9 +659,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Returns: Mapping of `{hierarchy: transform_list}`, where `hierarchy` is a tuple of the form - `(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx4 ndarray - with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. + `(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx5 ndarray + with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x, scale)`. """ + _validate_dangling_mode(dangling) graph_mode = 'ignore' if dangling == 'ignore' else 'include' if order is None: order = self.child_order(dangling=graph_mode) @@ -700,11 +712,15 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): -class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): +class ILibrary(ILibraryView, metaclass=ABCMeta): """ - Interface for a writeable library. + Interface for an insertable and deletable library. A library is a mapping from unique names (str) to collections of geometry (`Pattern`). + Assignment inserts new names but does not replace existing ones. `ILibrary` + intentionally does not implement `MutableMapping` or its generic mutation + helpers; use the library-specific insertion, deletion, rename, and add + operations instead. """ # inherited abstract functions #def __getitem__(self, key: str) -> 'Pattern': @@ -853,27 +869,35 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): def add( self, other: Mapping[str, Pattern], - rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + rename_theirs: Callable[[INameView, str], str] = _rename_patterns, mutate_other: bool = False, ) -> dict[str, str]: """ Add items from another library into this one. - If any name in `other` is already present in `self`, `rename_theirs(self, name)` is called - to pick a new name for the newly-added pattern. If the new name still conflicts with a name - in `self` a `LibraryError` is raised. All references to the original name (within `other)` - are updated to the new name. + If any name in `other` is already present in the prospective destination, + `rename_theirs(view, name)` is called to pick a new name for the newly-added pattern. + The name-only `INameView` includes destination names and names reserved earlier in + the same addition plan. + If the new name still conflicts, a `LibraryError` is raised. All references to the + original name within `other` are updated to the new name. If `mutate_other=False` (default), all changes are made to a deepcopy of `other`. + Name resolution, copying, and reference remapping complete before cells are inserted, + so failures during those phases do not partially modify either library. Custom + implementations of `_merge()` may still have partial effects if they raise while cells + are being committed. + By default, `rename_theirs` makes no changes to the name (causing a `LibraryError`) unless the name starts with `SINGLE_USE_PREFIX`. Prefixed names are truncated to before their first non-prefix '$' and then passed to `self.get_name()` to create a new unique name. Args: other: The library to insert keys from. - rename_theirs: Called as rename_theirs(self, name) for each duplicate name - encountered in `other`. Should return the new name for the pattern in - `other`. See above for default behavior. + rename_theirs: Called as `rename_theirs(view, name)` for each duplicate name + encountered in `other`, where `view` supports membership, iteration, + `len()`, and `get_name()`, but not pattern lookup. Should return the new + name for the pattern in `other`. See above for default behavior. mutate_other: If `True`, modify the original library and its contained patterns (e.g. when renaming patterns and updating refs). Otherwise, operate on a deepcopy (default). @@ -888,42 +912,34 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): from ..pattern import map_targets # noqa: PLC0415 from .mapping import Library # noqa: PLC0415 - duplicates = set(self.keys()) & set(other.keys()) - - if not duplicates: - if mutate_other: - temp = other - else: - temp = Library(copy.deepcopy(dict(other))) - - for key in temp: - self._merge(key, temp, key) - return {} + source_order = tuple(other.keys()) + source_to_visible = _plan_source_names( + self, + source_order, + rename_theirs = rename_theirs, + rename_when = 'conflict', + ) + rename_map = _source_rename_map(source_to_visible) if mutate_other: if isinstance(other, Library): temp = other else: - temp = Library(dict(other)) + temp = Library({name: other[name] for name in source_order}) else: - temp = Library(copy.deepcopy(dict(other))) - rename_map = {} - for old_name in temp: - if old_name in self: - new_name = rename_theirs(self, old_name) - if new_name in self: - raise LibraryError(f'Unresolved duplicate key encountered in library merge: {old_name} -> {new_name}') - rename_map[old_name] = new_name - else: - new_name = old_name + temp = Library(copy.deepcopy({name: other[name] for name in source_order})) - self._merge(new_name, temp, old_name) + if rename_map: + target_map = cast('dict[str | None, str | None]', rename_map) + remapped_refs = { + source_name: map_targets(temp[source_name].refs, lambda target: target_map.get(target, target)) + for source_name in source_order + } + for source_name, refs in remapped_refs.items(): + temp[source_name].refs = refs - # Update references in the newly-added cells - for old_name in temp: - new_name = rename_map.get(old_name, old_name) - pat = self[new_name] - pat.refs = map_targets(pat.refs, lambda tt: cast('dict[str | None, str | None]', rename_map).get(tt, tt)) + for source_name in source_order: + self._merge(source_to_visible[source_name], temp, source_name) return rename_map @@ -934,21 +950,18 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): based on `add()`'s default `rename_theirs` argument). Raises: - LibraryError if there is more than one topcell in `other`. + LibraryError if there is not exactly one topcell in `other`. """ from .mapping import LibraryView # noqa: PLC0415 - if len(other) == 1: - name = next(iter(other)) - else: - if not isinstance(other, ILibraryView): - other = LibraryView(other) + if not isinstance(other, ILibraryView): + other = LibraryView(other) - tops = other.tops() - if len(tops) > 1: - raise LibraryError('Received a library containing multiple topcells!') + tops = other.tops() + if len(tops) != 1: + raise LibraryError(f'Received a library without exactly one topcell: {pformat(tops)}') - name = tops[0] + name = tops[0] rename_map = self.add(other) new_name = rename_map.get(name, name) @@ -960,7 +973,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): of just the pattern's name. Raises: - LibraryError if there is more than one topcell in `other`. + LibraryError if there is not exactly one topcell in `other`. """ new_name = self << other return self.abstract(new_name) @@ -1178,7 +1191,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): if isinstance(tops, str): tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep = self.referenced_patterns(tops) keep |= set(tops) new = type(self)() @@ -1201,6 +1214,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): Returns: A set containing the names of all deleted patterns """ + _validate_dangling_mode(dangling) parent_graph = self.parent_graph(dangling=dangling) empty = {name for name, pat in self.items() if pat.is_empty()} trimmed = set() @@ -1257,3 +1271,6 @@ class AbstractView(Mapping[str, Abstract]): def __len__(self) -> int: return self.library.__len__() + + def __contains__(self, key: object) -> bool: + return key in self.library diff --git a/masque/library/build.py b/masque/library/build.py index 8582a52..c4b7f8f 100644 --- a/masque/library/build.py +++ b/masque/library/build.py @@ -2,32 +2,25 @@ from __future__ import annotations from collections import defaultdict -from contextvars import ContextVar from dataclasses import dataclass, replace from functools import wraps -from pprint import pformat +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, Self, cast import copy from ..error import BuildError, LibraryError from .base import ILibrary, ILibraryView -from .utils import TreeView, dangling_mode_t, _plan_source_names, _rename_patterns, _source_rename_map +from .capabilities import IMaterializable +from .utils import INameView, TreeView, dangling_mode_t, _plan_source_names, _rename_patterns, _source_rename_map from .mapping import Library, LibraryView from .overlay import OverlayLibrary if TYPE_CHECKING: - from collections.abc import Callable, Iterator, Mapping, Sequence + from collections.abc import Callable, Iterator, KeysView, Mapping, Sequence - from ..abstract import Abstract from ..pattern import Pattern -_ACTIVE_BUILD_SESSIONS: ContextVar[dict[int, _BuildSessionLibrary] | None] = ContextVar( - 'masque_active_build_sessions', - default=None, -) - - @dataclass(frozen=True) class CellProvenance: """ @@ -55,7 +48,7 @@ class CellProvenance: @dataclass(frozen=True) class BuildReport: """ - Immutable summary of one `BuildLibrary.validate()` or `.build()` run. + Immutable summary of one `LibraryBuilder.validate()` or `.build()` run. The report is designed to answer two questions after a build completes: which declared cells depended on which other declared cells, and where each @@ -72,6 +65,15 @@ class BuildReport: provenance: Mapping[str, CellProvenance] dependency_graph: Mapping[str, frozenset[str]] + def __post_init__(self) -> None: + object.__setattr__(self, 'requested_roots', tuple(self.requested_roots)) + object.__setattr__(self, 'provenance', MappingProxyType(dict(self.provenance))) + object.__setattr__( + self, + 'dependency_graph', + MappingProxyType({name: frozenset(deps) for name, deps in self.dependency_graph.items()}), + ) + @dataclass class _BuildRecipe: """ Captured deferred call to a pattern factory. """ @@ -98,42 +100,54 @@ def cell(func: Callable[..., Pattern]) -> Callable[..., _BuildRecipe]: return wrapper +class _LibraryPlaceholder: + """Identity token replaced by this builder's active build-session library.""" + __slots__ = ('_builder',) + + def __init__(self, builder: LibraryBuilder) -> None: + self._builder = builder + + def __repr__(self) -> str: + return '' + + class BuildCellsView: """ - Attribute-based declaration namespace for `BuildLibrary`. + Attribute-based declaration namespace for `LibraryBuilder`. This is the ergonomic authoring surface exposed as `builder.cells`. It is intentionally write-focused: attribute assignment and deletion register declarations, while attribute reads fail with guidance to build first and use the returned library. """ + __slots__ = ('_library',) - def __init__(self, library: BuildLibrary) -> None: + def __init__(self, library: LibraryBuilder) -> None: object.__setattr__(self, '_library', library) def __getattr__(self, name: str) -> Pattern: raise BuildError( - f'BuildLibrary.cells.{name} is write-only during authoring. ' + f'LibraryBuilder.cells.{name} is write-only during authoring. ' 'Call build() and index the returned library instead.' ) def __setattr__(self, name: str, value: Pattern | _BuildRecipe) -> None: - if name.startswith('_'): + if name == '_library': object.__setattr__(self, name, value) return self._library[name] = value def __delattr__(self, name: str) -> None: - if name.startswith('_'): + if name == '_library': raise AttributeError(name) del self._library[name] -class BuildLibrary(ILibrary): +class LibraryBuilder(INameView): """ Two-phase declaration surface for mixed imported/generated libraries. - A `BuildLibrary` collects three kinds of inputs: + A `LibraryBuilder` collects three kinds of inputs: - direct declared `Pattern` objects - deferred recipes created with `cell(...)` - imported source-backed library views added with `add_source(...)` @@ -147,112 +161,82 @@ class BuildLibrary(ILibrary): def __init__(self) -> None: self.cells = BuildCellsView(self) + self._library_placeholder = _LibraryPlaceholder(self) self._frozen = False + self._building = False self._declarations: dict[str, Pattern | _BuildRecipe] = {} self._sources: list[tuple[ILibraryView, dict[str, str]]] = [] self._names: dict[str, None] = {} - def _active_session(self) -> _BuildSessionLibrary | None: - sessions = _ACTIVE_BUILD_SESSIONS.get() - if sessions is None: - return None - return sessions.get(id(self)) - - def _require_active_session(self, operation: str) -> _BuildSessionLibrary: - session = self._active_session() - if session is None: - raise BuildError( - f'BuildLibrary.{operation}() is only available while validate() or build() is running. ' - 'Use the built output library for reads.' - ) - return session + @property + def library(self) -> _LibraryPlaceholder: + """Read-only placeholder replaced by the active library in direct recipe arguments.""" + return self._library_placeholder def _assert_editable(self) -> None: if self._frozen: - raise BuildError('This BuildLibrary has already been built successfully and is now frozen.') + raise BuildError('This LibraryBuilder has already been built successfully and is now frozen.') + if self._building: + raise BuildError('Cannot modify a LibraryBuilder while validate() or build() is running.') def __iter__(self) -> Iterator[str]: - session = self._active_session() - if session is not None: - return iter(session) return iter(self._names) def __len__(self) -> int: - session = self._active_session() - if session is not None: - return len(session) return len(self._names) def __contains__(self, key: object) -> bool: - session = self._active_session() - if session is not None: - return key in session return key in self._names - def __getitem__(self, key: str) -> Pattern: - return self._require_active_session('__getitem__')[key] + def keys(self) -> KeysView[str]: + return self._names.keys() def __setitem__( self, key: str, value: Pattern | _BuildRecipe, ) -> None: - session = self._active_session() - if session is not None: - session[key] = value - return - self._assert_editable() if key in self._names: raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!') if isinstance(value, _BuildRecipe): + placeholders = ( + arg for arg in (*value.args, *value.kwargs.values()) + if isinstance(arg, _LibraryPlaceholder) + ) + if any(placeholder is not self.library for placeholder in placeholders): + raise BuildError('A recipe cannot use another LibraryBuilder.library placeholder.') declaration = value else: if callable(value): - raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.') + raise TypeError('LibraryBuilder recipes must be wrapped with cell(fn)(...) or @cell.') declaration = value self._declarations[key] = declaration self._names[key] = None def __delitem__(self, key: str) -> None: - session = self._active_session() - if session is not None: - del session[key] - return - self._assert_editable() if key not in self._declarations: raise KeyError(key) del self._declarations[key] del self._names[key] - def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: - session = self._active_session() - if session is not None: - session._merge(key_self, other, key_other) - return - self[key_self] = copy.deepcopy(other[key_other]) - def add( self, other: Mapping[str, Pattern], - rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + rename_theirs: Callable[[INameView, str], str] = _rename_patterns, mutate_other: bool = False, ) -> dict[str, str]: from ..pattern import map_targets # noqa: PLC0415 - session = self._active_session() - if session is not None: - return session.add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) - self._assert_editable() - source_backed = isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView) - if source_backed: + materializable = isinstance(other, IMaterializable) + if materializable: if mutate_other: - raise BuildError('BuildLibrary.add(..., mutate_other=True) is not supported for source-backed inputs.') + raise BuildError('LibraryBuilder.add(..., mutate_other=True) is not supported for source-backed inputs.') return self.add_source( other, rename_theirs = rename_theirs, @@ -263,7 +247,6 @@ class BuildLibrary(ILibrary): source_to_visible = _plan_source_names( self, source_order, - self._names, rename_theirs = rename_theirs, rename_when = 'conflict', ) @@ -272,95 +255,39 @@ class BuildLibrary(ILibrary): if mutate_other: temp = other else: - temp = Library(copy.deepcopy(dict(other))) + temp = Library(copy.deepcopy({name: other[name] for name in source_order})) + + if rename_map: + target_map = cast('dict[str | None, str | None]', rename_map) + remapped_refs = { + source_name: map_targets(temp[source_name].refs, lambda target: target_map.get(target, target)) + for source_name in source_order + } + for source_name, refs in remapped_refs.items(): + temp[source_name].refs = refs for source_name in source_order: visible_name = source_to_visible[source_name] pattern = temp[source_name] - if rename_map: - pattern.refs = map_targets( - pattern.refs, - lambda target: cast('dict[str | None, str | None]', rename_map).get(target, target), - ) self[visible_name] = pattern return rename_map def __lshift__(self, other: TreeView) -> str: - session = self._active_session() - if session is not None: - return session << other - self._assert_editable() - if len(other) == 1: - name = next(iter(other)) - elif isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView): - source_order = other.source_order() - child_graph = other.child_graph(dangling='include') - referenced = set().union(*child_graph.values()) if child_graph else set() - tops = [candidate for candidate in source_order if candidate not in referenced] - if len(tops) != 1: - raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}') - name = tops[0] - else: - return super().__lshift__(other) - - rename_map = self.add(other) - return rename_map.get(name, name) - - def __le__(self, other: Mapping[str, Pattern]) -> Abstract: - if self._active_session() is not None: - return super().__le__(other) - raise BuildError('BuildLibrary.__le__() is only available while validate() or build() is running.') - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> Self: - """ - Rename a helper cell during an active build session. - - During authoring, declared cells must be registered under their - intended final names and imported source cells must be renamed through - `add_source(...)`. - """ - session = self._active_session() - if session is not None: - session.rename(old_name, new_name, move_references=move_references) - return self - - self._assert_editable() - if old_name == new_name: - return self - if old_name in self._declarations: - raise BuildError( - f'Cannot rename declared build cell "{old_name}" during authoring. ' - 'Register it under the intended final name instead.' - ) - if old_name not in self._names: - raise LibraryError(f'"{old_name}" does not exist in the builder.') - raise BuildError( - f'Cannot rename imported source cell "{old_name}" during authoring. ' - 'Choose visible source names with add_source(..., rename_theirs=..., rename_when=...).' - ) - - def abstract(self, name: str) -> Abstract: - return self._require_active_session('abstract').abstract(name) - - def resolve( - self, - other: Abstract | str | Pattern | TreeView, - append: bool = False, - ) -> Abstract | Pattern: - return self._require_active_session('resolve').resolve(other, append=append) + view = other if isinstance(other, ILibraryView) else LibraryView(other) + tops = view.tops() + if len(tops) != 1: + raise LibraryError(f'Received a library without exactly one topcell: {tops}') + top = tops[0] + rename_map = self.add(view) + return rename_map.get(top, top) def add_source( self, source: Mapping[str, Pattern] | ILibraryView, *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_theirs: Callable[[INameView, str], str] | None = None, rename_when: Literal['conflict', 'always'] = 'conflict', ) -> dict[str, str]: """ @@ -369,13 +296,19 @@ class BuildLibrary(ILibrary): The source is not materialized immediately. Its names are scanned once to reserve visible builder names, then the source is read again when a build session starts. The source's cell membership must not be - structurally mutated between `add_source()` and `build()`/`validate()`. + closed or structurally mutated between `add_source()` and + `build()`/`validate()`. Source cells may be renamed on entry to avoid collisions with existing declarations or other imported sources. + Sources are borrowed rather than owned. An output built with + `output='overlay'` remains source-backed, so its sources must also stay + open and unchanged for the lifetime of that output. + Args: rename_theirs: Function used to choose visible names for imported - source cells. + source cells. Its `INameView` argument contains existing and + previously reserved names, but does not support pattern lookup. rename_when: If `'conflict'`, only conflicting names are renamed. If `'always'`, every imported source name is passed through `rename_theirs`. @@ -384,8 +317,6 @@ class BuildLibrary(ILibrary): Mapping of `{source_name: visible_name}` for imported names that were renamed while being added. """ - if self._active_session() is not None: - raise BuildError('BuildLibrary.add_source() is only available while authoring, not during validate() or build().') self._assert_editable() view = source if isinstance(source, ILibraryView) else LibraryView(source) @@ -393,7 +324,6 @@ class BuildLibrary(ILibrary): source_to_visible = _plan_source_names( self, source_order, - self._names, rename_theirs = rename_theirs, rename_when = rename_when, ) @@ -406,7 +336,7 @@ class BuildLibrary(ILibrary): def validate( self, - names: Sequence[str] | None = None, + names: str | Sequence[str] | None = None, *, allow_dangling: bool = False, ) -> BuildReport: @@ -416,6 +346,13 @@ class BuildLibrary(ILibrary): This is a dry run over the same dependency resolution and recipe execution path used by `build()`. Any generated library is discarded after validation completes. + + Args: + names: Declared root name or names to validate. `None` validates + every declaration. Duplicate names are ignored after their + first occurrence. + allow_dangling: If `False`, fail validation when the generated + library contains dangling references. """ _session, report = self._run_build(names=names, allow_dangling=allow_dangling) return report @@ -431,13 +368,19 @@ class BuildLibrary(ILibrary): Args: output: `'overlay'` preserves imported source-backed cells where - possible, while `'library'` eagerly materializes the full - result. + possible and continues borrowing their sources. `'library'` + eagerly materializes the full result and no longer needs the + sources afterward. allow_dangling: If `False`, fail the build when the completed library still contains dangling references. """ if output not in ('overlay', 'library'): raise ValueError(f'Unknown build output mode: {output!r}') + if self._building: + raise BuildError( + 'Cannot call build() or validate() recursively on a LibraryBuilder ' + 'from one of its own recipes.' + ) self._assert_editable() session, report = self._run_build(names=None, allow_dangling=allow_dangling) if output == 'library': @@ -450,24 +393,37 @@ class BuildLibrary(ILibrary): def _run_build( self, *, - names: Sequence[str] | None, + names: str | Sequence[str] | None, allow_dangling: bool, ) -> tuple[_BuildSessionLibrary, BuildReport]: - roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys())) + if self._building: + raise BuildError( + 'Cannot call build() or validate() recursively on a LibraryBuilder ' + 'from one of its own recipes.' + ) + + if names is None: + requested_names = tuple(self._declarations) + elif isinstance(names, str): + requested_names = (names,) + else: + requested_names = tuple(names) + if any(not isinstance(name, str) for name in requested_names): + raise TypeError('Build roots must be strings.') + + roots = tuple(dict.fromkeys(requested_names)) unknown = [name for name in roots if name not in self._names] if unknown: raise BuildError(f'Unknown build roots requested: {unknown}') - session = _BuildSessionLibrary(self) - sessions = dict(_ACTIVE_BUILD_SESSIONS.get() or {}) - sessions[id(self)] = session - token = _ACTIVE_BUILD_SESSIONS.set(sessions) + self._building = True try: + session = _BuildSessionLibrary(self) session.materialize_many(roots) if not allow_dangling: session.child_graph(dangling='error') finally: - _ACTIVE_BUILD_SESSIONS.reset(token) + self._building = False report = session.build_report(roots) return session, report @@ -475,7 +431,7 @@ class BuildLibrary(ILibrary): class _BuildSessionLibrary(ILibrary): """ - Internal overlay-backed library used while a `BuildLibrary` is executing. + Internal overlay-backed library used while a `LibraryBuilder` is executing. This object provides the mutable-library surface that recipes expect while also tracking declared-cell dependencies, helper-cell provenance, and @@ -483,7 +439,7 @@ class _BuildSessionLibrary(ILibrary): build run. """ - def __init__(self, builder: BuildLibrary) -> None: + def __init__(self, builder: LibraryBuilder) -> None: self._builder = builder self._overlay = OverlayLibrary() self._built: set[str] = set() @@ -512,7 +468,7 @@ class _BuildSessionLibrary(ILibrary): 'Do not structurally mutate source libraries between add_source() and build()/validate().' ) - def rename_source(_lib: ILibraryView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str: + def rename_source(_lib: INameView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str: return mapping[name] self._overlay.add_source( @@ -586,6 +542,8 @@ class _BuildSessionLibrary(ILibrary): return self def __getitem__(self, key: str) -> Pattern: + if key not in self._names: + raise KeyError(key) if key in self._builder._declarations: self._record_dependency(key) self._ensure_declared(key) @@ -637,7 +595,7 @@ class _BuildSessionLibrary(ILibrary): def add( self, other: Mapping[str, Pattern], - rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + rename_theirs: Callable[[INameView, str], str] = _rename_patterns, mutate_other: bool = False, ) -> dict[str, str]: rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) @@ -684,7 +642,15 @@ class _BuildSessionLibrary(ILibrary): if isinstance(declaration, _BuildRecipe): for dep in declaration.explicit_dependencies: self._ensure_named(dep) - pattern = declaration.func(*declaration.args, **declaration.kwargs) + args = tuple( + self if arg is self._builder.library else arg + for arg in declaration.args + ) + kwargs = { + key: self if value is self._builder.library else value + for key, value in declaration.kwargs.items() + } + pattern = declaration.func(*args, **kwargs) if not isinstance(pattern, Pattern): raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') # noqa: TRY301 else: @@ -722,6 +688,42 @@ class _BuildSessionLibrary(ILibrary): ) -> dict[str, set[str]]: return self._overlay.parent_graph(dangling=dangling) + def subtree( + self, + tops: str | Sequence[str], + ) -> Self: + if isinstance(tops, str): + tops = (tops,) + + keep = self._referenced_patterns_by_lookup(tops=tops) + keep &= set(self) + keep |= set(tops) + order = tuple(name for name in self._names if name in keep) + + patterns = {name: self[name] for name in order} + new = object.__new__(type(self)) + new._builder = self._builder + new._overlay = OverlayLibrary() + for name in order: + new._overlay[name] = patterns[name] + new._built = { + name for name in order + if name in self._builder._declarations + } + new._declared_stack = [] + new._names = dict.fromkeys(order) + new._provenance = { + name: self._provenance[name] + for name in order + if name in self._provenance + } + new._dependency_graph = defaultdict(set, { + name: set(dependencies) & keep + for name, dependencies in self._dependency_graph.items() + if name in keep + }) + return new + def build_report(self, requested_roots: Sequence[str]) -> BuildReport: dependency_graph = { name: frozenset(self._dependency_graph.get(name, set())) diff --git a/masque/library/capabilities.py b/masque/library/capabilities.py new file mode 100644 index 0000000..257393f --- /dev/null +++ b/masque/library/capabilities.py @@ -0,0 +1,42 @@ +"""Optional capabilities implemented by lazy and borrowing libraries.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + + from ..pattern import Pattern + from .base import ILibraryView + from .mapping import LibraryView + + +class IMaterializable(ABC): + """Capability for libraries which support explicit pattern materialization.""" + + @abstractmethod + def materialize(self, name: str, *, persist: bool = True) -> Pattern: + """Materialize one pattern, optionally retaining it in the library's cache.""" + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + """Materialize a de-duplicated sequence into a plain read-only view.""" + from .mapping import LibraryView # noqa: PLC0415 + + return LibraryView({ + name: self.materialize(name, persist=persist) + for name in dict.fromkeys(names) + }) + + +class IBorrowing(ABC): + """Capability for library views which directly borrow other libraries.""" + + @abstractmethod + def borrowed_sources(self) -> tuple[ILibraryView, ...]: + """Return the source views directly borrowed by this library.""" diff --git a/masque/library/lazy.py b/masque/library/lazy.py index 973a431..dcb26ed 100644 --- a/masque/library/lazy.py +++ b/masque/library/lazy.py @@ -7,9 +7,10 @@ import logging from ..error import LibraryError from .base import ILibrary +from .capabilities import IMaterializable if TYPE_CHECKING: - from collections.abc import Callable, Iterator, Mapping + from collections.abc import Callable, Iterator, Mapping, Sequence from ..pattern import Pattern @@ -17,7 +18,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class LazyLibrary(ILibrary): +class LazyLibrary(ILibrary, IMaterializable): """ This class is usually used to create a library of Patterns by mapping names to functions which generate or load the relevant `Pattern` object as-needed. @@ -56,6 +57,9 @@ class LazyLibrary(ILibrary): del self.cache[key] def __getitem__(self, key: str) -> Pattern: + return self.materialize(key, persist=True) + + def materialize(self, key: str, *, persist: bool = True) -> Pattern: logger.debug(f'loading {key}') if key in self.cache: logger.debug(f'found {key} in cache') @@ -76,7 +80,8 @@ class LazyLibrary(ILibrary): pat = func() finally: self._lookups_in_progress.pop() - self.cache[key] = pat + if persist: + self.cache[key] = pat return pat def __iter__(self) -> Iterator[str]: @@ -88,6 +93,15 @@ class LazyLibrary(ILibrary): def __contains__(self, key: object) -> bool: return key in self.mapping + def referenced_patterns( + self, + tops: str | Sequence[str] | None = None, + skip: set[str] | None = None, + ) -> set[str]: + # Closure-backed cells do not have hierarchy metadata. Preserve laziness + # by loading only patterns reached from the requested roots. + return self._referenced_patterns_by_lookup(tops=tops, skip=skip) + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: if isinstance(other, LazyLibrary): self.mapping[key_self] = other.mapping[key_other] diff --git a/masque/library/mapping.py b/masque/library/mapping.py index 95c7e73..8fb10dd 100644 --- a/masque/library/mapping.py +++ b/masque/library/mapping.py @@ -2,14 +2,19 @@ from __future__ import annotations from pprint import pformat -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING, Any, Self, cast from ..error import LibraryError from .base import ILibrary, ILibraryView +from .capabilities import IBorrowing, IMaterializable +from .utils import dangling_mode_t, _validate_dangling_mode if TYPE_CHECKING: from collections.abc import Callable, Iterator, Mapping, MutableMapping + import numpy + from numpy.typing import NDArray + from ..pattern import Pattern @@ -44,6 +49,106 @@ class LibraryView(ILibraryView): return f'' +class _SubtreeLibraryView(ILibraryView, IMaterializable, IBorrowing): + """Borrowed subtree view with snapshotted membership and hierarchy.""" + + def __init__( + self, + source: ILibraryView, + *, + names: set[str], + child_graph: Mapping[str, set[str]], + ) -> None: + self._source = source + source_order = source.source_order() + ordered = list(dict.fromkeys(name for name in source_order if name in names)) + seen = set(ordered) + ordered.extend(name for name in source if name in names and name not in seen) + self._order = tuple(ordered) + self._names = frozenset(self._order) + self._child_graph = { + name: set(child_graph.get(name, set())) + for name in self._order + } + if hasattr(source, 'library_info'): + self.library_info = cast('dict[str, Any]', source.library_info) + + def __getitem__(self, key: str) -> Pattern: + if key not in self._names: + raise KeyError(key) + return self._source[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._order) + + def __len__(self) -> int: + return len(self._order) + + def __contains__(self, key: object) -> bool: + return key in self._names + + def borrowed_sources(self) -> tuple[ILibraryView, ...]: + return (self._source,) + + def source_order(self) -> tuple[str, ...]: + return self._order + + def materialize(self, name: str, *, persist: bool = True) -> Pattern: + if name not in self._names: + raise KeyError(name) + if isinstance(self._source, IMaterializable): + return self._source.materialize(name, persist=persist) + return self._source[name] + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + _validate_dangling_mode(dangling) + graph = {name: set(children) for name, children in self._child_graph.items()} + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) if graph else set() + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(dangling_refs, 'building child graph') + return graph + if dangling == 'ignore': + return { + name: {child for child in children if child in existing} + for name, children in graph.items() + } + for target in dangling_refs: + graph.setdefault(target, set()) + return graph + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + _validate_dangling_mode(dangling) + if parent_graph is None: + graph_mode: dangling_mode_t = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + refs = self._source.find_refs_local(name, parent_graph=parent_graph, dangling=dangling) + return {parent: transforms for parent, transforms in refs.items() if parent in self._names} + + def raw_struct_bytes(self, name: str) -> bytes: + if name not in self._names: + raise KeyError(name) + reader = getattr(self._source, 'raw_struct_bytes', None) + if not callable(reader): + raise TypeError('raw_struct_bytes') + return cast('bytes', reader(name)) + + def can_copy_raw_struct(self, name: str) -> bool: + if name not in self._names: + return False + can_copy = getattr(self._source, 'can_copy_raw_struct', None) + return bool(callable(can_copy) and can_copy(name)) + + class Library(ILibrary): """ Default implementation for a writeable library. diff --git a/masque/library/overlay.py b/masque/library/overlay.py index a325bc6..519393e 100644 --- a/masque/library/overlay.py +++ b/masque/library/overlay.py @@ -3,34 +3,32 @@ from __future__ import annotations from collections import defaultdict from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, Self, cast import copy -import numpy - from ..error import LibraryError from ..pattern import Pattern, map_targets -from ..utils import apply_transforms, layer_t from .base import ILibrary, ILibraryView -from .utils import dangling_mode_t, _plan_source_names, _source_rename_map +from .capabilities import IBorrowing, IMaterializable +from .utils import INameView, dangling_mode_t, _plan_source_names, _source_rename_map, _validate_dangling_mode from .mapping import LibraryView if TYPE_CHECKING: from collections.abc import Callable, Iterator, Mapping, Sequence + import numpy from numpy.typing import NDArray from ..ports import Port + from ..utils import layer_t @dataclass class _SourceLayer: """ One imported source layer tracked by an `OverlayLibrary`. """ library: ILibraryView - source_to_visible: dict[str, str] - visible_to_source: dict[str, str] + source_target_map: dict[str, str] child_graph: dict[str, set[str]] - order: list[str] @dataclass(frozen=True) @@ -41,18 +39,19 @@ class _SourceEntry: def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: - func = getattr(view, '_materialize_pattern', None) - if callable(func): - return cast('Pattern', func(name, persist=False)) + if isinstance(view, IMaterializable): + return view.materialize(name, persist=False).deepcopy() return view[name].deepcopy() -class PortsLibraryView(ILibraryView): +class PortsLibraryView(ILibraryView, IMaterializable, IBorrowing): """ Read-only view which imports or applies ports on first materialization. The wrapped source remains untouched; this view owns a separate processed cache so direct-copy workflows can continue to use the raw source view. + The view borrows its source: callers must keep the source open for the + lifetime of the view and close the source themselves. Graph queries, source ordering, and copy-through capabilities are delegated to the wrapped source whenever possible, while `__getitem__` and @@ -84,7 +83,7 @@ class PortsLibraryView(ILibraryView): self.library_info = cast('dict[str, Any]', source.library_info) def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) + return self.materialize(key, persist=True) def __iter__(self) -> Iterator[str]: return iter(self._source) @@ -95,7 +94,7 @@ class PortsLibraryView(ILibraryView): def __contains__(self, key: object) -> bool: return key in self._source - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + def materialize(self, name: str, *, persist: bool = True) -> Pattern: from ..utils.ports2data import data_to_ports # noqa: PLC0415 if name in self._cache: @@ -134,72 +133,31 @@ class PortsLibraryView(ILibraryView): self._cache[name] = pat return pat - def materialize_many( - self, - names: Sequence[str], - *, - persist: bool = True, - ) -> LibraryView: - mats = { - name: self._materialize_pattern(name, persist=persist) - for name in dict.fromkeys(names) - } - return LibraryView(mats) - def source_order(self) -> tuple[str, ...]: return self._source.source_order() + def borrowed_sources(self) -> tuple[ILibraryView, ...]: + return (self._source,) + def child_graph( self, dangling: dangling_mode_t = 'error', ) -> dict[str, set[str]]: + _validate_dangling_mode(dangling) return self._source.child_graph(dangling=dangling) - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._source.parent_graph(dangling=dangling) - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) - keep |= set(tops) - return self.materialize_many(tuple(keep), persist=True) - - def tops(self) -> list[str]: - return self._source.tops() - def find_refs_local( self, name: str, parent_graph: dict[str, set[str]] | None = None, dangling: dangling_mode_t = 'error', ) -> dict[str, list[NDArray[numpy.float64]]]: + _validate_dangling_mode(dangling) finder = getattr(self._source, 'find_refs_local', None) if callable(finder): return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - finder = getattr(self._source, 'find_refs_global', None) - if callable(finder): - return cast( - 'dict[tuple[str, ...], NDArray[numpy.float64]]', - finder(name, order=order, parent_graph=parent_graph, dangling=dangling), - ) - return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) - def raw_struct_bytes(self, name: str) -> bytes: reader = getattr(self._source, 'raw_struct_bytes', None) if not callable(reader): @@ -207,30 +165,26 @@ class PortsLibraryView(ILibraryView): return cast('bytes', reader(name)) def can_copy_raw_struct(self, name: str) -> bool: + if name in self._cache: + return False can_copy = getattr(self._source, 'can_copy_raw_struct', None) if not callable(can_copy): return False return bool(can_copy(name)) - def close(self) -> None: - closer = getattr(self._source, 'close', None) - if callable(closer): - closer() - def __enter__(self) -> PortsLibraryView: - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - -class OverlayLibrary(ILibrary): +class OverlayLibrary(ILibrary, IMaterializable, IBorrowing): """ Mutable overlay over one or more source libraries. Source-backed cells remain lazy until accessed through `__getitem__`, at which point that visible cell is promoted into an overlay-owned materialized `Pattern`. + + Source libraries must remain open and must not be mutated after they are + added. The overlay borrows each source and snapshots its names, hierarchy, + and initial visible-name mapping while retaining the source itself for lazy + pattern materialization. """ def __init__(self) -> None: @@ -249,7 +203,7 @@ class OverlayLibrary(ILibrary): return key in self._entries def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) + return self.materialize(key, persist=True) def __setitem__( self, @@ -275,15 +229,19 @@ class OverlayLibrary(ILibrary): self, source: Mapping[str, Pattern] | ILibraryView, *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_theirs: Callable[[INameView, str], str] | None = None, rename_when: Literal['conflict', 'always'] = 'conflict', ) -> dict[str, str]: """ Add a source-backed library layer. + The source must remain open, and its names, hierarchy, and pattern + contents must remain unchanged for the lifetime of this overlay. + Args: rename_theirs: Function used to choose visible names for imported - source cells. + source cells. Its `INameView` argument contains existing and + previously reserved names, but does not support pattern lookup. rename_when: If `'conflict'`, only conflicting names are renamed. If `'always'`, every imported source name is passed through `rename_theirs`. @@ -295,18 +253,13 @@ class OverlayLibrary(ILibrary): source_to_visible = _plan_source_names( self, source_order, - self._entries, rename_theirs = rename_theirs, rename_when = rename_when, ) - visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()} - layer = _SourceLayer( library=view, - source_to_visible=source_to_visible, - visible_to_source=visible_to_source, + source_target_map=dict(source_to_visible), child_graph=child_graph, - order=[source_to_visible[name] for name in source_order], ) layer_index = len(self._layers) self._layers.append(layer) @@ -333,11 +286,6 @@ class OverlayLibrary(ILibrary): entry = self._entries.pop(old_name) self._entries[new_name] = entry - if isinstance(entry, _SourceEntry): - layer = self._layers[entry.layer_index] - layer.source_to_visible[entry.source_name] = new_name - del layer.visible_to_source[old_name] - layer.visible_to_source[new_name] = entry.source_name idx = self._order.index(old_name) self._order[idx] = new_name @@ -375,10 +323,10 @@ class OverlayLibrary(ILibrary): return self def _effective_target(self, layer: _SourceLayer, target: str) -> str: - visible = layer.source_to_visible.get(target, target) + visible = layer.source_target_map.get(target, target) return self._resolve_target(visible) - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + def materialize(self, name: str, *, persist: bool = True) -> Pattern: if name not in self._entries: raise KeyError(name) entry = self._entries[name] @@ -386,7 +334,7 @@ class OverlayLibrary(ILibrary): return entry layer = self._layers[entry.layer_index] - source_pat = layer.library[entry.source_name].deepcopy() + source_pat = _materialize_detached_pattern(layer.library, entry.source_name) def remap(target: str | None) -> str | None: return None if target is None else self._effective_target(layer, target) @@ -402,6 +350,7 @@ class OverlayLibrary(ILibrary): self, dangling: dangling_mode_t = 'error', ) -> dict[str, set[str]]: + _validate_dangling_mode(dangling) graph: dict[str, set[str]] = {} for name in self._order: if name not in self._entries: @@ -427,33 +376,31 @@ class OverlayLibrary(ILibrary): graph.setdefault(cast('str', child), set()) return graph - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - def subtree( self, tops: str | Sequence[str], - ) -> ILibraryView: + ) -> Self: if isinstance(tops, str): tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + + graph = self.child_graph(dangling='include') + keep = self._referenced_patterns_from_graph(graph, tops=tops) + keep &= set(self) keep |= set(tops) - return LibraryView({name: self[name] for name in keep}) + + new = type(self)() + new._layers = [ + _SourceLayer( + library=layer.library, + source_target_map=dict(layer.source_target_map), + child_graph={name: set(children) for name, children in layer.child_graph.items()}, + ) + for layer in self._layers + ] + new._order = [name for name in self._order if name in keep and name in self._entries] + new._entries = {name: self._entries[name] for name in new._order} + new._target_remap = dict(self._target_remap) + return new def find_refs_local( self, @@ -461,6 +408,7 @@ class OverlayLibrary(ILibrary): parent_graph: dict[str, set[str]] | None = None, dangling: dangling_mode_t = 'error', ) -> dict[str, list[NDArray[numpy.float64]]]: + _validate_dangling_mode(dangling) instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) if parent_graph is None: graph_mode = 'ignore' if dangling == 'ignore' else 'include' @@ -475,57 +423,34 @@ class OverlayLibrary(ILibrary): return instances for parent in parent_graph.get(name, set()): - pat = self._materialize_pattern(parent, persist=False) + pat = self.materialize(parent, persist=False) for ref in pat.refs.get(name, []): instances[parent].append(ref.as_transforms()) return instances - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - result[(parent,) + path] = instances - return result - def source_order(self) -> tuple[str, ...]: return tuple(name for name in self._order if name in self._entries) + + def borrowed_sources(self) -> tuple[ILibraryView, ...]: + return tuple(layer.library for layer in self._layers) + + def can_copy_raw_struct(self, name: str) -> bool: + entry = self._entries.get(name) + if not isinstance(entry, _SourceEntry) or name != entry.source_name: + return False + layer = self._layers[entry.layer_index] + can_copy = getattr(layer.library, 'can_copy_raw_struct', None) + if not callable(can_copy) or not can_copy(entry.source_name): + return False + children = layer.child_graph.get(entry.source_name, set()) + return all(self._effective_target(layer, child) == child for child in children) + + def raw_struct_bytes(self, name: str) -> bytes: + entry = self._entries.get(name) + if not isinstance(entry, _SourceEntry): + raise TypeError('raw_struct_bytes') + layer = self._layers[entry.layer_index] + reader = getattr(layer.library, 'raw_struct_bytes', None) + if not callable(reader): + raise TypeError('raw_struct_bytes') + return cast('bytes', reader(entry.source_name)) diff --git a/masque/library/utils.py b/masque/library/utils.py index 138c91a..ffdb9ce 100644 --- a/masque/library/utils.py +++ b/masque/library/utils.py @@ -1,8 +1,11 @@ """Shared types and helpers for library implementations.""" from __future__ import annotations +from abc import ABC from typing import TYPE_CHECKING, Literal, Protocol, TypeAlias -from collections.abc import Callable, Container, Mapping, MutableMapping, Sequence +from collections.abc import Callable, Collection, Iterator, Mapping, MutableMapping, Sequence +import logging +import re from ..error import LibraryError @@ -11,7 +14,80 @@ if TYPE_CHECKING: from numpy.typing import NDArray from ..pattern import Pattern - from .base import ILibraryView + + +logger = logging.getLogger(__name__) + +SINGLE_USE_PREFIX = '_' +""" +Names starting with this prefix are assumed to refer to single-use patterns, +which may be renamed automatically by `ILibrary.add()` (via +`rename_theirs=_rename_patterns()` ) +""" +# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere? + + +class INameView(Collection[str], ABC): + """ + Read-only collection of reserved names with a shared name allocator. + + Name views support membership, iteration, `len()`, and `get_name()`. They + do not provide pattern lookup or the other operations of a library mapping. + """ + + def get_name( + self, + name: str = SINGLE_USE_PREFIX * 2, + sanitize: bool = True, + max_length: int = 32, + quiet: bool | None = None, + ) -> str: + """ + Find a unique name. + + This function may be overridden in a subclass or monkey-patched to fit + the caller's requirements. + + Args: + name: Preferred name. Default is `SINGLE_USE_PREFIX * 2`. + sanitize: Allow only alphanumeric characters and _?$, replacing + invalid characters with underscores. + max_length: Truncate names longer than this. + quiet: Suppress log messages when `True`. The default suppresses + messages only when `name` starts with `SINGLE_USE_PREFIX`. + + Returns: + A name unique within this view. + """ + if quiet is None: + quiet = name.startswith(SINGLE_USE_PREFIX) + + if sanitize: + sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name) + else: + sanitized_name = name + + suffixed_name = sanitized_name + if sanitized_name in self: + ii = sum(1 for nn in self if nn.startswith(sanitized_name)) + else: + ii = 0 + while suffixed_name in self or suffixed_name == '': + suffixed_name = sanitized_name + b64suffix(ii) + ii += 1 + + if len(suffixed_name) > max_length: + if name == '': + raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}') + + cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True) + else: + cropped_name = suffixed_name + + if not quiet: + logger.info(f'Requested name "{name}" changed to "{cropped_name}"') + + return cropped_name class visitor_function_t(Protocol): @@ -34,16 +110,9 @@ Tree: TypeAlias = MutableMapping[str, 'Pattern'] dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include'] """ How helpers should handle refs whose targets are not present in the library. """ -SINGLE_USE_PREFIX = '_' -""" -Names starting with this prefix are assumed to refer to single-use patterns, -which may be renamed automatically by `ILibrary.add()` (via -`rename_theirs=_rename_patterns()` ) -""" -# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere? -def _rename_patterns(lib: ILibraryView, name: str) -> str: +def _rename_patterns(lib: INameView, name: str) -> str: """ The default `rename_theirs` function for `ILibrary.add`. @@ -67,12 +136,41 @@ def _rename_patterns(lib: ILibraryView, name: str) -> str: return lib.get_name(SINGLE_USE_PREFIX + stem) +def _validate_dangling_mode(dangling: dangling_mode_t) -> None: + if dangling not in ('error', 'ignore', 'include'): + raise ValueError( + f'Unknown dangling-reference mode {dangling!r}; ' + 'expected one of "error", "ignore", or "include"' + ) + + +class _ProspectiveNames(INameView): + """Target names plus names reserved earlier in an addition plan.""" + + def __init__( + self, + target: INameView, + reserved: set[str], + ) -> None: + self._target = target + self._reserved = reserved + + def __iter__(self) -> Iterator[str]: + yield from self._target + yield from self._reserved + + def __len__(self) -> int: + return len(self._target) + len(self._reserved) + + def __contains__(self, key: object) -> bool: + return key in self._reserved or key in self._target + + def _plan_source_names( - target: ILibraryView, + target: INameView, source_order: Sequence[str], - existing_names: Container[str], *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_theirs: Callable[[INameView, str], str] | None = None, rename_when: Literal['conflict', 'always'] = 'conflict', ) -> dict[str, str]: if rename_when not in ('conflict', 'always'): @@ -81,21 +179,22 @@ def _plan_source_names( raise TypeError('rename_theirs is required when rename_when="always"') source_to_visible: dict[str, str] = {} - visible_names: set[str] = set() + reserved: set[str] = set() + prospective = _ProspectiveNames(target, reserved) for name in source_order: visible = name if rename_when == 'always': assert rename_theirs is not None - visible = rename_theirs(target, name) - elif visible in existing_names or visible in visible_names: + visible = rename_theirs(prospective, name) + elif visible in prospective: if rename_theirs is None: raise LibraryError(f'Conflicting name while adding source: {name!r}') - visible = rename_theirs(target, name) - if visible in existing_names or visible in visible_names: + visible = rename_theirs(prospective, name) + if visible in prospective: raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') source_to_visible[name] = visible - visible_names.add(visible) + reserved.add(visible) return source_to_visible diff --git a/masque/pattern.py b/masque/pattern.py index daf78e8..a630552 100644 --- a/masque/pattern.py +++ b/masque/pattern.py @@ -30,6 +30,41 @@ from .ports import Port, PortList logger = logging.getLogger(__name__) +def _check_ref_cycles( + pattern: 'Pattern', + library: Mapping[str, 'Pattern'], + *, + operation: str, + ) -> None: + visited: set[str] = set() + active: list[str] = [] + active_set: set[str] = set() + + def visit(current: 'Pattern') -> None: + for target, refs in current.refs.items(): + if target is None or not refs: + continue + if target in active_set: + cycle_start = active.index(target) + cycle = active[cycle_start:] + [target] + raise PatternError( + f'Circular reference while {operation}: {" -> ".join(cycle)}' + ) + if target in visited: + continue + + active.append(target) + active_set.add(target) + try: + visit(library[target]) + finally: + active.pop() + active_set.remove(target) + visited.add(target) + + visit(pattern) + + @functools.total_ordering class Pattern(PortList, AnnotatableImpl, Mirrorable): """ @@ -509,6 +544,8 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): layer: layer_t, flatten: bool = True, library: Mapping[str, 'Pattern'] | None = None, + *, + _cycle_checked: bool = False, ) -> list[Polygon]: """ Collect all geometry effectively on a given layer as a list of polygons. @@ -524,9 +561,15 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): Returns: A list of `Polygon` objects. + + Raises: + PatternError: If `flatten=True` and the referenced hierarchy contains a cycle. """ if flatten and self.has_refs() and library is None: raise PatternError("Must provide a library to layer_as_polygons() when flatten=True") + if flatten and self.has_refs() and not _cycle_checked: + assert library is not None + _check_ref_cycles(self, library, operation='collecting layer polygons') polys: list[Polygon] = [] @@ -543,12 +586,17 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): if flatten and self.has_refs(): assert library is not None for target, refs in self.refs.items(): - if target is None: + if target is None or not refs: continue target_pat = library[target] for ref in refs: # Get polygons from target pattern on the same layer - ref_polys = target_pat.layer_as_polygons(layer, flatten=True, library=library) + ref_polys = target_pat.layer_as_polygons( + layer, + flatten=True, + library=library, + _cycle_checked=True, + ) # Apply ref transformations for p in ref_polys: p_pat = ref.as_pattern(Pattern(shapes={layer: [p]})) @@ -566,13 +614,15 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): Returns: A set of all pattern names referenced by this pattern. """ - return set(self.refs.keys()) + return {target for target, refs in self.refs.items() if refs} def get_bounds( self, library: Mapping[str, 'Pattern'] | None = None, recurse: bool = True, cache: MutableMapping[str, NDArray[numpy.float64] | None] | None = None, + *, + _cycle_checked: bool = False, ) -> NDArray[numpy.float64] | None: """ Return a `numpy.ndarray` containing `[[x_min, y_min], [x_max, y_max]]`, corresponding to the @@ -589,7 +639,13 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): Returns: `[[x_min, y_min], [x_max, y_max]]` or `None` + + Raises: + PatternError: If recursive bounds are requested for a cyclic hierarchy. """ + if recurse and self.has_refs() and library is not None and not _cycle_checked: + _check_ref_cycles(self, library, operation='calculating bounds') + if self.is_empty(): return None @@ -628,7 +684,12 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): if target in cache: unrot_bounds = cache[target] elif any(numpy.isclose(ref.rotation % (pi / 2), 0) for ref in refs): - unrot_bounds = library[target].get_bounds(library=library, recurse=recurse, cache=cache) + unrot_bounds = library[target].get_bounds( + library=library, + recurse=recurse, + cache=cache, + _cycle_checked=True, + ) cache[target] = unrot_bounds for ref in refs: @@ -1127,8 +1188,17 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): overdraw: Whether to create a new figure or draw on a pre-existing one. filename: If provided, save the figure to this file instead of showing it. ports: If True, annotate the plot with arrows representing the ports. + + Raises: + PatternError: If the referenced hierarchy contains a cycle. """ # TODO: add text labels to visualize() + if self.has_refs() and library is None: + raise PatternError('Must provide a library when visualizing a pattern with refs') + if self.has_refs(): + assert library is not None + _check_ref_cycles(self, library, operation='visualizing pattern hierarchy') + try: from matplotlib import pyplot # type: ignore #noqa: PLC0415 import matplotlib.collections # type: ignore #noqa: PLC0415 @@ -1137,9 +1207,6 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): + 'Make sure to install masque with the [visualize] option to pull in the needed dependencies.') raise - if self.has_refs() and library is None: - raise PatternError('Must provide a library when visualizing a pattern with refs') - # Cache for {Pattern object ID: List of local polygon vertex arrays} # Polygons are stored relative to the pattern's origin (offset included) poly_cache: dict[int, list[NDArray[numpy.float64]]] = {} @@ -1203,7 +1270,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): # 3. Recurse into refs for target, refs in pat.refs.items(): - if target is None: + if target is None or not refs: continue assert library is not None target_pat = library[target] diff --git a/masque/test/test_autotool_planning.py b/masque/test/test_autotool_planning.py index 50df07f..a311985 100644 --- a/masque/test/test_autotool_planning.py +++ b/masque/test/test_autotool_planning.py @@ -842,7 +842,8 @@ def test_autotool_s_offer_uses_absolute_jog_range_for_both_signs() -> None: assert isinstance(pather._paths["A"][0].data, AutoTool.GeneratedData) -def test_autotool_sbend_registration_order_sets_priority() -> None: +@pytest.mark.parametrize('reverse', [False, True]) +def test_autotool_sbend_cost_is_independent_of_registration_order(reverse: bool) -> None: def first_sbend(jog: float) -> Pattern: pat = Pattern() pat.ports["A"] = Port((0, 0), 0, ptype="core") @@ -855,19 +856,116 @@ def test_autotool_sbend_registration_order_sets_priority() -> None: pat.ports["B"] = Port((5, jog), pi, ptype="core") return pat + tool = AutoTool() + generators = (second_sbend, first_sbend) if reverse else (first_sbend, second_sbend) + for generator in generators: + tool.add_sbend(generator, "core", "A", "B", jog_range=(0, 1e8)) + + _offer, out_port, data = selected_offer(tool, "s", 4, in_ptype="core") + + assert isinstance(data, AutoTool.GeneratedData) + assert data.fn is second_sbend + assert_allclose(out_port.offset, [5, 4]) + + +def test_autotool_sbend_explicit_cost_can_override_geometric_cost() -> None: + def long_sbend(jog: float) -> Pattern: + pat = Pattern() + pat.ports["A"] = Port((0, 0), 0, ptype="core") + pat.ports["B"] = Port((20, jog), pi, ptype="core") + return pat + + def short_sbend(jog: float) -> Pattern: + pat = Pattern() + pat.ports["A"] = Port((0, 0), 0, ptype="core") + pat.ports["B"] = Port((5, jog), pi, ptype="core") + return pat + tool = ( AutoTool() - .add_sbend(first_sbend, "core", "A", "B", jog_range=(0, 1e8)) - .add_sbend(second_sbend, "core", "A", "B", jog_range=(0, 1e8)) + .add_sbend(short_sbend, "core", "A", "B", jog_range=(0, 1e8), cost=10) + .add_sbend(long_sbend, "core", "A", "B", jog_range=(0, 1e8), cost=1) ) _offer, out_port, data = selected_offer(tool, "s", 4, in_ptype="core") assert isinstance(data, AutoTool.GeneratedData) - assert data.fn is first_sbend + assert data.fn is long_sbend assert_allclose(out_port.offset, [20, 4]) +def test_autotool_add_methods_propagate_callable_cost_to_all_created_offers() -> None: + def cost(parameter: float, endpoint: Port) -> float: + return abs(parameter) + abs(endpoint.x) + abs(endpoint.y) + + def make_straight(length: float) -> Pattern: + return _make_transition_straight(length, ptype="core") + + def make_sbend(jog: float) -> Pattern: + pat = Pattern() + pat.ports["A"] = Port((0, 0), 0, ptype="core") + pat.ports["B"] = Port((5, jog), pi, ptype="core") + return pat + + lib = Library() + + bend = Pattern() + bend.ports["A"] = Port((0, 0), 0, ptype="core") + bend.ports["B"] = Port((2, -2), pi / 2, ptype="core") + lib["bend"] = bend + + uturn = Pattern() + uturn.ports["A"] = Port((0, 0), 0, ptype="core") + uturn.ports["B"] = Port((3, 4), 0, ptype="core") + lib["uturn"] = uturn + + transition = Pattern() + transition.ports["EXT"] = Port((0, 0), 0, ptype="external") + transition.ports["CORE"] = Port((1, 0), pi, ptype="core") + lib["transition"] = transition + + tool = ( + AutoTool() + .add_straight(make_straight, "core", "in", cost=cost) + .add_bend(lib.abstract("bend"), "A", "B", clockwise=True, cost=cost) + .add_sbend( + make_sbend, + "core", + "A", + "B", + endpoint=lambda jog: Port((5, jog), pi, ptype="core"), + cost=cost, + ) + .add_uturn(lib.abstract("uturn"), "A", "B", cost=cost) + .add_transition(lib.abstract("transition"), "EXT", "CORE", cost=cost) + ) + + offers = [ + *(offer for offer in tool.primitive_offers("straight", in_ptype="core") + if offer.in_ptype == offer.out_ptype == "core"), + *tool.primitive_offers("bend", in_ptype="core", ccw=False), + *tool.primitive_offers("bend", in_ptype="core", ccw=True), + *(offer for offer in tool.primitive_offers("s", in_ptype="core") + if offer.in_ptype == offer.out_ptype == "core"), + *tool.primitive_offers("u", in_ptype="core"), + *(offer for offer in tool.primitive_offers("straight", in_ptype="external") + if offer.in_ptype == "external" and offer.out_ptype == "core"), + *(offer for offer in tool.primitive_offers("straight", in_ptype="core") + if offer.in_ptype == "core" and offer.out_ptype == "external"), + ] + + assert len(offers) == 9 + assert all(offer.cost is cost for offer in offers) + + +def test_autotool_validates_cost_before_registering_any_offers() -> None: + def unused_sbend(_jog: float) -> Pattern: + raise AssertionError('invalid cost should be rejected before metadata inference') + + with pytest.raises(BuildError, match='cost factor'): + AutoTool().add_sbend(unused_sbend, jog_range=(-1, 1), cost=-1) + + def test_autotool_s_offer_singleton_jog_range_includes_both_signs() -> None: tool = make_sbend_tool((4, 4)) offers = tool.primitive_offers('s', in_ptype="core") diff --git a/masque/test/test_build_library.py b/masque/test/test_build_library.py index 6a0712e..36dd05d 100644 --- a/masque/test/test_build_library.py +++ b/masque/test/test_build_library.py @@ -3,8 +3,20 @@ from collections.abc import Iterator import pytest from ..builder import Pather -from ..error import BuildError -from ..library import BuildLibrary, BuildReport, ILibraryView, Library, cell, dangling_mode_t +from ..error import BuildError, LibraryError +from ..library import ( + INameView, + ILibrary, + IMaterializable, + LibraryBuilder, + BuildReport, + CellProvenance, + ILibraryView, + Library, + LibraryView, + cell, + dangling_mode_t, +) from ..pattern import Pattern from ..ports import Port @@ -42,16 +54,73 @@ class _MetadataSource(ILibraryView): return self._child_graph -def test_build_library_traces_declared_dependencies_out_of_order() -> None: - builder = BuildLibrary() +class _MaterializableMetadataSource(_MetadataSource, IMaterializable): + def materialize(self, name: str, *, persist: bool = True) -> Pattern: # noqa: ARG002 + return self[name] - def make_parent(lib: BuildLibrary) -> Pattern: + +def test_metadata_source_base_tops_stays_lazy() -> None: + source = _MetadataSource( + {"child": Pattern(), "top": Pattern()}, + {"child": set(), "top": {"child"}}, + ) + + assert source.tops() == ["top"] + assert source.loads == 0 + + +def test_metadata_source_reachability_and_subtree_stay_lazy() -> None: + source = _MetadataSource( + {"child": Pattern(), "top": Pattern(), "unused": Pattern()}, + {"child": set(), "top": {"child"}, "unused": set()}, + ) + + assert source.referenced_patterns("top") == {"child"} + assert source.dangling_refs("top") == set() + subtree = source.subtree("top") + + assert source.loads == 0 + assert subtree.source_order() == ("child", "top") + _ = subtree["top"] + assert source.loads == 1 + + +def test_build_report_defensively_freezes_mappings() -> None: + provenance = { + "top": CellProvenance( + requested_name="top", + kind="declared", + owner_declared_name="top", + build_chain=("top",), + ), + } + dependencies = {"top": frozenset({"child"})} + report = BuildReport( + requested_roots=("top",), + provenance=provenance, + dependency_graph=dependencies, + ) + provenance.clear() + dependencies.clear() + + assert set(report.provenance) == {"top"} + assert report.dependency_graph == {"top": frozenset({"child"})} + with pytest.raises(TypeError): + report.provenance["other"] = report.provenance["top"] # type: ignore[index] + with pytest.raises(TypeError): + report.dependency_graph["top"] = frozenset() # type: ignore[index] + + +def test_build_library_traces_declared_dependencies_out_of_order() -> None: + builder = LibraryBuilder() + + def make_parent(lib: ILibrary) -> Pattern: pat = Pattern() pat.ref("child") assert lib.abstract("child").name == "child" return pat - builder.cells.parent = cell(make_parent)(builder) + builder.cells.parent = cell(make_parent)(builder.library) builder["child"] = Pattern(ports={"p": Port((0, 0), 0)}) built, report = builder.build() @@ -62,10 +131,44 @@ def test_build_library_traces_declared_dependencies_out_of_order() -> None: assert report.provenance["parent"].kind == "declared" -def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None: - builder = BuildLibrary() +def test_build_cells_view_supports_underscore_declarations() -> None: + builder = LibraryBuilder() + builder.cells._helper = Pattern() - def make_top(lib: BuildLibrary) -> Pattern: + assert "_helper" in builder + with pytest.raises(BuildError, match="write-only"): + _value = builder.cells._helper + report = builder.validate() + assert report.provenance["_helper"].kind == "declared" + + del builder.cells._helper + assert "_helper" not in builder + + +def test_build_cells_view_builds_underscore_declaration() -> None: + builder = LibraryBuilder() + builder.cells._helper = Pattern() + + built, _report = builder.build(output="library") + + assert "_helper" in built + + +def test_build_library_tree_merge_rejects_single_cell_cycle() -> None: + tree = Library({"loop": Pattern()}) + tree["loop"].ref("loop") + builder = LibraryBuilder() + + with pytest.raises(LibraryError, match="exactly one topcell"): + builder << tree + + assert not builder + + +def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None: + builder = LibraryBuilder() + + def make_top(lib: ILibrary) -> Pattern: tree = Library({"_helper": Pattern()}) name_a = lib << tree name_b = lib << tree @@ -74,7 +177,7 @@ def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None top.ref(name_b) return top - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) _built, report = builder.build() helpers = [ @@ -88,7 +191,7 @@ def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None def test_build_library_authoring_tree_merge_renames_repeated_single_use_names() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() tree = Library({"_helper": Pattern()}) name_a = builder << tree @@ -103,7 +206,7 @@ def test_build_library_authoring_tree_merge_renames_repeated_single_use_names() def test_build_library_authoring_tree_merge_remaps_internal_refs() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["_helper"] = Pattern() helper = Pattern() top = Pattern() @@ -117,12 +220,14 @@ def test_build_library_authoring_tree_merge_remaps_internal_refs() -> None: assert any(name != "_helper" for name in built[top_name].refs) -def test_build_library_requires_build_session_for_reads_and_freezes_after_build() -> None: - builder = BuildLibrary() +def test_library_builder_is_not_a_readable_library_and_freezes_after_build() -> None: + builder = LibraryBuilder() builder["leaf"] = Pattern() - with pytest.raises(BuildError, match="validate\\(\\) or build\\(\\)"): - _ = builder["leaf"] + assert isinstance(builder, INameView) + assert not isinstance(builder, ILibraryView) + with pytest.raises(TypeError, match="not subscriptable"): + _ = builder["leaf"] # type: ignore[index] with pytest.raises(BuildError, match="write-only"): _ = builder.cells.leaf @@ -139,15 +244,15 @@ def test_build_library_requires_build_session_for_reads_and_freezes_after_build( def test_build_library_validate_is_retryable_after_failure() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() - def make_parent(lib: BuildLibrary) -> Pattern: + def make_parent(lib: ILibrary) -> Pattern: pat = Pattern() pat.ref("child") lib.abstract("child") return pat - builder.cells.parent = cell(make_parent)(builder) + builder.cells.parent = cell(make_parent)(builder.library) with pytest.raises(BuildError, match='Failed while building declared cell "parent"'): builder.validate() @@ -159,7 +264,7 @@ def test_build_library_validate_is_retryable_after_failure() -> None: def test_build_library_depends_on_supports_hidden_dependencies_for_partial_validation() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["child"] = Pattern() def make_parent() -> Pattern: @@ -174,8 +279,64 @@ def test_build_library_depends_on_supports_hidden_dependencies_for_partial_valid assert report.dependency_graph["parent"] == frozenset({"child"}) +def test_build_library_validate_accepts_single_string_and_deduplicates_roots() -> None: + builder = LibraryBuilder() + builder["top"] = Pattern() + + single = builder.validate(names="top") + duplicate = builder.validate(names=("top", "top")) + + assert single.requested_roots == ("top",) + assert duplicate.requested_roots == ("top",) + + +def test_build_library_validate_rejects_non_string_roots() -> None: + builder = LibraryBuilder() + builder["top"] = Pattern() + + with pytest.raises(TypeError, match="roots must be strings"): + builder.validate(names=("top", 1)) # type: ignore[arg-type] + + +@pytest.mark.parametrize("operation", ["build", "validate"]) +def test_build_library_rejects_same_builder_reentrancy(operation: str) -> None: + builder = LibraryBuilder() + calls = 0 + + def make_top() -> Pattern: + nonlocal calls + calls += 1 + if operation == "build": + builder.build() + else: + builder.validate(names=()) + return Pattern() + + builder.cells.top = cell(make_top)() + + with pytest.raises(BuildError, match="recursively"): + builder.build() + assert calls == 1 + + +def test_build_library_allows_nested_build_of_different_builder() -> None: + inner = LibraryBuilder() + inner["leaf"] = Pattern() + outer = LibraryBuilder() + + def make_top() -> Pattern: + built, _report = inner.build(output="library") + assert "leaf" in built + return Pattern() + + outer.cells.top = cell(make_top)() + built, _report = outer.build(output="library") + + assert "top" in built + + def test_build_library_validate_rejects_removed_output_argument() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["leaf"] = Pattern() with pytest.raises(TypeError): @@ -183,7 +344,7 @@ def test_build_library_validate_rejects_removed_output_argument() -> None: def test_build_library_rejects_unknown_build_output_mode() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["leaf"] = Pattern() with pytest.raises(ValueError, match="Unknown build output mode"): @@ -191,10 +352,10 @@ def test_build_library_rejects_unknown_build_output_mode() -> None: def test_build_library_allows_helper_writes_via_pather() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["leaf"] = Pattern(ports={"a": Port((0, 0), 0)}) - def make_top(lib: BuildLibrary) -> Pattern: + def make_top(lib: ILibrary) -> Pattern: helper = Pather(library=lib, ports="leaf", name="_route") top = Pattern() top.ref("_route") @@ -202,7 +363,7 @@ def test_build_library_allows_helper_writes_via_pather() -> None: top.ports.update(helper.pattern.ports) return top - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) _built, report = builder.build() helper_prov = report.provenance["_route"] @@ -211,11 +372,11 @@ def test_build_library_allows_helper_writes_via_pather() -> None: def test_build_library_contains_tracks_active_session_names() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() builder["leaf"] = Pattern() builder.add_source(Library({"src": Pattern()})) - def make_top(lib: BuildLibrary) -> Pattern: + def make_top(lib: ILibrary) -> Pattern: assert "leaf" in lib assert "src" in lib assert "_helper" not in lib @@ -223,7 +384,7 @@ def test_build_library_contains_tracks_active_session_names() -> None: assert "_helper" in lib return Pattern() - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) built, _report = builder.build() assert "_helper" in built @@ -231,7 +392,7 @@ def test_build_library_contains_tracks_active_session_names() -> None: def test_build_library_preserves_source_cells_and_records_source_provenance() -> None: source = Library({"src": Pattern()}) - builder = BuildLibrary() + builder = LibraryBuilder() builder.add_source(source) builder.cells.top = cell(lambda: Pattern())() @@ -241,6 +402,18 @@ def test_build_library_preserves_source_cells_and_records_source_provenance() -> assert report.provenance["src"].kind == "source" +def test_build_library_add_reserves_all_planned_names() -> None: + builder = LibraryBuilder() + builder["_shape$A"] = Pattern() + builder["_shape$B"] = Pattern() + source = Library({"_shape$A": Pattern(), "_shape$B": Pattern()}) + + rename_map = builder.add(source) + + assert len(set(rename_map.values())) == 2 + assert set(rename_map.values()) <= set(builder) + + def test_build_library_add_source_can_rename_every_source_cell() -> None: source = Library() source["child"] = Pattern() @@ -248,7 +421,7 @@ def test_build_library_add_source_can_rename_every_source_cell() -> None: parent.ref("child") source["parent"] = parent - builder = BuildLibrary() + builder = LibraryBuilder() rename_map = builder.add_source( source, rename_theirs=lambda _lib, name: f"mapped_{name}", @@ -264,7 +437,7 @@ def test_build_library_add_source_can_rename_every_source_cell() -> None: assert report.provenance["mapped_child"].requested_name == "child" -def test_build_library_authoring_tree_merge_keeps_source_view_lazy() -> None: +def test_library_builder_adds_an_ordinary_view_eagerly() -> None: child = Pattern() top = Pattern() top.ref("child") @@ -273,16 +446,16 @@ def test_build_library_authoring_tree_merge_keeps_source_view_lazy() -> None: {"child": set(), "top": {"child"}}, ) - builder = BuildLibrary() + builder = LibraryBuilder() top_name = builder << source built, _report = builder.build() assert top_name == "top" assert "top" in built - assert source.loads == 0 + assert source.loads == 2 -def test_build_library_authoring_source_tree_merge_returns_renamed_top() -> None: +def test_library_builder_adds_and_renames_an_ordinary_view_eagerly() -> None: existing = Pattern() source_top = Pattern() source = _MetadataSource( @@ -290,14 +463,41 @@ def test_build_library_authoring_source_tree_merge_returns_renamed_top() -> None {"_helper": set()}, ) - builder = BuildLibrary() + builder = LibraryBuilder() builder["_helper"] = existing top_name = builder << source built, _report = builder.build() assert top_name != "_helper" assert top_name in built + assert source.loads == 1 + + +def test_library_builder_add_borrows_materializable_views() -> None: + source = _MaterializableMetadataSource( + {"child": Pattern(), "top": Pattern()}, + {"child": set(), "top": {"child"}}, + ) + source.mapping["top"].ref("child") + builder = LibraryBuilder() + + builder.add(source) + built, _report = builder.build() + assert source.loads == 0 + assert set(built) == {"child", "top"} + + +def test_library_view_wrapper_intentionally_erases_materializable_marker() -> None: + source = _MaterializableMetadataSource( + {"top": Pattern()}, + {"top": set()}, + ) + builder = LibraryBuilder() + + builder.add(LibraryView(source)) + + assert source.loads == 1 def test_build_library_authoring_source_tree_merge_remaps_renamed_child_on_materialization() -> None: @@ -309,7 +509,7 @@ def test_build_library_authoring_source_tree_merge_remaps_renamed_child_on_mater {"_helper": set(), "top": {"_helper"}}, ) - builder = BuildLibrary() + builder = LibraryBuilder() builder["_helper"] = Pattern() top_name = builder << source built, _report = builder.build(output="library") @@ -319,18 +519,19 @@ def test_build_library_authoring_source_tree_merge_remaps_renamed_child_on_mater assert source.loads == 2 -def test_build_library_rejects_authoring_tree_le_before_mutating() -> None: - builder = BuildLibrary() +def test_library_builder_does_not_expose_library_hierarchy_operations() -> None: + builder = LibraryBuilder() - with pytest.raises(BuildError, match="__le__"): + with pytest.raises(TypeError): _abstract = builder <= Library({"leaf": Pattern()}) - - assert list(builder) == [] + assert not hasattr(builder, "abstract") + assert not hasattr(builder, "resolve") + assert not hasattr(builder, "rename") def test_build_library_rejects_source_cells_added_after_add_source() -> None: source = Library({"src": Pattern()}) - builder = BuildLibrary() + builder = LibraryBuilder() builder.add_source(source) source["late"] = Pattern() @@ -340,7 +541,7 @@ def test_build_library_rejects_source_cells_added_after_add_source() -> None: def test_build_library_rejects_source_cells_removed_after_add_source() -> None: source = Library({"src": Pattern()}) - builder = BuildLibrary() + builder = LibraryBuilder() builder.add_source(source) del source["src"] @@ -349,45 +550,29 @@ def test_build_library_rejects_source_cells_removed_after_add_source() -> None: def test_build_library_rejects_add_source_during_build() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() - def make_top(lib: BuildLibrary) -> Pattern: - lib.add_source(Library({"src": Pattern()})) + def make_top() -> Pattern: + builder.add_source(Library({"src": Pattern()})) return Pattern() - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)() - with pytest.raises(BuildError, match="add_source"): + with pytest.raises(BuildError, match="Cannot modify"): builder.build() -def test_build_library_rejects_renaming_imported_source_cells_during_authoring() -> None: - builder = BuildLibrary() - builder.add_source(Library({"src": Pattern()})) - - with pytest.raises(BuildError, match="add_source"): - builder.rename("src", "renamed_src", move_references=True) - - -def test_build_library_rejects_renaming_declared_cells_during_authoring() -> None: - builder = BuildLibrary() - builder["declared"] = Pattern() - - with pytest.raises(BuildError, match='Cannot rename declared build cell "declared"'): - builder.rename("declared", "renamed_declared") - - def test_build_library_helper_rename_updates_provenance_owner() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() - def make_top(lib: BuildLibrary) -> Pattern: + def make_top(lib: ILibrary) -> Pattern: lib["_helper"] = Pattern() lib.rename("_helper", "final_helper") top = Pattern() top.ref("final_helper") return top - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) built, report = builder.build() assert "final_helper" in built @@ -401,14 +586,14 @@ def test_build_library_helper_rename_updates_provenance_owner() -> None: def test_build_library_helper_delete_removes_provenance_and_ownership() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() - def make_top(lib: BuildLibrary) -> Pattern: + def make_top(lib: ILibrary) -> Pattern: lib["_helper"] = Pattern() del lib["_helper"] return Pattern() - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) built, report = builder.build() assert "_helper" not in built @@ -417,9 +602,9 @@ def test_build_library_helper_delete_removes_provenance_and_ownership() -> None: def test_build_library_helper_rename_after_auto_rename_preserves_requested_name() -> None: - builder = BuildLibrary() + builder = LibraryBuilder() - def make_top(lib: BuildLibrary) -> Pattern: + def make_top(lib: ILibrary) -> Pattern: tree = Library({"_helper": Pattern()}) _ = lib << tree renamed = lib << tree @@ -429,7 +614,7 @@ def test_build_library_helper_rename_after_auto_rename_preserves_requested_name( top.ref("final_helper") return top - builder.cells.top = cell(make_top)(builder) + builder.cells.top = cell(make_top)(builder.library) built, report = builder.build() assert "final_helper" in built @@ -438,48 +623,158 @@ def test_build_library_helper_rename_after_auto_rename_preserves_requested_name( def test_build_library_rejects_renaming_declared_or_source_cells_during_build() -> None: - declared = BuildLibrary() + declared = LibraryBuilder() declared["leaf"] = Pattern() - def rename_declared(lib: BuildLibrary) -> Pattern: + def rename_declared(lib: ILibrary) -> Pattern: lib.rename("leaf", "renamed_leaf") return Pattern() - declared.cells.top = cell(rename_declared)(declared) + declared.cells.top = cell(rename_declared)(declared.library) with pytest.raises(BuildError, match='Cannot rename declared build cell "leaf"'): declared.build() - source = BuildLibrary() + source = LibraryBuilder() source.add_source(Library({"src": Pattern()})) - def rename_source(lib: BuildLibrary) -> Pattern: + def rename_source(lib: ILibrary) -> Pattern: lib.rename("src", "renamed_src") return Pattern() - source.cells.top = cell(rename_source)(source) + source.cells.top = cell(rename_source)(source.library) with pytest.raises(BuildError, match='Cannot rename imported source cell "src"'): source.build() def test_build_library_rejects_deleting_declared_or_source_cells_during_build() -> None: - declared = BuildLibrary() + declared = LibraryBuilder() declared["leaf"] = Pattern() - def delete_declared(lib: BuildLibrary) -> Pattern: + def delete_declared(lib: ILibrary) -> Pattern: del lib["leaf"] return Pattern() - declared.cells.top = cell(delete_declared)(declared) + declared.cells.top = cell(delete_declared)(declared.library) with pytest.raises(BuildError, match='Cannot delete declared build cell "leaf"'): declared.build() - source = BuildLibrary() + source = LibraryBuilder() source.add_source(Library({"src": Pattern()})) - def delete_source(lib: BuildLibrary) -> Pattern: + def delete_source(lib: ILibrary) -> Pattern: del lib["src"] return Pattern() - source.cells.top = cell(delete_source)(source) + source.cells.top = cell(delete_source)(source.library) with pytest.raises(BuildError, match='Cannot delete imported source cell "src"'): source.build() + + +def test_library_builder_replaces_its_library_placeholder_in_direct_recipe_arguments() -> None: + builder = LibraryBuilder() + builder["leaf"] = Pattern() + + def make_top(positional: ILibrary, *, keyword: ILibrary) -> Pattern: + assert positional is keyword + assert positional.abstract("leaf").name == "leaf" + return Pattern() + + builder.cells.top = cell(make_top)(builder.library, keyword=builder.library) + built, _report = builder.build(output="library") + + assert "top" in built + + +def test_library_builder_library_placeholder_is_read_only() -> None: + builder = LibraryBuilder() + placeholder = builder.library + + with pytest.raises(AttributeError): + builder.library = object() # type: ignore[misc] + + assert builder.library is placeholder + builder.cells.top = cell(lambda _lib: Pattern())(builder.library) + built, _report = builder.build(output="library") + assert "top" in built + + +def test_library_builder_rejects_another_builders_direct_placeholder() -> None: + builder = LibraryBuilder() + other = LibraryBuilder() + + with pytest.raises(BuildError, match="another LibraryBuilder"): + builder.cells.top = cell(lambda _lib: Pattern())(other.library) + + +def test_library_builder_does_not_substitute_nested_placeholders() -> None: + builder = LibraryBuilder() + + def make_top(values: tuple[object, ...]) -> Pattern: + assert values == (builder.library,) + return Pattern() + + builder.cells.top = cell(make_top)((builder.library,)) + builder.build() + + +def test_library_builder_keeps_context_free_recipes_and_name_queries() -> None: + builder = LibraryBuilder() + builder.cells.top = cell(Pattern)() + + assert tuple(builder.keys()) == ("top",) + assert builder.get_name("top") != "top" + + +def test_library_builder_uses_library_name_allocation() -> None: + builder = LibraryBuilder() + builder["cell_name"] = Pattern() + library = Library({"cell_name": Pattern()}) + + requests = ( + ("cell name", True, 32), + ("a name that needs truncation", True, 12), + ("unsanitized name", False, 32), + ("", True, 32), + ) + for name, sanitize, max_length in requests: + assert builder.get_name(name, sanitize=sanitize, max_length=max_length) == library.get_name( + name, + sanitize=sanitize, + max_length=max_length, + ) + + +def test_build_session_subtree_preserves_type_and_builds_deferred_dependencies() -> None: + builder = LibraryBuilder() + child_calls = 0 + + def make_top(lib: ILibrary) -> Pattern: + subtree = lib.subtree("child") + assert type(subtree) is type(lib) + assert set(subtree) == {"child", "leaf"} + assert subtree["child"] is lib["child"] + with pytest.raises(KeyError): + _ = subtree["unrelated"] + subtree["_local"] = Pattern() + assert "_local" not in lib + + top = Pattern() + top.ref("child") + return top + + def make_child() -> Pattern: + nonlocal child_calls + child_calls += 1 + child = Pattern() + child.ref("leaf") + return child + + builder.cells.top = cell(make_top)(builder.library) + builder.cells.child = cell(make_child)() + builder.cells.leaf = Pattern() + builder.cells.unrelated = Pattern() + + built, _report = builder.build(output="library") + + assert child_calls == 1 + assert set(built) == {"top", "child", "leaf", "unrelated"} diff --git a/masque/test/test_gdsii_lazy.py b/masque/test/test_gdsii_lazy.py index b9da929..243b499 100644 --- a/masque/test/test_gdsii_lazy.py +++ b/masque/test/test_gdsii_lazy.py @@ -1,4 +1,5 @@ from pathlib import Path +import io import numpy import pytest @@ -6,9 +7,10 @@ from numpy.testing import assert_allclose from ..file import gdsii from ..file.gdsii import lazy as gdsii_lazy +from ..error import LibraryError from ..pattern import Pattern from ..ports import Port -from ..library import Library, OverlayLibrary +from ..library import IBorrowing, IMaterializable, Library, OverlayLibrary def _make_lazy_port_library() -> Library: @@ -29,6 +31,14 @@ def _make_lazy_port_library() -> Library: return lib +def test_gdsii_lazy_write_ignores_non_mapping_library_info() -> None: + lib = Library({'top': Pattern()}) + lib.library_info = None # type: ignore[attr-defined] + + with pytest.raises(LibraryError, match='required for non-GDS-backed lazy writes'): + gdsii_lazy.write(lib, io.BytesIO()) + + def test_gdsii_lazy_source_exposes_order_and_graph_without_materializing(tmp_path: Path) -> None: gds_file = tmp_path / 'lazy_source.gds' src = _make_lazy_port_library() @@ -43,13 +53,74 @@ def test_gdsii_lazy_source_exposes_order_and_graph_without_materializing(tmp_pat 'child': {'leaf'}, 'top': {'child'}, } + assert lib.parent_graph() == { + 'leaf': {'child'}, + 'child': {'top'}, + 'top': set(), + } + assert lib.tops() == ['top'] + global_refs = lib.find_refs_global('leaf') + assert_allclose(global_refs[('top', 'child', 'leaf')], [[110, 220, numpy.pi / 2, 0, 1]]) assert not lib._cache + with pytest.raises(ValueError, match='dangling-reference mode'): + lib.child_graph(dangling='typo') + with pytest.raises(ValueError, match='dangling-reference mode'): + lib.find_refs_local('leaf', dangling='typo') + child = lib['child'] assert list(child.refs.keys()) == ['leaf'] assert set(lib._cache) == {'child'} +def test_gdsii_lazy_subtree_stays_borrowed_and_preserves_write_metadata(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_subtree_source.gds' + src = _make_lazy_port_library() + src['unused'] = Pattern() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-subtree') + + raw, _ = gdsii_lazy.readfile(gds_file) + subtree = raw.subtree('top') + + assert isinstance(raw, IMaterializable) + assert not isinstance(raw, IBorrowing) + assert isinstance(subtree, IMaterializable) + assert isinstance(subtree, IBorrowing) + assert subtree.source_order() == ('leaf', 'child', 'top') + assert not raw._cache + + out_file = tmp_path / 'lazy_subtree_out.gds' + gdsii_lazy.writefile(subtree, out_file) + assert not raw._cache + + roundtrip, info = gdsii.readfile(out_file) + assert info['name'] == 'classic-subtree' + assert set(roundtrip) == {'leaf', 'child', 'top'} + + +def test_gdsii_lazy_overlay_subtree_preserves_source_laziness(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_overlay_subtree_source.gds' + src = _make_lazy_port_library() + src['unused'] = Pattern() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-subtree') + + raw, _ = gdsii_lazy.readfile(gds_file) + overlay = OverlayLibrary() + overlay.add_source(raw) + subtree = overlay.subtree('top') + + assert isinstance(subtree, OverlayLibrary) + assert subtree.borrowed_sources() == (raw,) + + out_file = tmp_path / 'lazy_overlay_subtree_out.gds' + gdsii_lazy.writefile(subtree, out_file) + + assert not raw._cache + roundtrip, info = gdsii.readfile(out_file) + assert info['name'] == 'overlay-subtree' + assert set(roundtrip) == {'leaf', 'child', 'top'} + + def test_gdsii_lazy_ports_view_keeps_raw_source_unmodified(tmp_path: Path) -> None: gds_file = tmp_path / 'lazy_ports.gds' src = _make_lazy_port_library() @@ -67,6 +138,28 @@ def test_gdsii_lazy_ports_view_keeps_raw_source_unmodified(tmp_path: Path) -> No assert not raw_top.ports +def test_gdsii_lazy_ports_view_detaches_previously_cached_source(tmp_path: Path) -> None: + gds_file = tmp_path / 'lazy_cached_ports.gds' + src = _make_lazy_port_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='classic-cached-ports') + + raw, _ = gdsii_lazy.readfile(gds_file) + raw_top = raw['top'] + processed = raw.with_port_overrides({ + 'top': { + 'P': Port((1, 2), rotation=0, ptype='wire'), + }, + }) + + processed_top = processed['top'] + + assert processed_top is not raw_top + assert not raw_top.ports + assert set(processed_top.ports) == {'P'} + assert raw['top'] is raw_top + assert not hasattr(processed, 'close') + + def test_gdsii_lazy_port_overrides_without_data_stay_lazy(tmp_path: Path) -> None: gds_file = tmp_path / 'lazy_port_overrides.gds' src = _make_lazy_port_library() diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 0447921..b7419d8 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -10,7 +10,7 @@ import pytest pytest.importorskip('pyarrow') from .. import PatternError -from ..library import Library, OverlayLibrary +from ..library import IBorrowing, IMaterializable, Library, OverlayLibrary from ..pattern import Pattern from ..repetition import Grid from ..file import gdsii @@ -223,6 +223,72 @@ def test_gdsii_lazy_arrow_untouched_write_is_copy_through(tmp_path: Path) -> Non assert out_file.read_bytes() == gds_file.read_bytes() +def test_gdsii_lazy_arrow_processed_cell_edit_disables_raw_copy(tmp_path: Path) -> None: + gds_file = tmp_path / 'processed_edit_source.gds' + src = _make_small_library() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='processed-edit') + + raw, _ = gdsii_lazy_arrow.readfile(gds_file) + processed = raw.with_port_overrides({}) + processed['top'].polygon((7, 0), vertices=[[0, 0], [4, 0], [0, 4]]) + + out_file = tmp_path / 'processed_edit_out.gds' + gdsii_lazy_arrow.writefile(processed, out_file) + + roundtrip, _ = gdsii.readfile(out_file) + assert len(roundtrip['top'].shapes[(7, 0)]) == 1 + + +def test_gdsii_lazy_arrow_subtree_preserves_raw_copy_and_ref_queries(tmp_path: Path) -> None: + gds_file = tmp_path / 'subtree_copy_source.gds' + src = _make_small_library() + src['unused'] = Pattern() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='subtree-copy') + + raw, _ = gdsii_lazy_arrow.readfile(gds_file) + subtree = raw.subtree('top') + + assert isinstance(raw, IMaterializable) + assert not isinstance(raw, IBorrowing) + assert isinstance(subtree, IMaterializable) + assert isinstance(subtree, IBorrowing) + assert subtree.source_order() == ('leaf', 'mid', 'top') + assert _global_refs_key(subtree.find_refs_global('leaf')) == _global_refs_key(raw.find_refs_global('leaf')) + assert not raw._cache + + out_file = tmp_path / 'subtree_copy_out.gds' + gdsii_lazy_arrow.writefile(subtree, out_file) + + assert not raw._cache + roundtrip, info = gdsii.readfile(out_file) + assert info['name'] == 'subtree-copy' + assert set(roundtrip) == {'leaf', 'mid', 'top'} + + +def test_gdsii_lazy_arrow_overlay_subtree_preserves_raw_copy(tmp_path: Path) -> None: + gds_file = tmp_path / 'overlay_subtree_source.gds' + src = _make_small_library() + src['unused'] = Pattern() + gdsii.writefile(src, gds_file, meters_per_unit=1e-9, library_name='overlay-subtree-copy') + + raw, _ = gdsii_lazy_arrow.readfile(gds_file) + overlay = OverlayLibrary() + overlay.add_source(raw) + subtree = overlay.subtree('top') + + assert isinstance(subtree, OverlayLibrary) + assert subtree.borrowed_sources() == (raw,) + assert not raw._cache + + out_file = tmp_path / 'overlay_subtree_out.gds' + gdsii_lazy_arrow.writefile(subtree, out_file) + + assert not raw._cache + roundtrip, info = gdsii.readfile(out_file) + assert info['name'] == 'overlay-subtree-copy' + assert set(roundtrip) == {'leaf', 'mid', 'top'} + + def test_gdsii_lazy_arrow_gzipped_copy_through(tmp_path: Path) -> None: gds_file = tmp_path / 'copy_source.gds.gz' src = _make_small_library() diff --git a/masque/test/test_library.py b/masque/test/test_library.py index ce564aa..157b0a1 100644 --- a/masque/test/test_library.py +++ b/masque/test/test_library.py @@ -1,7 +1,8 @@ import pytest +from collections.abc import Iterator, Mapping, MutableMapping from typing import cast, TYPE_CHECKING from numpy.testing import assert_allclose -from ..library import Library, LazyLibrary +from ..library import IBorrowing, INameView, IMaterializable, ILibraryView, Library, LibraryView, LazyLibrary, OverlayLibrary, PortsLibraryView from ..pattern import Pattern from ..error import LibraryError, PatternError from ..ports import Port @@ -18,6 +19,7 @@ def test_library_basic() -> None: pat = Pattern() lib["cell1"] = pat + assert isinstance(lib, INameView) assert "cell1" in lib assert lib["cell1"] is pat assert len(lib) == 1 @@ -26,6 +28,22 @@ def test_library_basic() -> None: lib["cell1"] = Pattern() # Overwriting not allowed +@pytest.mark.parametrize("library_cls", [Library, LazyLibrary, OverlayLibrary]) +def test_writable_libraries_are_restricted_mappings( + library_cls: type[Library] | type[LazyLibrary] | type[OverlayLibrary], + ) -> None: + lib = library_cls() + + assert isinstance(lib, Mapping) + assert not isinstance(lib, MutableMapping) + for method in ("update", "setdefault", "pop", "popitem", "clear"): + assert not hasattr(lib, method) + + lib["top"] = Pattern() + del lib["top"] + assert not lib + + def test_library_tops() -> None: lib = Library() lib["child"] = Pattern() @@ -36,6 +54,23 @@ def test_library_tops() -> None: assert lib.top() == "parent" +def test_empty_ref_buckets_do_not_create_hierarchy_edges() -> None: + parent = Pattern() + parent.refs["ghost"] + parent.refs["parent"] + lib = Library({"parent": parent, "ghost": Pattern()}) + + assert not parent.has_refs() + assert parent.referenced_patterns() == set() + assert set(lib.tops()) == {"parent", "ghost"} + assert lib.child_graph() == {"parent": set(), "ghost": set()} + assert lib.parent_graph() == {"parent": set(), "ghost": set()} + + lib.dfs(parent, hierarchy=("parent",)) + flat = lib.flatten("parent")["parent"] + assert not flat.refs + + def test_library_dangling() -> None: lib = Library() lib["parent"] = Pattern() @@ -44,6 +79,16 @@ def test_library_dangling() -> None: assert lib.dangling_refs() == {"missing"} +def test_library_reachability_ignores_unnamed_refs() -> None: + pattern = Pattern() + pattern.ref(None) + lib = Library({"top": pattern}) + + assert pattern.referenced_patterns() == {None} + assert lib.referenced_patterns() == set() + assert lib.dangling_refs() == set() + + def test_library_dangling_graph_modes() -> None: lib = Library() lib["parent"] = Pattern() @@ -65,6 +110,24 @@ def test_library_dangling_graph_modes() -> None: assert lib.child_order(dangling="include") == ["missing", "parent"] +@pytest.mark.parametrize( + ("method", "args"), + [ + ("child_graph", ()), + ("parent_graph", ()), + ("child_order", ()), + ("find_refs_local", ("top",)), + ("find_refs_global", ("top",)), + ("prune_empty", ()), + ], +) +def test_library_rejects_unknown_dangling_mode(method: str, args: tuple[object, ...]) -> None: + lib = Library({"top": Pattern()}) + + with pytest.raises(ValueError, match="dangling-reference mode"): + getattr(lib, method)(*args, dangling="typo") + + def test_find_refs_with_dangling_modes() -> None: lib = Library() lib["target"] = Pattern() @@ -97,6 +160,16 @@ def test_find_refs_with_dangling_modes() -> None: assert_allclose(global_target[("top", "mid", "target")], [[7, 0, 0, 0, 1]]) +def test_find_refs_global_includes_composed_scale_column() -> None: + lib = Library({"leaf": Pattern(), "child": Pattern(), "top": Pattern()}) + lib["child"].ref("leaf", scale=2) + lib["top"].ref("child", scale=3) + + transforms = lib.find_refs_global("leaf") + + assert_allclose(transforms[("top", "child", "leaf")], [[0, 0, 0, 0, 6]]) + + def test_preflight_prune_empty_preserves_dangling_policy(caplog: pytest.LogCaptureFixture) -> None: def make_lib() -> Library: lib = Library() @@ -137,6 +210,42 @@ def test_library_flatten() -> None: assert tuple(assert_vertices[0]) == (10.0, 10.0) +def test_pattern_polygon_traversal_ignores_empty_dangling_bucket() -> None: + child = Pattern() + child.polygon((1, 0), vertices=[[0, 0], [1, 0], [0, 1]]) + parent = Pattern() + parent.ref("child") + parent.refs["missing"] + lib = Library({"child": child, "parent": parent}) + + polygons = parent.layer_as_polygons((1, 0), flatten=True, library=lib) + + assert len(polygons) == 1 + + +def test_recursive_geometry_rejects_reference_cycles() -> None: + lib = Library({"a": Pattern(), "b": Pattern()}) + lib["a"].ref("b") + lib["b"].ref("a") + + with pytest.raises(PatternError, match=r"calculating bounds: .* -> .* ->"): + lib["a"].get_bounds(library=lib) + with pytest.raises(PatternError, match=r"collecting layer polygons: .* -> .* ->"): + lib["a"].layer_as_polygons((1, 0), library=lib) + with pytest.raises(PatternError, match=r"visualizing pattern hierarchy: .* -> .* ->"): + lib["a"].visualize(library=lib) + with pytest.raises(PatternError, match="Circular reference"): + lib["a"].deepcopy().flatten(library=lib) + + +def test_nonrecursive_geometry_allows_reference_cycles() -> None: + lib = Library({"loop": Pattern()}) + lib["loop"].ref("loop") + + assert lib["loop"].get_bounds(library=lib, recurse=False) is None + assert lib["loop"].layer_as_polygons((1, 0), flatten=False, library=lib) == [] + + def test_library_flatten_preserves_ports_only_child() -> None: lib = Library() child = Pattern(ports={"P1": Port((1, 2), 0)}) @@ -207,6 +316,42 @@ def test_lazy_library() -> None: assert pat is pat2 +def test_lazy_library_reachability_loads_only_reachable_cells() -> None: + lib = LazyLibrary() + calls = {"top": 0, "child": 0, "unused": 0} + + def make(name: str, target: str | None = None) -> Pattern: + calls[name] += 1 + pattern = Pattern() + if target is not None: + pattern.ref(target) + return pattern + + lib["top"] = lambda: make("top", "child") + lib["child"] = lambda: make("child") + lib["unused"] = lambda: make("unused") + + assert lib.referenced_patterns("top") == {"child"} + assert calls == {"top": 1, "child": 1, "unused": 0} + + +def test_abstract_view_membership_does_not_materialize_lazy_cells() -> None: + lib = LazyLibrary() + calls = 0 + + def make_pat() -> Pattern: + nonlocal calls + calls += 1 + return Pattern() + + lib["lazy"] = make_pat + abstracts = lib.abstract_view() + + assert "lazy" in abstracts + assert "missing" not in abstracts + assert calls == 0 + + def test_library_rename() -> None: lib = Library() lib["old"] = Pattern() @@ -221,7 +366,7 @@ def test_library_rename() -> None: assert "old" not in lib["parent"].refs -@pytest.mark.parametrize("library_cls", (Library, LazyLibrary)) +@pytest.mark.parametrize("library_cls", [Library, LazyLibrary]) def test_library_rename_self_is_noop(library_cls: type[Library] | type[LazyLibrary]) -> None: lib = library_cls() lib["top"] = Pattern() @@ -235,7 +380,7 @@ def test_library_rename_self_is_noop(library_cls: type[Library] | type[LazyLibra assert len(lib["parent"].refs["top"]) == 1 -@pytest.mark.parametrize("library_cls", (Library, LazyLibrary)) +@pytest.mark.parametrize("library_cls", [Library, LazyLibrary]) def test_library_rename_top_self_is_noop(library_cls: type[Library] | type[LazyLibrary]) -> None: lib = library_cls() lib["top"] = Pattern() @@ -245,7 +390,7 @@ def test_library_rename_top_self_is_noop(library_cls: type[Library] | type[LazyL assert list(lib.keys()) == ["top"] -@pytest.mark.parametrize("library_cls", (Library, LazyLibrary)) +@pytest.mark.parametrize("library_cls", [Library, LazyLibrary]) def test_library_rename_missing_raises_library_error(library_cls: type[Library] | type[LazyLibrary]) -> None: lib = library_cls() lib["top"] = Pattern() @@ -254,7 +399,7 @@ def test_library_rename_missing_raises_library_error(library_cls: type[Library] lib.rename("missing", "new") -@pytest.mark.parametrize("library_cls", (Library, LazyLibrary)) +@pytest.mark.parametrize("library_cls", [Library, LazyLibrary]) def test_library_move_references_same_target_is_noop(library_cls: type[Library] | type[LazyLibrary]) -> None: lib = library_cls() lib["top"] = Pattern() @@ -338,6 +483,80 @@ def test_library_add_returns_only_renamed_entries() -> None: assert "keep" not in rename_map +def test_library_add_name_failure_is_atomic() -> None: + destination = Library({"x": Pattern(), "y": Pattern()}) + source_parent = Pattern() + source_parent.ref("x") + source = Library({"x": Pattern(), "y": source_parent}) + + with pytest.raises(LibraryError, match="Unresolved duplicate"): + destination.add(source, rename_theirs=lambda _lib, _name: "z", mutate_other=True) + + assert set(destination) == {"x", "y"} + assert set(source) == {"x", "y"} + assert set(source["y"].refs) == {"x"} + + +def test_library_add_callback_sees_earlier_name_reservations() -> None: + destination = Library({"_shape$A": Pattern(), "_shape$B": Pattern()}) + source = Library({"_shape$A": Pattern(), "_shape$B": Pattern()}) + callback_views: list[set[str]] = [] + + def rename(view: INameView, _name: str) -> str: + assert isinstance(view, INameView) + assert not isinstance(view, Mapping) + assert not hasattr(view, "__getitem__") + callback_views.append(set(view)) + return view.get_name("_shape") + + rename_map = destination.add(source, rename_theirs=rename) + + assert len(set(rename_map.values())) == 2 + assert rename_map["_shape$A"] in callback_views[1] + assert set(rename_map.values()) <= set(destination) + + +def test_library_can_add_itself_with_mutate_other() -> None: + lib = Library({"_helper": Pattern()}) + + rename_map = lib.add(lib, mutate_other=True) + + assert rename_map["_helper"] in lib + assert len(lib) == 2 + + +def test_overlay_add_source_callback_sees_earlier_name_reservations() -> None: + overlay = OverlayLibrary() + overlay["_shape$A"] = Pattern() + overlay["_shape$B"] = Pattern() + source = Library({"_shape$A": Pattern(), "_shape$B": Pattern()}) + + rename_map = overlay.add_source( + source, + rename_theirs=lambda view, _name: view.get_name("_shape"), + ) + + assert len(set(rename_map.values())) == 2 + assert set(rename_map.values()) <= set(overlay) + + +def test_ports_view_detaches_already_materialized_overlay_pattern() -> None: + overlay = OverlayLibrary() + overlay.add_source(Library({"top": Pattern()})) + raw = overlay["top"] + processed = PortsLibraryView( + overlay, + ports={"top": {"P": Port((1, 2), 0)}}, + ) + + processed_top = processed["top"] + + assert processed_top is not raw + assert not raw.ports + assert set(processed_top.ports) == {"P"} + assert not hasattr(processed, "close") + + def test_library_subtree() -> None: lib = Library() lib["a"] = Pattern() @@ -346,9 +565,230 @@ def test_library_subtree() -> None: lib["a"].ref("b") sub = lib.subtree("a") + assert isinstance(sub, Library) assert "a" in sub assert "b" in sub assert "c" not in sub + assert sub["a"] is lib["a"] + + del sub["b"] + assert "b" in lib + + +def test_lazy_library_subtree_preserves_type_and_shares_materialized_patterns() -> None: + lib = LazyLibrary() + child = Pattern() + top = Pattern() + top.ref("child") + lib["child"] = lambda: child + lib["top"] = lambda: top + lib["unused"] = Pattern() + + subtree = lib.subtree("top") + + assert isinstance(subtree, LazyLibrary) + assert set(subtree) == {"child", "top"} + assert subtree["top"] is lib["top"] + del subtree["child"] + assert "child" in lib + + +def test_overlay_subtree_preserves_type_sources_and_independent_promotion() -> None: + source = Library({"child": Pattern(), "top": Pattern(), "unused": Pattern()}) + source["top"].ref("child") + overlay = OverlayLibrary() + overlay.add_source(source) + + subtree = overlay.subtree("top") + + assert isinstance(subtree, OverlayLibrary) + assert set(subtree) == {"child", "top"} + assert subtree.borrowed_sources() == overlay.borrowed_sources() + subtree_top = subtree["top"] + overlay_top = overlay["top"] + assert subtree_top is not overlay_top + + shared_subtree = overlay.subtree("top") + assert shared_subtree["top"] is overlay_top + del shared_subtree["child"] + assert "child" in overlay + + +def test_overlay_subtree_preserves_reference_remaps() -> None: + source = Library({"leaf": Pattern(), "parent": Pattern()}) + source["parent"].ref("leaf") + overlay = OverlayLibrary() + overlay.add_source(source) + overlay.rename("leaf", "renamed", move_references=True) + + subtree = overlay.subtree("parent") + + assert isinstance(subtree, OverlayLibrary) + assert set(subtree) == {"parent", "renamed"} + assert set(subtree["parent"].refs) == {"renamed"} + + +def test_read_only_subtree_is_a_borrowed_subtree_view() -> None: + source = Library({"child": Pattern(), "top": Pattern(), "unused": Pattern()}) + source["top"].ref("child") + view = LibraryView(source) + + subtree = view.subtree("top") + + assert subtree.source_order() == ("child", "top") + assert subtree["top"] is source["top"] + assert "unused" not in subtree + with pytest.raises(KeyError): + _ = subtree["unused"] + + +def test_library_materialization_and_borrowing_capabilities() -> None: + lazy = LazyLibrary() + lazy["top"] = lambda: Pattern() + + assert isinstance(lazy, IMaterializable) + assert not isinstance(lazy, IBorrowing) + assert not hasattr(lazy, "borrowed_sources") + transient = lazy.materialize("top", persist=False) + assert "top" not in lazy.cache + assert transient is not lazy["top"] + + overlay = OverlayLibrary() + overlay.add_source(lazy) + ports = PortsLibraryView(overlay) + subtree = ports.subtree("top") + + assert isinstance(overlay, IMaterializable) + assert isinstance(overlay, IBorrowing) + assert isinstance(ports, IMaterializable) + assert isinstance(ports, IBorrowing) + assert isinstance(subtree, IMaterializable) + assert isinstance(subtree, IBorrowing) + assert overlay.borrowed_sources() == (lazy,) + assert ports.borrowed_sources() == (overlay,) + assert subtree.borrowed_sources() == (ports,) + eager = Library({"top": Pattern()}) + plain_view = LibraryView(lazy) + assert not isinstance(eager, IMaterializable | IBorrowing) + assert not isinstance(plain_view, IMaterializable | IBorrowing) + + +class _RawCopyView(ILibraryView): + def __init__(self) -> None: + self.mapping = {"top": Pattern()} + + def __getitem__(self, key: str) -> Pattern: + return self.mapping[key] + + def __iter__(self) -> Iterator[str]: + return iter(self.mapping) + + def __len__(self) -> int: + return len(self.mapping) + + def __contains__(self, key: object) -> bool: + return key in self.mapping + + def raw_struct_bytes(self, name: str) -> bytes: + return name.encode() + + def can_copy_raw_struct(self, name: str) -> bool: + return name in self.mapping + + +def test_ports_view_raw_copy_eligibility_tracks_persistent_materialization() -> None: + processed = PortsLibraryView(_RawCopyView()) + + assert processed.can_copy_raw_struct("top") + _ = processed.materialize_many(("top",), persist=False) + assert processed.can_copy_raw_struct("top") + + subtree = processed.subtree("top") + assert subtree.can_copy_raw_struct("top") + _ = subtree["top"] + assert not processed.can_copy_raw_struct("top") + assert not subtree.can_copy_raw_struct("top") + + +@pytest.mark.parametrize("materialize_parent", [False, True]) +@pytest.mark.parametrize("move_references", [False, True]) +def test_overlay_rename_is_independent_of_materialization( + materialize_parent: bool, + move_references: bool, + ) -> None: + source = Library({"leaf": Pattern(), "parent": Pattern()}) + source["parent"].ref("leaf") + overlay = OverlayLibrary() + overlay.add_source(source) + + if materialize_parent: + _ = overlay["parent"] + + overlay.rename("leaf", "renamed", move_references=move_references) + expected_target = "renamed" if move_references else "leaf" + + assert overlay.child_graph(dangling="include")["parent"] == {expected_target} + assert set(overlay["parent"].refs) == {expected_target} + assert overlay.child_graph(dangling="include")["parent"] == {expected_target} + + +def test_overlay_rejects_unknown_dangling_mode() -> None: + overlay = OverlayLibrary() + overlay.add_source(Library({"top": Pattern()})) + + with pytest.raises(ValueError, match="dangling-reference mode"): + overlay.child_graph(dangling="typo") + with pytest.raises(ValueError, match="dangling-reference mode"): + overlay.find_refs_local("top", dangling="typo") + + +def test_overlay_rename_preserves_initial_import_target_without_moving_refs() -> None: + source = Library({"leaf": Pattern(), "parent": Pattern()}) + source["parent"].ref("leaf") + overlay = OverlayLibrary() + overlay["leaf"] = Pattern() + rename_map = overlay.add_source(source, rename_theirs=lambda lib, name: lib.get_name(name)) + imported_leaf = rename_map["leaf"] + + overlay.rename(imported_leaf, "renamed", move_references=False) + + assert set(overlay["parent"].refs) == {imported_leaf} + + +def test_overlay_chained_rename_moves_unmaterialized_references() -> None: + source = Library({"leaf": Pattern(), "parent": Pattern()}) + source["parent"].ref("leaf") + overlay = OverlayLibrary() + overlay.add_source(source) + + overlay.rename("leaf", "middle", move_references=True) + overlay.rename("middle", "final", move_references=True) + + assert set(overlay["parent"].refs) == {"final"} + + +def _assert_tree_merge_rejected(tree: Library) -> None: + destination = Library({"existing": Pattern()}) + + with pytest.raises(LibraryError, match="exactly one topcell"): + destination << tree + + assert set(destination) == {"existing"} + + +def test_library_tree_merge_rejects_empty_tree() -> None: + _assert_tree_merge_rejected(Library()) + + +def test_library_tree_merge_rejects_cycle_without_top() -> None: + tree = Library({"a": Pattern(), "b": Pattern()}) + tree["a"].ref("b") + tree["b"].ref("a") + _assert_tree_merge_rejected(tree) + + +def test_library_tree_merge_rejects_multiple_tops() -> None: + _assert_tree_merge_rejected(Library({"a": Pattern(), "b": Pattern()})) def test_library_child_order_cycle_raises_library_error() -> None: diff --git a/masque/test/test_pather_constraints.py b/masque/test/test_pather_constraints.py index 30cf34f..56be6ff 100644 --- a/masque/test/test_pather_constraints.py +++ b/masque/test/test_pather_constraints.py @@ -164,7 +164,7 @@ class PreferredMinimumTool(PlanningOnlyTool): BendOffer( in_ptype=in_ptype, out_ptype='wide', - priority_bias=100, + cost=100, ccw=ccw, length_domain=(2, 2), endpoint_planner=lambda _length: Port((2, jog), rotation, ptype='wide'), diff --git a/masque/test/test_pather_primitive_offers.py b/masque/test/test_pather_primitive_offers.py index 7dbce90..0ab7ea4 100644 --- a/masque/test/test_pather_primitive_offers.py +++ b/masque/test/test_pather_primitive_offers.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from dataclasses import dataclass from typing import Any, Literal import numpy @@ -146,14 +147,15 @@ def test_straight_offer_generated_factory_uses_default_endpoint_and_data_bbox() 'wire', data_at, length_domain=(2, 8), - priority_bias=3, + cost=3, bbox_for_data=lambda data: numpy.array([[0, 0], [data['length'], 1]]), ) endpoint = offer.endpoint_at(4) assert offer.length_domain == (2, 8) - assert offer.priority_bias == 3 + assert offer.cost == 3 + assert offer.cost_at(4) == 12 assert numpy.allclose(endpoint.offset, [4, 0]) assert endpoint.rotation == pi assert endpoint.ptype == 'wire' @@ -256,9 +258,43 @@ def test_prebuilt_offer_factories_return_fresh_endpoint_copies() -> None: assert numpy.allclose(offer.bbox_at(parameter), [[-1, -2], [6, 3]]) -def test_offer_rejects_negative_priority_bias() -> None: - with pytest.raises(BuildError, match='priority_bias must be nonnegative'): - StraightOffer(in_ptype='wire', out_ptype='wire', priority_bias=-1) +@pytest.mark.parametrize('cost', [-1, numpy.inf, numpy.nan, 'expensive']) +def test_offer_rejects_invalid_cost_factor(cost: Any) -> None: + with pytest.raises(BuildError, match='cost factor'): + StraightOffer(in_ptype='wire', out_ptype='wire', cost=cost) + + +@pytest.mark.parametrize('result', [-1, numpy.inf, numpy.nan, 'expensive']) +def test_offer_rejects_invalid_callable_cost_result(result: Any) -> None: + offer = StraightOffer( + in_ptype='wire', + out_ptype='wire', + cost=lambda _parameter, _endpoint: result, + **offer_callbacks(lambda length: (Port((length, 0), rotation=pi, ptype='wire'), None)), + ) + + with pytest.raises(BuildError, match='Primitive cost'): + offer.cost_at(5) + + +def test_offer_callable_cost_replaces_default_cost() -> None: + seen: list[tuple[float, Port]] = [] + + def cost(parameter: float, endpoint: Port) -> float: + seen.append((parameter, endpoint)) + return parameter + endpoint.y + + offer = SOffer( + in_ptype='wire', + out_ptype='wire', + cost=cost, + jog_domain=(2, 2), + **offer_callbacks(lambda jog: (Port((10, jog), rotation=pi, ptype='wire'), None)), + ) + + assert offer.cost_at(2 + 1e-13) == 4 + assert seen[0][0] == 2 + assert numpy.allclose(seen[0][1].offset, (10, 2)) def test_offer_rejects_one_sided_split_callbacks() -> None: @@ -416,8 +452,8 @@ def test_pather_selects_lowest_cost_offer() -> None: return Port((length, 0), rotation=pi, ptype=out_ptype or in_ptype), {'kind': 'low'} return ( - StraightOffer(in_ptype=in_ptype, out_ptype=out_ptype, priority_bias=10, **offer_callbacks(high)), - StraightOffer(in_ptype=in_ptype, out_ptype=out_ptype, priority_bias=0, **offer_callbacks(low)), + StraightOffer(in_ptype=in_ptype, out_ptype=out_ptype, cost=10, **offer_callbacks(high)), + StraightOffer(in_ptype=in_ptype, out_ptype=out_ptype, cost=1, **offer_callbacks(low)), ) p = Pather(Library(), tools=MultiOfferTool(), render='deferred') @@ -429,6 +465,51 @@ def test_pather_selects_lowest_cost_offer() -> None: assert numpy.allclose(p.ports['A'].offset, (-7, 0)) +def test_planner_deduplication_distinguishes_offer_costs() -> None: + @dataclass(frozen=True, slots=True) + class MarkedStraightOffer(StraightOffer): + marker: str = '' + + def commit(self, parameter: float) -> dict[str, str | float]: + return {'kind': self.marker, 'length': self.canonicalize_parameter(parameter)} + + def endpoint(length: float) -> Port: + return Port((length, 0), rotation=pi, ptype='wire') + + def shared_commit(length: float) -> dict[str, float]: + return {'length': length} + + class DuplicateGeometryTool(PlanningOnlyTool): + def primitive_offers( + self, + kind: Literal['straight', 'bend', 's', 'u'], + *, + in_ptype: str | None = None, + out_ptype: str | None = None, + **kwargs: Any, + ) -> tuple[PrimitiveOffer, ...]: + _ = kwargs + if kind != 'straight': + return () + common = { + 'in_ptype': in_ptype, + 'out_ptype': out_ptype, + 'endpoint_planner': endpoint, + 'commit_planner': shared_commit, + } + return ( + MarkedStraightOffer(cost=100, marker='expensive', **common), + MarkedStraightOffer(cost=1, marker='cheap', **common), + ) + + pather = Pather(Library(), tools=DuplicateGeometryTool(), render='deferred') + pather.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + pather.straight('A', 7) + + assert pather._paths['A'][0].data == {'kind': 'cheap', 'length': 7} + + class StrategyTieTool(PlanningOnlyTool): def __init__(self) -> None: self.seen_kwargs: list[dict[str, Any]] = [] @@ -591,7 +672,7 @@ def test_pather_commits_only_selected_offer() -> None: if kind != 'straight': return () - def make(label: str, priority: float) -> StraightOffer: + def make(label: str, cost: float) -> StraightOffer: def endpoint(length: float) -> Port: return Port((length, 0), rotation=pi, ptype=out_ptype or in_ptype) @@ -602,12 +683,12 @@ def test_pather_commits_only_selected_offer() -> None: return StraightOffer( in_ptype=in_ptype, out_ptype=out_ptype, - priority_bias=priority, + cost=cost, endpoint_planner=endpoint, commit_planner=commit, ) - return (make('expensive', 100), make('cheap', 0)) + return (make('expensive', 100), make('cheap', 1)) p = Pather(Library(), tools=RecordingTool(), render='deferred') p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') diff --git a/masque/test/test_ports2data.py b/masque/test/test_ports2data.py index 3f642ab..11b4619 100644 --- a/masque/test/test_ports2data.py +++ b/masque/test/test_ports2data.py @@ -60,6 +60,15 @@ def test_data_to_ports_hierarchical() -> None: assert_allclose(parent.ports["A"].rotation, numpy.pi / 2, atol=1e-10) +def test_data_to_ports_ignores_empty_dangling_ref_bucket() -> None: + parent = Pattern() + parent.refs["missing"] + + data_to_ports([(10, 0)], Library(), parent, max_depth=1) + + assert not parent.ports + + def test_data_to_ports_hierarchical_scaled_ref() -> None: lib = Library() diff --git a/masque/utils/ports2data.py b/masque/utils/ports2data.py index 44a0ec3..6ecc253 100644 --- a/masque/utils/ports2data.py +++ b/masque/utils/ports2data.py @@ -105,8 +105,8 @@ def data_to_ports( # Load ports for all subpatterns, and use any we find found_ports = False - for target in pattern.refs: - if target is None: + for target, refs in pattern.refs.items(): + if target is None or not refs: continue pp = data_to_ports( layers = layers,