jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[BuildLibrary] eliminate BuiltLibrary and BuiltOverlayLibrary

Browse source
This commit is contained in:
Jan Petykiewicz 2026-06-19 21:04:18 -07:00
commit 3dea61b05e
6 changed files with 95 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/tutorial/README.md
View file

@ -22,7 +22,7 @@ Contents
- [library](library.py)
* Continue from `devices.py` by declaring a mixed library with `BuildLibrary`
* Import source-backed GDS cells and register python-generated recipes together
* Call `build()` to produce a normal library for downstream `Pather` usage and writing
* Call `build()` to produce a normal library and report for downstream `Pather` usage and writing
* Explore alternate ways of specifying a pattern for `.plug()` and `.place()`
- [pather](pather.py)
* Use `Pather` to route individual wires and wire bundles

3
examples/tutorial/library.py
View file

@ -81,8 +81,9 @@ def main() -> None:
# Build the declaration set into a normal library.
#
built = builder.build()
built, report = builder.build()
print('Built library contains:\n' + pformat(list(built.keys())))
print('Build dependency graph:\n' + pformat(report.dependency_graph))
#
# Continue designing against the built library.

1
masque/__init__.py
View file

@ -63,7 +63,6 @@ from .library import (
ILibrary as ILibrary,
LibraryView as LibraryView,
Library as Library,
BuiltLibrary as BuiltLibrary,
BuildLibrary as BuildLibrary,
BuildReport as BuildReport,
CellProvenance as CellProvenance,

14
masque/file/gdsii_lazy_core.py
View file

@ -549,20 +549,6 @@ class OverlayLibrary(ILibrary):
return tuple(name for name in self._order if name in self._entries)
class BuiltOverlayLibrary(OverlayLibrary):
"""
Internal overlay output returned by `BuildLibrary.build(output='overlay')`.
The type is intentionally not part of the public API. It exists so build
outputs can carry a `build_report` while still behaving like an
`OverlayLibrary`.
"""
def __init__(self, *, build_report: Any | None = None) -> None:
super().__init__()
self.build_report = build_report
def _iter_library_infos(library: Mapping[str, Pattern] | ILibraryView) -> Iterator[dict[str, Any]]:
info = getattr(library, 'library_info', None)
if isinstance(info, dict):

168
masque/library.py
View file

@ -69,9 +69,6 @@ Tree: TypeAlias = MutableMapping[str, 'Pattern']
dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include']
""" How helpers should handle refs whose targets are not present in the library. """
emitted_via_t: TypeAlias = Literal['declaration', 'helper_write', 'tree_merge', 'source_import']
""" Build-provenance origin tags for emitted cells. """
@dataclass(frozen=True)
class CellProvenance:
@ -83,29 +80,22 @@ class CellProvenance:
chosen.
Attributes:
final_name: Name exposed by the completed library.
requested_name: First name requested for this cell during the build.
kind: Whether the cell came from a declaration, helper emission, or an
imported source library.
owner_declared_name: Declared cell responsible for this output cell, if
any. Imported source cells leave this as `None`.
emitted_via: High-level path by which the cell entered the output.
build_chain: Declared-cell dependency chain that was active when the
cell was emitted.
renamed_from: Original requested name when the final name differs.
source_name: Original on-source name for imported cells.
source_metadata: Optional source-library metadata copied through from
lazy GDS readers.
"""
final_name: str
requested_name: str
kind: Literal['declared', 'helper', 'source']
owner_declared_name: str | None
emitted_via: emitted_via_t
build_chain: tuple[str, ...]
renamed_from: str | None = None
source_name: str | None = None
source_metadata: dict[str, Any] | None = None
@dataclass(frozen=True)
@ -121,15 +111,11 @@ class BuildReport:
requested_roots: Roots explicitly requested for the run. A full
`build()` uses all declared cells.
provenance: Mapping from final output name to provenance metadata.
owned_cells: Mapping from declared cell name to all final output cell
names it owns, including helper cells emitted while that declared
cell was building.
dependency_graph: Declared-cell dependency graph discovered through
library-mediated reads and explicit recipe hints.
"""
requested_roots: tuple[str, ...]
provenance: Mapping[str, CellProvenance]
owned_cells: Mapping[str, tuple[str, ...]]
dependency_graph: Mapping[str, frozenset[str]]
@ -1467,25 +1453,6 @@ class Library(ILibrary):
return tree, pat
class BuiltLibrary(Library):
"""
Eager library returned by `BuildLibrary.build(output='library')`.
This is a normal materialized `Library` with one additional attribute,
`build_report`, which records how the library was assembled from
declarations, helper emissions, and imported source-backed cells.
"""
def __init__(
self,
mapping: MutableMapping[str, 'Pattern'] | None = None,
*,
build_report: BuildReport | None = None,
) -> None:
super().__init__(mapping=mapping)
self.build_report = build_report
class _CellFactory:
"""
Adapter that turns a plain pattern factory into a deferred recipe factory.
@ -1516,18 +1483,6 @@ class _BuildRecipe:
return self
@dataclass(frozen=True)
class _PatternDeclaration:
""" Declared cell backed by an already-built `Pattern`. """
pattern: 'Pattern'
@dataclass(frozen=True)
class _RecipeDeclaration:
""" Declared cell backed by a deferred recipe. """
recipe: _BuildRecipe
@dataclass(frozen=True)
class _SourceDeclaration:
"""
@ -1596,16 +1551,15 @@ class BuildLibrary(ILibrary):
The builder itself is not a normal readable library during authoring.
Instead, `validate()` and `build()` create a temporary build-session library
that recipes can read from and write helper cells into while dependencies
are resolved. `build()` then freezes the builder on success and returns a
normal library-like object carrying a `build_report`.
are resolved. `build()` then freezes the builder on success and returns the
built library plus a `BuildReport`.
"""
def __init__(self, *, check_on_register: bool = False) -> None:
self.check_on_register = check_on_register
self.cells = BuildCellsView(self)
self.last_build_report: BuildReport | None = None
self._frozen = False
self._declarations: dict[str, _PatternDeclaration | _RecipeDeclaration] = {}
self._declarations: dict[str, 'Pattern | _BuildRecipe'] = {}
self._sources: list[_SourceDeclaration] = []
self._names: set[str] = set()
self._order: list[str] = []
@ -1664,13 +1618,12 @@ class BuildLibrary(ILibrary):
if key in self._names:
raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!')
declaration: _PatternDeclaration | _RecipeDeclaration
if isinstance(value, _BuildRecipe):
declaration = _RecipeDeclaration(value)
declaration = value
else:
if callable(value):
raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.')
declaration = _PatternDeclaration(value)
declaration = value
self._declarations[key] = declaration
self._names.add(key)
@ -1813,6 +1766,8 @@ class BuildLibrary(ILibrary):
Mapping of `{source_name: visible_name}` for imported names that
were renamed while being added.
"""
if self._active_session() is not None:
raise BuildError('BuildLibrary.add_source() is only available while authoring, not during validate() or build().')
self._assert_editable()
view = source if isinstance(source, ILibraryView) else LibraryView(source)
@ -1862,8 +1817,7 @@ class BuildLibrary(ILibrary):
execution path used by `build()`. Any generated library is discarded
after validation completes.
"""
report, _output = self._run_build(names=names, output='overlay', allow_dangling=allow_dangling, persist_output=False)
self.last_build_report = report
_session, report = self._run_build(names=names, allow_dangling=allow_dangling)
return report
def build(
@ -1871,9 +1825,9 @@ class BuildLibrary(ILibrary):
*,
output: Literal['overlay', 'library'] = 'overlay',
allow_dangling: bool = False,
) -> 'BuiltLibrary | ILibrary':
) -> tuple[ILibrary, BuildReport]:
"""
Materialize declarations and return a usable output library.
Materialize declarations and return a usable output library plus report.
Args:
output: `'overlay'` preserves imported source-backed cells where
@ -1882,20 +1836,23 @@ class BuildLibrary(ILibrary):
allow_dangling: If `False`, fail the build when the completed
library still contains dangling references.
"""
if output not in ('overlay', 'library'):
raise ValueError(f'Unknown build output mode: {output!r}')
self._assert_editable()
report, built_output = self._run_build(names=None, output=output, allow_dangling=allow_dangling, persist_output=True)
session, report = self._run_build(names=None, allow_dangling=allow_dangling)
if output == 'library':
built_output = session.to_library()
else:
built_output = session.to_overlay()
self._frozen = True
self.last_build_report = report
return built_output
return built_output, report
def _run_build(
self,
*,
names: Sequence[str] | None,
output: Literal['overlay', 'library'],
allow_dangling: bool,
persist_output: bool,
) -> tuple[BuildReport, BuiltLibrary | ILibrary | None]:
) -> tuple['_BuildSessionLibrary', BuildReport]:
roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys()))
unknown = [name for name in roots if name not in self._names]
if unknown:
@ -1909,19 +1866,11 @@ class BuildLibrary(ILibrary):
session.materialize_many(roots)
if not allow_dangling:
session.child_graph(dangling='error')
if output == 'library':
built_output = session.to_library() if persist_output else None
elif persist_output:
built_output = session.to_overlay()
else:
built_output = None
finally:
_ACTIVE_BUILD_SESSIONS.reset(token)
report = session.build_report(roots)
if built_output is not None:
built_output.build_report = report
return report, built_output
return session, report
class _BuildSessionLibrary(ILibrary):
@ -1935,22 +1884,19 @@ class _BuildSessionLibrary(ILibrary):
"""
def __init__(self, builder: BuildLibrary) -> None:
from .file.gdsii_lazy_core import BuiltOverlayLibrary, _SourceEntry, _SourceLayer # noqa: PLC0415
from .file.gdsii_lazy_core import OverlayLibrary, _SourceEntry, _SourceLayer # noqa: PLC0415
self._builder = builder
self._overlay = BuiltOverlayLibrary()
self._overlay = OverlayLibrary()
self._source_entry_type = _SourceEntry
self._source_layer_type = _SourceLayer
self._states: dict[str, Literal['unbuilt', 'building', 'built']] = {
name: 'unbuilt' for name in builder._declarations
}
self._declared_stack: list[str] = []
self._emission_stack: list[str] = []
self._emission_via_stack: list[emitted_via_t] = []
self._names = set(builder._names)
self._order = list(builder._order)
self._provenance: dict[str, CellProvenance] = {}
self._owned_cells: defaultdict[str, list[str]] = defaultdict(list)
self._dependency_graph: defaultdict[str, set[str]] = defaultdict(set)
self._install_sources()
@ -1965,8 +1911,6 @@ class _BuildSessionLibrary(ILibrary):
)
layer_index = len(self._overlay._layers)
self._overlay._layers.append(layer)
source_info = getattr(spec.library, 'library_info', None)
source_meta = dict(source_info) if isinstance(source_info, dict) else None
for source_name, visible_name in spec.source_to_visible.items():
self._overlay._entries[visible_name] = self._source_entry_type(
@ -1976,15 +1920,12 @@ class _BuildSessionLibrary(ILibrary):
if visible_name not in self._overlay._order:
self._overlay._order.append(visible_name)
self._provenance[visible_name] = CellProvenance(
final_name = visible_name,
requested_name = source_name,
kind = 'source',
owner_declared_name = None,
emitted_via = 'source_import',
build_chain = (),
renamed_from = source_name if visible_name != source_name else None,
source_name = source_name,
source_metadata = source_meta,
)
def __iter__(self) -> Iterator[str]:
@ -2020,14 +1961,6 @@ class _BuildSessionLibrary(ILibrary):
if provenance is not None and provenance.kind == 'source':
raise BuildError(f'Cannot {operation} imported source cell "{key}" during an active build session.')
def _remove_owned_cell(self, owner: str | None, name: str) -> None:
if owner is None or owner not in self._owned_cells:
return
cells = self._owned_cells[owner]
self._owned_cells[owner] = [cell for cell in cells if cell != name]
if not self._owned_cells[owner]:
del self._owned_cells[owner]
def rename(
self,
old_name: str,
@ -2056,16 +1989,8 @@ class _BuildSessionLibrary(ILibrary):
requested_name = provenance.requested_name
self._provenance[new_name] = replace(
provenance,
final_name = new_name,
renamed_from = requested_name if new_name != requested_name else None,
)
owner = provenance.owner_declared_name
if owner is not None and owner in self._owned_cells:
self._owned_cells[owner] = [
new_name if cell_name == old_name else cell_name
for cell_name in self._owned_cells[owner]
]
return self
def __getitem__(self, key: str) -> 'Pattern':
@ -2090,28 +2015,19 @@ class _BuildSessionLibrary(ILibrary):
self._touch_name(key)
kind: Literal['declared', 'helper']
via = self._emission_via_stack[-1] if self._emission_via_stack else 'helper_write'
if current is not None and key == current:
kind = 'declared'
via = 'declaration'
else:
kind = 'helper'
if not self._emission_via_stack:
via = 'helper_write'
self._emission_stack.append(key)
try:
self._record_provenance(
final_name = key,
name = key,
requested_name = key,
kind = kind,
owner_declared_name = current if kind == 'helper' else key,
emitted_via = via,
build_chain = tuple(self._declared_stack),
renamed_from = None,
)
finally:
self._emission_stack.pop()
def __delitem__(self, key: str) -> None:
if key not in self._overlay:
@ -2120,15 +2036,12 @@ class _BuildSessionLibrary(ILibrary):
raise KeyError(key)
self._guard_mutable_output_name(key, operation='delete')
provenance = self._provenance.get(key)
if key in self._overlay:
del self._overlay[key]
self._names.discard(key)
if key in self._order:
self._order.remove(key)
self._provenance.pop(key, None)
if provenance is not None:
self._remove_owned_cell(provenance.owner_declared_name, key)
def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None:
self[key_self] = copy.deepcopy(other[key_other])
@ -2139,11 +2052,7 @@ class _BuildSessionLibrary(ILibrary):
rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns,
mutate_other: bool = False,
) -> dict[str, str]:
self._emission_via_stack.append('tree_merge')
try:
rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other)
finally:
self._emission_via_stack.pop()
current = self._current_declared()
for old_name, new_name in rename_map.items():
@ -2159,32 +2068,24 @@ class _BuildSessionLibrary(ILibrary):
def _record_provenance(
self,
*,
final_name: str,
name: str,
requested_name: str,
kind: Literal['declared', 'helper'],
owner_declared_name: str | None,
emitted_via: emitted_via_t,
build_chain: tuple[str, ...],
renamed_from: str | None,
) -> None:
self._provenance[final_name] = CellProvenance(
final_name = final_name,
self._provenance[name] = CellProvenance(
requested_name = requested_name,
kind = kind,
owner_declared_name = owner_declared_name,
emitted_via = emitted_via,
build_chain = build_chain,
renamed_from = renamed_from,
)
if owner_declared_name is not None and final_name not in self._owned_cells[owner_declared_name]:
self._owned_cells[owner_declared_name].append(final_name)
def _wrap_error(self, name: str, exc: Exception) -> BuildError:
helper = self._emission_stack[-1] if self._emission_stack else None
chain = tuple(self._declared_stack)
msg = [f'Failed while building declared cell "{name}"']
if helper is not None and helper != name:
msg.append(f'while materializing helper/output "{helper}"')
if chain:
msg.append(f'Dependency chain: {" -> ".join(chain)}')
msg.append(f'Cause: {exc}')
@ -2213,14 +2114,14 @@ class _BuildSessionLibrary(ILibrary):
self._states[name] = 'building'
self._declared_stack.append(name)
try:
if isinstance(declaration, _PatternDeclaration):
pattern = declaration.pattern.deepcopy()
else:
for dep in declaration.recipe.explicit_dependencies:
if isinstance(declaration, _BuildRecipe):
for dep in declaration.explicit_dependencies:
self._ensure_named(dep)
pattern = declaration.recipe.func(*declaration.recipe.args, **declaration.recipe.kwargs)
pattern = declaration.func(*declaration.args, **declaration.kwargs)
if not isinstance(pattern, Pattern):
raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern')
else:
pattern = declaration.deepcopy()
if name in self._overlay:
if self._overlay[name] is not pattern:
@ -2261,23 +2162,18 @@ class _BuildSessionLibrary(ILibrary):
for name in self._builder._declarations
if name in self._dependency_graph or name in requested_roots
}
owned_cells = {
name: tuple(cells)
for name, cells in self._owned_cells.items()
}
return BuildReport(
requested_roots = tuple(dict.fromkeys(requested_roots)),
provenance = dict(self._provenance),
owned_cells = owned_cells,
dependency_graph = dependency_graph,
)
def to_overlay(self) -> ILibrary:
return self._overlay
def to_library(self) -> BuiltLibrary:
def to_library(self) -> Library:
mapping = {name: self._overlay[name] for name in self._overlay.source_order()}
return BuiltLibrary(mapping)
return Library(mapping)
class LazyLibrary(ILibrary):

80
masque/test/test_build_library.py
View file

@ -2,11 +2,18 @@ import pytest
from ..builder import Pather
from ..error import BuildError
from ..library import BuildLibrary, BuiltLibrary, Library, cell
from ..library import BuildLibrary, Library, cell
from ..pattern import Pattern
from ..ports import Port
def _owned_by(report, owner: str) -> set[str]:
return {
name for name, prov in report.provenance.items()
if prov.owner_declared_name == owner
}
def test_build_library_traces_declared_dependencies_out_of_order() -> None:
builder = BuildLibrary()
@ -19,12 +26,12 @@ def test_build_library_traces_declared_dependencies_out_of_order() -> None:
builder.cells.parent = cell(make_parent)(builder)
builder["child"] = Pattern(ports={"p": Port((0, 0), 0)})
built = builder.build()
built, report = builder.build()
assert "parent" in built
assert "child" in built
assert built.build_report.dependency_graph["parent"] == frozenset({"child"})
assert built.build_report.provenance["parent"].kind == "declared"
assert report.dependency_graph["parent"] == frozenset({"child"})
assert report.provenance["parent"].kind == "declared"
def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None:
@ -40,17 +47,15 @@ def test_build_library_tracks_helper_provenance_and_tree_merge_renames() -> None
return top
builder.cells.top = cell(make_top)(builder)
built = builder.build()
report = built.build_report
_built, report = builder.build()
helpers = [
prov for prov in report.provenance.values()
if prov.owner_declared_name == "top" and prov.kind == "helper"
]
assert "top" in report.owned_cells["top"]
assert "top" in _owned_by(report, "top")
assert len(helpers) == 2
assert all(prov.emitted_via == "tree_merge" for prov in helpers)
assert any(prov.renamed_from == "_helper" for prov in helpers)
@ -64,10 +69,10 @@ def test_build_library_requires_build_session_for_reads_and_freezes_after_build(
with pytest.raises(BuildError, match="write-only"):
_ = builder.cells.leaf
built = builder.build(output="library")
built, report = builder.build(output="library")
assert isinstance(built, BuiltLibrary)
assert built.build_report.requested_roots == ("leaf",)
assert isinstance(built, Library)
assert report.requested_roots == ("leaf",)
with pytest.raises(BuildError, match="frozen"):
builder["later"] = Pattern()
@ -134,6 +139,14 @@ def test_build_library_validate_rejects_removed_output_argument() -> None:
builder.validate(output="library") # type: ignore[call-arg]
def test_build_library_rejects_unknown_build_output_mode() -> None:
builder = BuildLibrary()
builder["leaf"] = Pattern()
with pytest.raises(ValueError, match="Unknown build output mode"):
builder.build(output="bad") # type: ignore[arg-type]
def test_build_library_allows_helper_writes_via_pather() -> None:
builder = BuildLibrary()
builder["leaf"] = Pattern(ports={"a": Port((0, 0), 0)})
@ -147,9 +160,9 @@ def test_build_library_allows_helper_writes_via_pather() -> None:
return top
builder.cells.top = cell(make_top)(builder)
built = builder.build()
_built, report = builder.build()
helper_prov = built.build_report.provenance["_route"]
helper_prov = report.provenance["_route"]
assert helper_prov.kind == "helper"
assert helper_prov.owner_declared_name == "top"
@ -160,11 +173,23 @@ def test_build_library_preserves_source_cells_and_records_source_provenance() ->
builder.add_source(source)
builder.cells.top = cell(lambda: Pattern())()
built = builder.build()
built, report = builder.build()
assert "src" in built
assert built.build_report.provenance["src"].kind == "source"
assert built.build_report.provenance["src"].emitted_via == "source_import"
assert report.provenance["src"].kind == "source"
def test_build_library_rejects_add_source_during_build() -> None:
builder = BuildLibrary()
def make_top(lib: BuildLibrary) -> Pattern:
lib.add_source(Library({"src": Pattern()}))
return Pattern()
builder.cells.top = cell(make_top)(builder)
with pytest.raises(BuildError, match="add_source"):
builder.build()
def test_build_library_can_rename_imported_source_cells_during_authoring() -> None:
@ -178,12 +203,12 @@ def test_build_library_can_rename_imported_source_cells_during_authoring() -> No
builder.add_source(source)
builder.rename("child", "renamed_child")
built = builder.build()
built, report = builder.build()
assert "renamed_child" in built
assert "child" not in built
assert "renamed_child" in built["parent"].refs
assert built.build_report.provenance["renamed_child"].source_name == "child"
assert report.provenance["renamed_child"].source_name == "child"
def test_build_library_rejects_move_references_for_source_rename() -> None:
@ -202,7 +227,7 @@ def test_build_library_rejects_renaming_declared_cells_during_authoring() -> Non
builder.rename("declared", "renamed_declared")
def test_build_library_helper_rename_updates_provenance_and_owned_cells() -> None:
def test_build_library_helper_rename_updates_provenance_owner() -> None:
builder = BuildLibrary()
def make_top(lib: BuildLibrary) -> Pattern:
@ -213,18 +238,17 @@ def test_build_library_helper_rename_updates_provenance_and_owned_cells() -> Non
return top
builder.cells.top = cell(make_top)(builder)
built = builder.build()
report = built.build_report
built, report = builder.build()
assert "final_helper" in built
assert "_helper" not in built
assert "final_helper" in report.owned_cells["top"]
assert "_helper" not in report.owned_cells["top"]
owned = _owned_by(report, "top")
assert "final_helper" in owned
assert "_helper" not in owned
prov = report.provenance["final_helper"]
assert prov.kind == "helper"
assert prov.requested_name == "_helper"
assert prov.renamed_from == "_helper"
assert prov.final_name == "final_helper"
def test_build_library_helper_delete_removes_provenance_and_ownership() -> None:
@ -236,12 +260,11 @@ def test_build_library_helper_delete_removes_provenance_and_ownership() -> None:
return Pattern()
builder.cells.top = cell(make_top)(builder)
built = builder.build()
report = built.build_report
built, report = builder.build()
assert "_helper" not in built
assert "_helper" not in report.provenance
assert report.owned_cells["top"] == ("top",)
assert _owned_by(report, "top") == {"top"}
def test_build_library_helper_rename_after_auto_rename_preserves_requested_name() -> None:
@ -258,8 +281,7 @@ def test_build_library_helper_rename_after_auto_rename_preserves_requested_name(
return top
builder.cells.top = cell(make_top)(builder)
built = builder.build()
report = built.build_report
built, report = builder.build()
assert "final_helper" in built
prov = report.provenance["final_helper"]