jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[library] More library rework

Browse source
This commit is contained in:
Jan Petykiewicz 2026-07-13 12:24:33 -07:00
commit 0edd735a11
25 changed files with 2357 additions and 870 deletions
Download patch file Download diff file Expand all files Collapse all files

7
masque/library/__init__.py
View file

@ -1,5 +1,6 @@
"""Library classes for managing name-to-pattern mappings."""
from .utils import (
INameView as INameView,
SINGLE_USE_PREFIX as SINGLE_USE_PREFIX,
Tree as Tree,
TreeView as TreeView,
@ -12,6 +13,10 @@ from .base import (
ILibrary as ILibrary,
ILibraryView as ILibraryView,
)
from .capabilities import (
IBorrowing as IBorrowing,
IMaterializable as IMaterializable,
)
from .mapping import (
Library as Library,
LibraryView as LibraryView,
@ -21,7 +26,7 @@ from .overlay import (
PortsLibraryView as PortsLibraryView,
)
from .build import (
BuildLibrary as BuildLibrary,
LibraryBuilder as LibraryBuilder,
BuildReport as BuildReport,
CellProvenance as CellProvenance,
cell as cell,

325
masque/library/base.py
View file

@ -8,8 +8,6 @@ from graphlib import CycleError, TopologicalSorter
from pprint import pformat
from typing import TYPE_CHECKING, Self, cast
import copy
import logging
import re
import numpy
@ -18,7 +16,18 @@ from ..error import LibraryError, PatternError
from ..pattern import Pattern, map_layers
from ..shapes import Polygon, Shape
from ..utils import apply_transforms, layer_t
from .utils import SINGLE_USE_PREFIX, TreeView, b64suffix, dangling_mode_t, _rename_patterns, visitor_function_t
from .utils import (
SINGLE_USE_PREFIX,
INameView,
TreeView,
b64suffix,
dangling_mode_t,
_plan_source_names,
_rename_patterns,
_source_rename_map,
_validate_dangling_mode,
visitor_function_t,
)
if TYPE_CHECKING:
from collections.abc import Iterator
@ -28,10 +37,7 @@ if TYPE_CHECKING:
from ..label import Label
logger = logging.getLogger(__name__)
class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
class ILibraryView(Mapping[str, 'Pattern'], INameView, metaclass=ABCMeta):
"""
Interface for a read-only library.
@ -78,7 +84,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
def dangling_refs(
self,
tops: str | Sequence[str] | None = None,
) -> set[str | None]:
) -> set[str]:
"""
Get the set of all pattern names not present in the library but referenced
by `tops`, recursively traversing any refs.
@ -101,8 +107,8 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
def referenced_patterns(
self,
tops: str | Sequence[str] | None = None,
skip: set[str | None] | None = None,
) -> set[str | None]:
skip: set[str] | None = None,
) -> set[str]:
"""
Get the set of all pattern names referenced by `tops`. Recursively traverses into any refs.
@ -116,29 +122,78 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
Set of all referenced pattern names
"""
graph = self.child_graph(dangling='include')
return self._referenced_patterns_from_graph(graph, tops=tops, skip=skip)
def _referenced_patterns_from_graph(
self,
graph: Mapping[str, set[str]],
*,
tops: str | Sequence[str] | None,
skip: set[str] | None = None,
) -> set[str]:
"""Traverse named references in an already-computed child graph."""
existing = set(self.keys())
if tops is None:
tops = tuple(self.keys())
roots = tuple(existing)
elif isinstance(tops, str):
roots = (tops,)
else:
roots = tuple(tops)
for root in roots:
if root not in existing:
raise KeyError(root)
if skip is None:
skip = {None}
skip = set()
root_set = set(roots)
skip |= root_set
if isinstance(tops, str):
tops = (tops,)
tops = set(tops)
skip |= tops # don't re-visit tops
targets: set[str] = set()
stack = list(root_set)
while stack:
name = stack.pop()
children = graph.get(name, set())
targets |= children
for child in children - skip:
skip.add(child)
if child in existing:
stack.append(child)
return targets
# Get referenced patterns for all tops
targets = set()
for top in set(tops):
targets |= self[top].referenced_patterns()
def _referenced_patterns_by_lookup(
self,
tops: str | Sequence[str] | None = None,
skip: set[str] | None = None,
) -> set[str]:
"""Traverse refs by loading only reachable patterns."""
if tops is None:
roots = tuple(self.keys())
elif isinstance(tops, str):
roots = (tops,)
else:
roots = tuple(tops)
# Perform recursive lookups, but only once for each name
for target in targets - skip:
assert target is not None
skip.add(target)
if target in self:
targets |= self.referenced_patterns(target, skip=skip)
if skip is None:
skip = set()
root_set = set(roots)
skip |= root_set
targets: set[str] = set()
stack = list(root_set)
while stack:
name = stack.pop()
children = {
target
for target in self[name].referenced_patterns()
if target is not None
}
targets |= children
for child in children - skip:
skip.add(child)
if child in self:
stack.append(child)
return targets
def subtree(
@ -154,19 +209,22 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
tops: Name(s) of patterns to keep
Returns:
A `LibraryView` containing only `tops` and the patterns they reference.
A borrowed view containing only `tops` and the patterns they reference.
The returned view snapshots membership and hierarchy, but borrows patterns
and source capabilities from this library. Keep this library open and do
not structurally mutate it for the lifetime of the returned view.
"""
if isinstance(tops, str):
tops = (tops,)
keep = cast('set[str]', self.referenced_patterns(tops) - {None})
graph = self.child_graph(dangling='include')
keep = self._referenced_patterns_from_graph(graph, tops=tops)
keep &= set(self.keys())
keep |= set(tops)
from .mapping import LibraryView # noqa: PLC0415
filtered = {kk: vv for kk, vv in self.items() if kk in keep}
new = LibraryView(filtered)
return new
from .mapping import _SubtreeLibraryView # noqa: PLC0415
return _SubtreeLibraryView(self, names=keep, child_graph=graph)
def polygonize(
self,
@ -242,6 +300,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
def flatten_single(name: str) -> None:
flattened[name] = None
pat = self[name].deepcopy()
pat.prune_refs()
refs_by_target = tuple((target, tuple(refs)) for target, refs in pat.refs.items())
for target, refs in refs_by_target:
@ -281,59 +340,6 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
assert None not in flattened.values()
return cast('dict[str, Pattern]', flattened)
def get_name(
self,
name: str = SINGLE_USE_PREFIX * 2,
sanitize: bool = True,
max_length: int = 32,
quiet: bool | None = None,
) -> str:
"""
Find a unique name for the pattern.
This function may be overridden in a subclass or monkey-patched to fit the caller's requirements.
Args:
name: Preferred name for the pattern. Default is `SINGLE_USE_PREFIX * 2`.
sanitize: Allows only alphanumeric charaters and _?$. Replaces invalid characters with underscores.
max_length: Names longer than this will be truncated.
quiet: If `True`, suppress log messages. Default `None` suppresses messages only if
the name starts with `SINGLE_USE_PREFIX`.
Returns:
Name, unique within this library.
"""
if quiet is None:
quiet = name.startswith(SINGLE_USE_PREFIX)
if sanitize:
# Remove invalid characters
sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name)
else:
sanitized_name = name
suffixed_name = sanitized_name
if sanitized_name in self:
ii = sum(1 for nn in self.keys() if nn.startswith(sanitized_name))
else:
ii = 0
while suffixed_name in self or suffixed_name == '':
suffixed_name = sanitized_name + b64suffix(ii)
ii += 1
if len(suffixed_name) > max_length:
if name == '':
raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}')
cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True)
else:
cropped_name = suffixed_name
if not quiet:
logger.info(f'Requested name "{name}" changed to "{cropped_name}"')
return cropped_name
def tops(self) -> list[str]:
"""
Return the list of all patterns that are not referenced by any other pattern in the library.
@ -341,13 +347,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
A list of pattern names in which no pattern is referenced by any other pattern.
"""
graph = self.child_graph(dangling='ignore')
names = set(self.keys())
not_toplevel: set[str | None] = set()
for name in names:
not_toplevel |= set(self[name].refs.keys())
toplevel = list(names - not_toplevel)
return toplevel
referenced = set().union(*graph.values()) if graph else set()
return list(names - referenced)
def top(self) -> str:
"""
@ -412,7 +415,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
tuple of all parent-and-higher pattern names. Top pattern name may be
`None` if not provided in first call to .dfs()
`transform`: numpy.ndarray containing cumulative
[x_offset, y_offset, rotation (rad), mirror_x (0 or 1)]
[x_offset, y_offset, rotation (rad), mirror_x (0 or 1), scale]
for the instance being visited
`memo`: Arbitrary dict (not altered except by `visit_before()` and `visit_after()`)
@ -450,13 +453,13 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
if visit_before is not None:
pattern = visit_before(pattern, hierarchy=hierarchy, memo=memo, transform=transform)
for target in pattern.refs:
if target is None:
for target, refs in pattern.refs.items():
if target is None or not refs:
continue
if target in hierarchy:
raise LibraryError(f'.dfs() called on pattern with circular reference to "{target}"')
for ref in pattern.refs[target]:
for ref in refs:
ref_transforms: list[bool] | NDArray[numpy.float64]
if transform is not False:
ref_transforms = apply_transforms(transform, ref.as_transforms())
@ -509,6 +512,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
Mapping from pattern name to a set of all pattern names it references.
"""
_validate_dangling_mode(dangling)
graph, dangling_refs = self._raw_child_graph()
if dangling == 'error':
if dangling_refs:
@ -538,13 +542,18 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
Mapping from pattern name to a set of all patterns which reference it.
"""
child_graph, dangling_refs = self._raw_child_graph()
_validate_dangling_mode(dangling)
graph_mode: dangling_mode_t = 'ignore' if dangling == 'ignore' else 'include'
child_graph = self.child_graph(dangling=graph_mode)
existing = set(self.keys())
dangling_refs = set(child_graph) - existing
if dangling == 'error' and dangling_refs:
raise self._dangling_refs_error(dangling_refs, 'building parent graph')
existing = set(child_graph)
igraph: dict[str, set[str]] = {name: set() for name in existing}
for parent, children in child_graph.items():
graph_names = set(child_graph) if dangling == 'include' else existing
igraph: dict[str, set[str]] = {name: set() for name in graph_names}
for parent in existing:
children = child_graph.get(parent, set())
for child in children:
if child in existing:
igraph[child].add(parent)
@ -566,6 +575,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Return:
Topologically sorted list of pattern names.
"""
_validate_dangling_mode(dangling)
try:
return cast('list[str]', list(TopologicalSorter(self.child_graph(dangling=dangling)).static_order()))
except CycleError as exc:
@ -596,9 +606,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
Mapping of {parent_name: transform_list}, where transform_list
is an Nx4 ndarray with rows
`(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`.
is an Nx5 ndarray with rows
`(x_offset, y_offset, rotation_ccw_rad, mirror_across_x, scale)`.
"""
_validate_dangling_mode(dangling)
instances = defaultdict(list)
if parent_graph is None:
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
@ -648,9 +659,10 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
Mapping of `{hierarchy: transform_list}`, where `hierarchy` is a tuple of the form
`(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx4 ndarray
with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`.
`(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx5 ndarray
with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x, scale)`.
"""
_validate_dangling_mode(dangling)
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
if order is None:
order = self.child_order(dangling=graph_mode)
@ -700,11 +712,15 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta):
class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
class ILibrary(ILibraryView, metaclass=ABCMeta):
"""
Interface for a writeable library.
Interface for an insertable and deletable library.
A library is a mapping from unique names (str) to collections of geometry (`Pattern`).
Assignment inserts new names but does not replace existing ones. `ILibrary`
intentionally does not implement `MutableMapping` or its generic mutation
helpers; use the library-specific insertion, deletion, rename, and add
operations instead.
"""
# inherited abstract functions
#def __getitem__(self, key: str) -> 'Pattern':
@ -853,27 +869,35 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
def add(
self,
other: Mapping[str, Pattern],
rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns,
rename_theirs: Callable[[INameView, str], str] = _rename_patterns,
mutate_other: bool = False,
) -> dict[str, str]:
"""
Add items from another library into this one.
If any name in `other` is already present in `self`, `rename_theirs(self, name)` is called
to pick a new name for the newly-added pattern. If the new name still conflicts with a name
in `self` a `LibraryError` is raised. All references to the original name (within `other)`
are updated to the new name.
If any name in `other` is already present in the prospective destination,
`rename_theirs(view, name)` is called to pick a new name for the newly-added pattern.
The name-only `INameView` includes destination names and names reserved earlier in
the same addition plan.
If the new name still conflicts, a `LibraryError` is raised. All references to the
original name within `other` are updated to the new name.
If `mutate_other=False` (default), all changes are made to a deepcopy of `other`.
Name resolution, copying, and reference remapping complete before cells are inserted,
so failures during those phases do not partially modify either library. Custom
implementations of `_merge()` may still have partial effects if they raise while cells
are being committed.
By default, `rename_theirs` makes no changes to the name (causing a `LibraryError`) unless the
name starts with `SINGLE_USE_PREFIX`. Prefixed names are truncated to before their first
non-prefix '$' and then passed to `self.get_name()` to create a new unique name.
Args:
other: The library to insert keys from.
rename_theirs: Called as rename_theirs(self, name) for each duplicate name
encountered in `other`. Should return the new name for the pattern in
`other`. See above for default behavior.
rename_theirs: Called as `rename_theirs(view, name)` for each duplicate name
encountered in `other`, where `view` supports membership, iteration,
`len()`, and `get_name()`, but not pattern lookup. Should return the new
name for the pattern in `other`. See above for default behavior.
mutate_other: If `True`, modify the original library and its contained patterns
(e.g. when renaming patterns and updating refs). Otherwise, operate on a deepcopy
(default).
@ -888,42 +912,34 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
from ..pattern import map_targets # noqa: PLC0415
from .mapping import Library # noqa: PLC0415
duplicates = set(self.keys()) & set(other.keys())
if not duplicates:
if mutate_other:
temp = other
else:
temp = Library(copy.deepcopy(dict(other)))
for key in temp:
self._merge(key, temp, key)
return {}
source_order = tuple(other.keys())
source_to_visible = _plan_source_names(
self,
source_order,
rename_theirs = rename_theirs,
rename_when = 'conflict',
)
rename_map = _source_rename_map(source_to_visible)
if mutate_other:
if isinstance(other, Library):
temp = other
else:
temp = Library(dict(other))
temp = Library({name: other[name] for name in source_order})
else:
temp = Library(copy.deepcopy(dict(other)))
rename_map = {}
for old_name in temp:
if old_name in self:
new_name = rename_theirs(self, old_name)
if new_name in self:
raise LibraryError(f'Unresolved duplicate key encountered in library merge: {old_name} -> {new_name}')
rename_map[old_name] = new_name
else:
new_name = old_name
temp = Library(copy.deepcopy({name: other[name] for name in source_order}))
self._merge(new_name, temp, old_name)
if rename_map:
target_map = cast('dict[str | None, str | None]', rename_map)
remapped_refs = {
source_name: map_targets(temp[source_name].refs, lambda target: target_map.get(target, target))
for source_name in source_order
}
for source_name, refs in remapped_refs.items():
temp[source_name].refs = refs
# Update references in the newly-added cells
for old_name in temp:
new_name = rename_map.get(old_name, old_name)
pat = self[new_name]
pat.refs = map_targets(pat.refs, lambda tt: cast('dict[str | None, str | None]', rename_map).get(tt, tt))
for source_name in source_order:
self._merge(source_to_visible[source_name], temp, source_name)
return rename_map
@ -934,21 +950,18 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
based on `add()`'s default `rename_theirs` argument).
Raises:
LibraryError if there is more than one topcell in `other`.
LibraryError if there is not exactly one topcell in `other`.
"""
from .mapping import LibraryView # noqa: PLC0415
if len(other) == 1:
name = next(iter(other))
else:
if not isinstance(other, ILibraryView):
other = LibraryView(other)
if not isinstance(other, ILibraryView):
other = LibraryView(other)
tops = other.tops()
if len(tops) > 1:
raise LibraryError('Received a library containing multiple topcells!')
tops = other.tops()
if len(tops) != 1:
raise LibraryError(f'Received a library without exactly one topcell: {pformat(tops)}')
name = tops[0]
name = tops[0]
rename_map = self.add(other)
new_name = rename_map.get(name, name)
@ -960,7 +973,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
of just the pattern's name.
Raises:
LibraryError if there is more than one topcell in `other`.
LibraryError if there is not exactly one topcell in `other`.
"""
new_name = self << other
return self.abstract(new_name)
@ -1178,7 +1191,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
if isinstance(tops, str):
tops = (tops,)
keep = cast('set[str]', self.referenced_patterns(tops) - {None})
keep = self.referenced_patterns(tops)
keep |= set(tops)
new = type(self)()
@ -1201,6 +1214,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta):
Returns:
A set containing the names of all deleted patterns
"""
_validate_dangling_mode(dangling)
parent_graph = self.parent_graph(dangling=dangling)
empty = {name for name, pat in self.items() if pat.is_empty()}
trimmed = set()
@ -1257,3 +1271,6 @@ class AbstractView(Mapping[str, Abstract]):
def __len__(self) -> int:
return self.library.__len__()
def __contains__(self, key: object) -> bool:
return key in self.library

334
masque/library/build.py
View file

@ -2,32 +2,25 @@
from __future__ import annotations
from collections import defaultdict
from contextvars import ContextVar
from dataclasses import dataclass, replace
from functools import wraps
from pprint import pformat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Literal, Self, cast
import copy
from ..error import BuildError, LibraryError
from .base import ILibrary, ILibraryView
from .utils import TreeView, dangling_mode_t, _plan_source_names, _rename_patterns, _source_rename_map
from .capabilities import IMaterializable
from .utils import INameView, TreeView, dangling_mode_t, _plan_source_names, _rename_patterns, _source_rename_map
from .mapping import Library, LibraryView
from .overlay import OverlayLibrary
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, KeysView, Mapping, Sequence
from ..abstract import Abstract
from ..pattern import Pattern
_ACTIVE_BUILD_SESSIONS: ContextVar[dict[int, _BuildSessionLibrary] | None] = ContextVar(
'masque_active_build_sessions',
default=None,
)
@dataclass(frozen=True)
class CellProvenance:
"""
@ -55,7 +48,7 @@ class CellProvenance:
@dataclass(frozen=True)
class BuildReport:
"""
Immutable summary of one `BuildLibrary.validate()` or `.build()` run.
Immutable summary of one `LibraryBuilder.validate()` or `.build()` run.
The report is designed to answer two questions after a build completes:
which declared cells depended on which other declared cells, and where each
@ -72,6 +65,15 @@ class BuildReport:
provenance: Mapping[str, CellProvenance]
dependency_graph: Mapping[str, frozenset[str]]
def __post_init__(self) -> None:
object.__setattr__(self, 'requested_roots', tuple(self.requested_roots))
object.__setattr__(self, 'provenance', MappingProxyType(dict(self.provenance)))
object.__setattr__(
self,
'dependency_graph',
MappingProxyType({name: frozenset(deps) for name, deps in self.dependency_graph.items()}),
)
@dataclass
class _BuildRecipe:
""" Captured deferred call to a pattern factory. """
@ -98,42 +100,54 @@ def cell(func: Callable[..., Pattern]) -> Callable[..., _BuildRecipe]:
return wrapper
class _LibraryPlaceholder:
"""Identity token replaced by this builder's active build-session library."""
__slots__ = ('_builder',)
def __init__(self, builder: LibraryBuilder) -> None:
self._builder = builder
def __repr__(self) -> str:
return '<LibraryBuilder.library>'
class BuildCellsView:
"""
Attribute-based declaration namespace for `BuildLibrary`.
Attribute-based declaration namespace for `LibraryBuilder`.
This is the ergonomic authoring surface exposed as `builder.cells`. It is
intentionally write-focused: attribute assignment and deletion register
declarations, while attribute reads fail with guidance to build first and
use the returned library.
"""
__slots__ = ('_library',)
def __init__(self, library: BuildLibrary) -> None:
def __init__(self, library: LibraryBuilder) -> None:
object.__setattr__(self, '_library', library)
def __getattr__(self, name: str) -> Pattern:
raise BuildError(
f'BuildLibrary.cells.{name} is write-only during authoring. '
f'LibraryBuilder.cells.{name} is write-only during authoring. '
'Call build() and index the returned library instead.'
)
def __setattr__(self, name: str, value: Pattern | _BuildRecipe) -> None:
if name.startswith('_'):
if name == '_library':
object.__setattr__(self, name, value)
return
self._library[name] = value
def __delattr__(self, name: str) -> None:
if name.startswith('_'):
if name == '_library':
raise AttributeError(name)
del self._library[name]
class BuildLibrary(ILibrary):
class LibraryBuilder(INameView):
"""
Two-phase declaration surface for mixed imported/generated libraries.
A `BuildLibrary` collects three kinds of inputs:
A `LibraryBuilder` collects three kinds of inputs:
- direct declared `Pattern` objects
- deferred recipes created with `cell(...)`
- imported source-backed library views added with `add_source(...)`
@ -147,112 +161,82 @@ class BuildLibrary(ILibrary):
def __init__(self) -> None:
self.cells = BuildCellsView(self)
self._library_placeholder = _LibraryPlaceholder(self)
self._frozen = False
self._building = False
self._declarations: dict[str, Pattern | _BuildRecipe] = {}
self._sources: list[tuple[ILibraryView, dict[str, str]]] = []
self._names: dict[str, None] = {}
def _active_session(self) -> _BuildSessionLibrary | None:
sessions = _ACTIVE_BUILD_SESSIONS.get()
if sessions is None:
return None
return sessions.get(id(self))
def _require_active_session(self, operation: str) -> _BuildSessionLibrary:
session = self._active_session()
if session is None:
raise BuildError(
f'BuildLibrary.{operation}() is only available while validate() or build() is running. '
'Use the built output library for reads.'
)
return session
@property
def library(self) -> _LibraryPlaceholder:
"""Read-only placeholder replaced by the active library in direct recipe arguments."""
return self._library_placeholder
def _assert_editable(self) -> None:
if self._frozen:
raise BuildError('This BuildLibrary has already been built successfully and is now frozen.')
raise BuildError('This LibraryBuilder has already been built successfully and is now frozen.')
if self._building:
raise BuildError('Cannot modify a LibraryBuilder while validate() or build() is running.')
def __iter__(self) -> Iterator[str]:
session = self._active_session()
if session is not None:
return iter(session)
return iter(self._names)
def __len__(self) -> int:
session = self._active_session()
if session is not None:
return len(session)
return len(self._names)
def __contains__(self, key: object) -> bool:
session = self._active_session()
if session is not None:
return key in session
return key in self._names
def __getitem__(self, key: str) -> Pattern:
return self._require_active_session('__getitem__')[key]
def keys(self) -> KeysView[str]:
return self._names.keys()
def __setitem__(
self,
key: str,
value: Pattern | _BuildRecipe,
) -> None:
session = self._active_session()
if session is not None:
session[key] = value
return
self._assert_editable()
if key in self._names:
raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!')
if isinstance(value, _BuildRecipe):
placeholders = (
arg for arg in (*value.args, *value.kwargs.values())
if isinstance(arg, _LibraryPlaceholder)
)
if any(placeholder is not self.library for placeholder in placeholders):
raise BuildError('A recipe cannot use another LibraryBuilder.library placeholder.')
declaration = value
else:
if callable(value):
raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.')
raise TypeError('LibraryBuilder recipes must be wrapped with cell(fn)(...) or @cell.')
declaration = value
self._declarations[key] = declaration
self._names[key] = None
def __delitem__(self, key: str) -> None:
session = self._active_session()
if session is not None:
del session[key]
return
self._assert_editable()
if key not in self._declarations:
raise KeyError(key)
del self._declarations[key]
del self._names[key]
def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None:
session = self._active_session()
if session is not None:
session._merge(key_self, other, key_other)
return
self[key_self] = copy.deepcopy(other[key_other])
def add(
self,
other: Mapping[str, Pattern],
rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns,
rename_theirs: Callable[[INameView, str], str] = _rename_patterns,
mutate_other: bool = False,
) -> dict[str, str]:
from ..pattern import map_targets # noqa: PLC0415
session = self._active_session()
if session is not None:
return session.add(other, rename_theirs=rename_theirs, mutate_other=mutate_other)
self._assert_editable()
source_backed = isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView)
if source_backed:
materializable = isinstance(other, IMaterializable)
if materializable:
if mutate_other:
raise BuildError('BuildLibrary.add(..., mutate_other=True) is not supported for source-backed inputs.')
raise BuildError('LibraryBuilder.add(..., mutate_other=True) is not supported for source-backed inputs.')
return self.add_source(
other,
rename_theirs = rename_theirs,
@ -263,7 +247,6 @@ class BuildLibrary(ILibrary):
source_to_visible = _plan_source_names(
self,
source_order,
self._names,
rename_theirs = rename_theirs,
rename_when = 'conflict',
)
@ -272,95 +255,39 @@ class BuildLibrary(ILibrary):
if mutate_other:
temp = other
else:
temp = Library(copy.deepcopy(dict(other)))
temp = Library(copy.deepcopy({name: other[name] for name in source_order}))
if rename_map:
target_map = cast('dict[str | None, str | None]', rename_map)
remapped_refs = {
source_name: map_targets(temp[source_name].refs, lambda target: target_map.get(target, target))
for source_name in source_order
}
for source_name, refs in remapped_refs.items():
temp[source_name].refs = refs
for source_name in source_order:
visible_name = source_to_visible[source_name]
pattern = temp[source_name]
if rename_map:
pattern.refs = map_targets(
pattern.refs,
lambda target: cast('dict[str | None, str | None]', rename_map).get(target, target),
)
self[visible_name] = pattern
return rename_map
def __lshift__(self, other: TreeView) -> str:
session = self._active_session()
if session is not None:
return session << other
self._assert_editable()
if len(other) == 1:
name = next(iter(other))
elif isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView):
source_order = other.source_order()
child_graph = other.child_graph(dangling='include')
referenced = set().union(*child_graph.values()) if child_graph else set()
tops = [candidate for candidate in source_order if candidate not in referenced]
if len(tops) != 1:
raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}')
name = tops[0]
else:
return super().__lshift__(other)
rename_map = self.add(other)
return rename_map.get(name, name)
def __le__(self, other: Mapping[str, Pattern]) -> Abstract:
if self._active_session() is not None:
return super().__le__(other)
raise BuildError('BuildLibrary.__le__() is only available while validate() or build() is running.')
def rename(
self,
old_name: str,
new_name: str,
move_references: bool = False,
) -> Self:
"""
Rename a helper cell during an active build session.
During authoring, declared cells must be registered under their
intended final names and imported source cells must be renamed through
`add_source(...)`.
"""
session = self._active_session()
if session is not None:
session.rename(old_name, new_name, move_references=move_references)
return self
self._assert_editable()
if old_name == new_name:
return self
if old_name in self._declarations:
raise BuildError(
f'Cannot rename declared build cell "{old_name}" during authoring. '
'Register it under the intended final name instead.'
)
if old_name not in self._names:
raise LibraryError(f'"{old_name}" does not exist in the builder.')
raise BuildError(
f'Cannot rename imported source cell "{old_name}" during authoring. '
'Choose visible source names with add_source(..., rename_theirs=..., rename_when=...).'
)
def abstract(self, name: str) -> Abstract:
return self._require_active_session('abstract').abstract(name)
def resolve(
self,
other: Abstract | str | Pattern | TreeView,
append: bool = False,
) -> Abstract | Pattern:
return self._require_active_session('resolve').resolve(other, append=append)
view = other if isinstance(other, ILibraryView) else LibraryView(other)
tops = view.tops()
if len(tops) != 1:
raise LibraryError(f'Received a library without exactly one topcell: {tops}')
top = tops[0]
rename_map = self.add(view)
return rename_map.get(top, top)
def add_source(
self,
source: Mapping[str, Pattern] | ILibraryView,
*,
rename_theirs: Callable[[ILibraryView, str], str] | None = None,
rename_theirs: Callable[[INameView, str], str] | None = None,
rename_when: Literal['conflict', 'always'] = 'conflict',
) -> dict[str, str]:
"""
@ -369,13 +296,19 @@ class BuildLibrary(ILibrary):
The source is not materialized immediately. Its names are scanned once
to reserve visible builder names, then the source is read again when a
build session starts. The source's cell membership must not be
structurally mutated between `add_source()` and `build()`/`validate()`.
closed or structurally mutated between `add_source()` and
`build()`/`validate()`.
Source cells may be renamed on entry to avoid collisions with existing
declarations or other imported sources.
Sources are borrowed rather than owned. An output built with
`output='overlay'` remains source-backed, so its sources must also stay
open and unchanged for the lifetime of that output.
Args:
rename_theirs: Function used to choose visible names for imported
source cells.
source cells. Its `INameView` argument contains existing and
previously reserved names, but does not support pattern lookup.
rename_when: If `'conflict'`, only conflicting names are renamed.
If `'always'`, every imported source name is passed through
`rename_theirs`.
@ -384,8 +317,6 @@ class BuildLibrary(ILibrary):
Mapping of `{source_name: visible_name}` for imported names that
were renamed while being added.
"""
if self._active_session() is not None:
raise BuildError('BuildLibrary.add_source() is only available while authoring, not during validate() or build().')
self._assert_editable()
view = source if isinstance(source, ILibraryView) else LibraryView(source)
@ -393,7 +324,6 @@ class BuildLibrary(ILibrary):
source_to_visible = _plan_source_names(
self,
source_order,
self._names,
rename_theirs = rename_theirs,
rename_when = rename_when,
)
@ -406,7 +336,7 @@ class BuildLibrary(ILibrary):
def validate(
self,
names: Sequence[str] | None = None,
names: str | Sequence[str] | None = None,
*,
allow_dangling: bool = False,
) -> BuildReport:
@ -416,6 +346,13 @@ class BuildLibrary(ILibrary):
This is a dry run over the same dependency resolution and recipe
execution path used by `build()`. Any generated library is discarded
after validation completes.
Args:
names: Declared root name or names to validate. `None` validates
every declaration. Duplicate names are ignored after their
first occurrence.
allow_dangling: If `False`, fail validation when the generated
library contains dangling references.
"""
_session, report = self._run_build(names=names, allow_dangling=allow_dangling)
return report
@ -431,13 +368,19 @@ class BuildLibrary(ILibrary):
Args:
output: `'overlay'` preserves imported source-backed cells where
possible, while `'library'` eagerly materializes the full
result.
possible and continues borrowing their sources. `'library'`
eagerly materializes the full result and no longer needs the
sources afterward.
allow_dangling: If `False`, fail the build when the completed
library still contains dangling references.
"""
if output not in ('overlay', 'library'):
raise ValueError(f'Unknown build output mode: {output!r}')
if self._building:
raise BuildError(
'Cannot call build() or validate() recursively on a LibraryBuilder '
'from one of its own recipes.'
)
self._assert_editable()
session, report = self._run_build(names=None, allow_dangling=allow_dangling)
if output == 'library':
@ -450,24 +393,37 @@ class BuildLibrary(ILibrary):
def _run_build(
self,
*,
names: Sequence[str] | None,
names: str | Sequence[str] | None,
allow_dangling: bool,
) -> tuple[_BuildSessionLibrary, BuildReport]:
roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys()))
if self._building:
raise BuildError(
'Cannot call build() or validate() recursively on a LibraryBuilder '
'from one of its own recipes.'
)
if names is None:
requested_names = tuple(self._declarations)
elif isinstance(names, str):
requested_names = (names,)
else:
requested_names = tuple(names)
if any(not isinstance(name, str) for name in requested_names):
raise TypeError('Build roots must be strings.')
roots = tuple(dict.fromkeys(requested_names))
unknown = [name for name in roots if name not in self._names]
if unknown:
raise BuildError(f'Unknown build roots requested: {unknown}')
session = _BuildSessionLibrary(self)
sessions = dict(_ACTIVE_BUILD_SESSIONS.get() or {})
sessions[id(self)] = session
token = _ACTIVE_BUILD_SESSIONS.set(sessions)
self._building = True
try:
session = _BuildSessionLibrary(self)
session.materialize_many(roots)
if not allow_dangling:
session.child_graph(dangling='error')
finally:
_ACTIVE_BUILD_SESSIONS.reset(token)
self._building = False
report = session.build_report(roots)
return session, report
@ -475,7 +431,7 @@ class BuildLibrary(ILibrary):
class _BuildSessionLibrary(ILibrary):
"""
Internal overlay-backed library used while a `BuildLibrary` is executing.
Internal overlay-backed library used while a `LibraryBuilder` is executing.
This object provides the mutable-library surface that recipes expect while
also tracking declared-cell dependencies, helper-cell provenance, and
@ -483,7 +439,7 @@ class _BuildSessionLibrary(ILibrary):
build run.
"""
def __init__(self, builder: BuildLibrary) -> None:
def __init__(self, builder: LibraryBuilder) -> None:
self._builder = builder
self._overlay = OverlayLibrary()
self._built: set[str] = set()
@ -512,7 +468,7 @@ class _BuildSessionLibrary(ILibrary):
'Do not structurally mutate source libraries between add_source() and build()/validate().'
)
def rename_source(_lib: ILibraryView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str:
def rename_source(_lib: INameView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str:
return mapping[name]
self._overlay.add_source(
@ -586,6 +542,8 @@ class _BuildSessionLibrary(ILibrary):
return self
def __getitem__(self, key: str) -> Pattern:
if key not in self._names:
raise KeyError(key)
if key in self._builder._declarations:
self._record_dependency(key)
self._ensure_declared(key)
@ -637,7 +595,7 @@ class _BuildSessionLibrary(ILibrary):
def add(
self,
other: Mapping[str, Pattern],
rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns,
rename_theirs: Callable[[INameView, str], str] = _rename_patterns,
mutate_other: bool = False,
) -> dict[str, str]:
rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other)
@ -684,7 +642,15 @@ class _BuildSessionLibrary(ILibrary):
if isinstance(declaration, _BuildRecipe):
for dep in declaration.explicit_dependencies:
self._ensure_named(dep)
pattern = declaration.func(*declaration.args, **declaration.kwargs)
args = tuple(
self if arg is self._builder.library else arg
for arg in declaration.args
)
kwargs = {
key: self if value is self._builder.library else value
for key, value in declaration.kwargs.items()
}
pattern = declaration.func(*args, **kwargs)
if not isinstance(pattern, Pattern):
raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') # noqa: TRY301
else:
@ -722,6 +688,42 @@ class _BuildSessionLibrary(ILibrary):
) -> dict[str, set[str]]:
return self._overlay.parent_graph(dangling=dangling)
def subtree(
self,
tops: str | Sequence[str],
) -> Self:
if isinstance(tops, str):
tops = (tops,)
keep = self._referenced_patterns_by_lookup(tops=tops)
keep &= set(self)
keep |= set(tops)
order = tuple(name for name in self._names if name in keep)
patterns = {name: self[name] for name in order}
new = object.__new__(type(self))
new._builder = self._builder
new._overlay = OverlayLibrary()
for name in order:
new._overlay[name] = patterns[name]
new._built = {
name for name in order
if name in self._builder._declarations
}
new._declared_stack = []
new._names = dict.fromkeys(order)
new._provenance = {
name: self._provenance[name]
for name in order
if name in self._provenance
}
new._dependency_graph = defaultdict(set, {
name: set(dependencies) & keep
for name, dependencies in self._dependency_graph.items()
if name in keep
})
return new
def build_report(self, requested_roots: Sequence[str]) -> BuildReport:
dependency_graph = {
name: frozenset(self._dependency_graph.get(name, set()))

42
masque/library/capabilities.py Normal file
View file

@ -0,0 +1,42 @@
"""Optional capabilities implemented by lazy and borrowing libraries."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Sequence
from ..pattern import Pattern
from .base import ILibraryView
from .mapping import LibraryView
class IMaterializable(ABC):
"""Capability for libraries which support explicit pattern materialization."""
@abstractmethod
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
"""Materialize one pattern, optionally retaining it in the library's cache."""
def materialize_many(
self,
names: Sequence[str],
*,
persist: bool = True,
) -> LibraryView:
"""Materialize a de-duplicated sequence into a plain read-only view."""
from .mapping import LibraryView # noqa: PLC0415
return LibraryView({
name: self.materialize(name, persist=persist)
for name in dict.fromkeys(names)
})
class IBorrowing(ABC):
"""Capability for library views which directly borrow other libraries."""
@abstractmethod
def borrowed_sources(self) -> tuple[ILibraryView, ...]:
"""Return the source views directly borrowed by this library."""

20
masque/library/lazy.py
View file

@ -7,9 +7,10 @@ import logging
from ..error import LibraryError
from .base import ILibrary
from .capabilities import IMaterializable
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Mapping
from collections.abc import Callable, Iterator, Mapping, Sequence
from ..pattern import Pattern
@ -17,7 +18,7 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
class LazyLibrary(ILibrary):
class LazyLibrary(ILibrary, IMaterializable):
"""
This class is usually used to create a library of Patterns by mapping names to
functions which generate or load the relevant `Pattern` object as-needed.
@ -56,6 +57,9 @@ class LazyLibrary(ILibrary):
del self.cache[key]
def __getitem__(self, key: str) -> Pattern:
return self.materialize(key, persist=True)
def materialize(self, key: str, *, persist: bool = True) -> Pattern:
logger.debug(f'loading {key}')
if key in self.cache:
logger.debug(f'found {key} in cache')
@ -76,7 +80,8 @@ class LazyLibrary(ILibrary):
pat = func()
finally:
self._lookups_in_progress.pop()
self.cache[key] = pat
if persist:
self.cache[key] = pat
return pat
def __iter__(self) -> Iterator[str]:
@ -88,6 +93,15 @@ class LazyLibrary(ILibrary):
def __contains__(self, key: object) -> bool:
return key in self.mapping
def referenced_patterns(
self,
tops: str | Sequence[str] | None = None,
skip: set[str] | None = None,
) -> set[str]:
# Closure-backed cells do not have hierarchy metadata. Preserve laziness
# by loading only patterns reached from the requested roots.
return self._referenced_patterns_by_lookup(tops=tops, skip=skip)
def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None:
if isinstance(other, LazyLibrary):
self.mapping[key_self] = other.mapping[key_other]

107
masque/library/mapping.py
View file

@ -2,14 +2,19 @@
from __future__ import annotations
from pprint import pformat
from typing import TYPE_CHECKING, Self
from typing import TYPE_CHECKING, Any, Self, cast
from ..error import LibraryError
from .base import ILibrary, ILibraryView
from .capabilities import IBorrowing, IMaterializable
from .utils import dangling_mode_t, _validate_dangling_mode
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Mapping, MutableMapping
import numpy
from numpy.typing import NDArray
from ..pattern import Pattern
@ -44,6 +49,106 @@ class LibraryView(ILibraryView):
return f'<LibraryView ({type(self.mapping)}) with keys\n' + pformat(list(self.keys())) + '>'
class _SubtreeLibraryView(ILibraryView, IMaterializable, IBorrowing):
"""Borrowed subtree view with snapshotted membership and hierarchy."""
def __init__(
self,
source: ILibraryView,
*,
names: set[str],
child_graph: Mapping[str, set[str]],
) -> None:
self._source = source
source_order = source.source_order()
ordered = list(dict.fromkeys(name for name in source_order if name in names))
seen = set(ordered)
ordered.extend(name for name in source if name in names and name not in seen)
self._order = tuple(ordered)
self._names = frozenset(self._order)
self._child_graph = {
name: set(child_graph.get(name, set()))
for name in self._order
}
if hasattr(source, 'library_info'):
self.library_info = cast('dict[str, Any]', source.library_info)
def __getitem__(self, key: str) -> Pattern:
if key not in self._names:
raise KeyError(key)
return self._source[key]
def __iter__(self) -> Iterator[str]:
return iter(self._order)
def __len__(self) -> int:
return len(self._order)
def __contains__(self, key: object) -> bool:
return key in self._names
def borrowed_sources(self) -> tuple[ILibraryView, ...]:
return (self._source,)
def source_order(self) -> tuple[str, ...]:
return self._order
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
if name not in self._names:
raise KeyError(name)
if isinstance(self._source, IMaterializable):
return self._source.materialize(name, persist=persist)
return self._source[name]
def child_graph(
self,
dangling: dangling_mode_t = 'error',
) -> dict[str, set[str]]:
_validate_dangling_mode(dangling)
graph = {name: set(children) for name, children in self._child_graph.items()}
existing = set(graph)
dangling_refs = set().union(*(children - existing for children in graph.values())) if graph else set()
if dangling == 'error':
if dangling_refs:
raise self._dangling_refs_error(dangling_refs, 'building child graph')
return graph
if dangling == 'ignore':
return {
name: {child for child in children if child in existing}
for name, children in graph.items()
}
for target in dangling_refs:
graph.setdefault(target, set())
return graph
def find_refs_local(
self,
name: str,
parent_graph: dict[str, set[str]] | None = None,
dangling: dangling_mode_t = 'error',
) -> dict[str, list[NDArray[numpy.float64]]]:
_validate_dangling_mode(dangling)
if parent_graph is None:
graph_mode: dangling_mode_t = 'ignore' if dangling == 'ignore' else 'include'
parent_graph = self.parent_graph(dangling=graph_mode)
refs = self._source.find_refs_local(name, parent_graph=parent_graph, dangling=dangling)
return {parent: transforms for parent, transforms in refs.items() if parent in self._names}
def raw_struct_bytes(self, name: str) -> bytes:
if name not in self._names:
raise KeyError(name)
reader = getattr(self._source, 'raw_struct_bytes', None)
if not callable(reader):
raise TypeError('raw_struct_bytes')
return cast('bytes', reader(name))
def can_copy_raw_struct(self, name: str) -> bool:
if name not in self._names:
return False
can_copy = getattr(self._source, 'can_copy_raw_struct', None)
return bool(callable(can_copy) and can_copy(name))
class Library(ILibrary):
"""
Default implementation for a writeable library.

241
masque/library/overlay.py
View file

@ -3,34 +3,32 @@ from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, cast
from typing import TYPE_CHECKING, Any, Literal, Self, cast
import copy
import numpy
from ..error import LibraryError
from ..pattern import Pattern, map_targets
from ..utils import apply_transforms, layer_t
from .base import ILibrary, ILibraryView
from .utils import dangling_mode_t, _plan_source_names, _source_rename_map
from .capabilities import IBorrowing, IMaterializable
from .utils import INameView, dangling_mode_t, _plan_source_names, _source_rename_map, _validate_dangling_mode
from .mapping import LibraryView
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Mapping, Sequence
import numpy
from numpy.typing import NDArray
from ..ports import Port
from ..utils import layer_t
@dataclass
class _SourceLayer:
""" One imported source layer tracked by an `OverlayLibrary`. """
library: ILibraryView
source_to_visible: dict[str, str]
visible_to_source: dict[str, str]
source_target_map: dict[str, str]
child_graph: dict[str, set[str]]
order: list[str]
@dataclass(frozen=True)
@ -41,18 +39,19 @@ class _SourceEntry:
def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern:
func = getattr(view, '_materialize_pattern', None)
if callable(func):
return cast('Pattern', func(name, persist=False))
if isinstance(view, IMaterializable):
return view.materialize(name, persist=False).deepcopy()
return view[name].deepcopy()
class PortsLibraryView(ILibraryView):
class PortsLibraryView(ILibraryView, IMaterializable, IBorrowing):
"""
Read-only view which imports or applies ports on first materialization.
The wrapped source remains untouched; this view owns a separate processed
cache so direct-copy workflows can continue to use the raw source view.
The view borrows its source: callers must keep the source open for the
lifetime of the view and close the source themselves.
Graph queries, source ordering, and copy-through capabilities are delegated
to the wrapped source whenever possible, while `__getitem__` and
@ -84,7 +83,7 @@ class PortsLibraryView(ILibraryView):
self.library_info = cast('dict[str, Any]', source.library_info)
def __getitem__(self, key: str) -> Pattern:
return self._materialize_pattern(key, persist=True)
return self.materialize(key, persist=True)
def __iter__(self) -> Iterator[str]:
return iter(self._source)
@ -95,7 +94,7 @@ class PortsLibraryView(ILibraryView):
def __contains__(self, key: object) -> bool:
return key in self._source
def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern:
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
from ..utils.ports2data import data_to_ports # noqa: PLC0415
if name in self._cache:
@ -134,72 +133,31 @@ class PortsLibraryView(ILibraryView):
self._cache[name] = pat
return pat
def materialize_many(
self,
names: Sequence[str],
*,
persist: bool = True,
) -> LibraryView:
mats = {
name: self._materialize_pattern(name, persist=persist)
for name in dict.fromkeys(names)
}
return LibraryView(mats)
def source_order(self) -> tuple[str, ...]:
return self._source.source_order()
def borrowed_sources(self) -> tuple[ILibraryView, ...]:
return (self._source,)
def child_graph(
self,
dangling: dangling_mode_t = 'error',
) -> dict[str, set[str]]:
_validate_dangling_mode(dangling)
return self._source.child_graph(dangling=dangling)
def parent_graph(
self,
dangling: dangling_mode_t = 'error',
) -> dict[str, set[str]]:
return self._source.parent_graph(dangling=dangling)
def subtree(
self,
tops: str | Sequence[str],
) -> ILibraryView:
if isinstance(tops, str):
tops = (tops,)
keep = cast('set[str]', self._source.referenced_patterns(tops) - {None})
keep |= set(tops)
return self.materialize_many(tuple(keep), persist=True)
def tops(self) -> list[str]:
return self._source.tops()
def find_refs_local(
self,
name: str,
parent_graph: dict[str, set[str]] | None = None,
dangling: dangling_mode_t = 'error',
) -> dict[str, list[NDArray[numpy.float64]]]:
_validate_dangling_mode(dangling)
finder = getattr(self._source, 'find_refs_local', None)
if callable(finder):
return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling))
return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling)
def find_refs_global(
self,
name: str,
order: list[str] | None = None,
parent_graph: dict[str, set[str]] | None = None,
dangling: dangling_mode_t = 'error',
) -> dict[tuple[str, ...], NDArray[numpy.float64]]:
finder = getattr(self._source, 'find_refs_global', None)
if callable(finder):
return cast(
'dict[tuple[str, ...], NDArray[numpy.float64]]',
finder(name, order=order, parent_graph=parent_graph, dangling=dangling),
)
return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling)
def raw_struct_bytes(self, name: str) -> bytes:
reader = getattr(self._source, 'raw_struct_bytes', None)
if not callable(reader):
@ -207,30 +165,26 @@ class PortsLibraryView(ILibraryView):
return cast('bytes', reader(name))
def can_copy_raw_struct(self, name: str) -> bool:
if name in self._cache:
return False
can_copy = getattr(self._source, 'can_copy_raw_struct', None)
if not callable(can_copy):
return False
return bool(can_copy(name))
def close(self) -> None:
closer = getattr(self._source, 'close', None)
if callable(closer):
closer()
def __enter__(self) -> PortsLibraryView:
return self
def __exit__(self, *_args: object) -> None:
self.close()
class OverlayLibrary(ILibrary):
class OverlayLibrary(ILibrary, IMaterializable, IBorrowing):
"""
Mutable overlay over one or more source libraries.
Source-backed cells remain lazy until accessed through `__getitem__`, at
which point that visible cell is promoted into an overlay-owned materialized
`Pattern`.
Source libraries must remain open and must not be mutated after they are
added. The overlay borrows each source and snapshots its names, hierarchy,
and initial visible-name mapping while retaining the source itself for lazy
pattern materialization.
"""
def __init__(self) -> None:
@ -249,7 +203,7 @@ class OverlayLibrary(ILibrary):
return key in self._entries
def __getitem__(self, key: str) -> Pattern:
return self._materialize_pattern(key, persist=True)
return self.materialize(key, persist=True)
def __setitem__(
self,
@ -275,15 +229,19 @@ class OverlayLibrary(ILibrary):
self,
source: Mapping[str, Pattern] | ILibraryView,
*,
rename_theirs: Callable[[ILibraryView, str], str] | None = None,
rename_theirs: Callable[[INameView, str], str] | None = None,
rename_when: Literal['conflict', 'always'] = 'conflict',
) -> dict[str, str]:
"""
Add a source-backed library layer.
The source must remain open, and its names, hierarchy, and pattern
contents must remain unchanged for the lifetime of this overlay.
Args:
rename_theirs: Function used to choose visible names for imported
source cells.
source cells. Its `INameView` argument contains existing and
previously reserved names, but does not support pattern lookup.
rename_when: If `'conflict'`, only conflicting names are renamed.
If `'always'`, every imported source name is passed through
`rename_theirs`.
@ -295,18 +253,13 @@ class OverlayLibrary(ILibrary):
source_to_visible = _plan_source_names(
self,
source_order,
self._entries,
rename_theirs = rename_theirs,
rename_when = rename_when,
)
visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()}
layer = _SourceLayer(
library=view,
source_to_visible=source_to_visible,
visible_to_source=visible_to_source,
source_target_map=dict(source_to_visible),
child_graph=child_graph,
order=[source_to_visible[name] for name in source_order],
)
layer_index = len(self._layers)
self._layers.append(layer)
@ -333,11 +286,6 @@ class OverlayLibrary(ILibrary):
entry = self._entries.pop(old_name)
self._entries[new_name] = entry
if isinstance(entry, _SourceEntry):
layer = self._layers[entry.layer_index]
layer.source_to_visible[entry.source_name] = new_name
del layer.visible_to_source[old_name]
layer.visible_to_source[new_name] = entry.source_name
idx = self._order.index(old_name)
self._order[idx] = new_name
@ -375,10 +323,10 @@ class OverlayLibrary(ILibrary):
return self
def _effective_target(self, layer: _SourceLayer, target: str) -> str:
visible = layer.source_to_visible.get(target, target)
visible = layer.source_target_map.get(target, target)
return self._resolve_target(visible)
def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern:
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
if name not in self._entries:
raise KeyError(name)
entry = self._entries[name]
@ -386,7 +334,7 @@ class OverlayLibrary(ILibrary):
return entry
layer = self._layers[entry.layer_index]
source_pat = layer.library[entry.source_name].deepcopy()
source_pat = _materialize_detached_pattern(layer.library, entry.source_name)
def remap(target: str | None) -> str | None:
return None if target is None else self._effective_target(layer, target)
@ -402,6 +350,7 @@ class OverlayLibrary(ILibrary):
self,
dangling: dangling_mode_t = 'error',
) -> dict[str, set[str]]:
_validate_dangling_mode(dangling)
graph: dict[str, set[str]] = {}
for name in self._order:
if name not in self._entries:
@ -427,33 +376,31 @@ class OverlayLibrary(ILibrary):
graph.setdefault(cast('str', child), set())
return graph
def parent_graph(
self,
dangling: dangling_mode_t = 'error',
) -> dict[str, set[str]]:
child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore')
existing = set(self.keys())
igraph: dict[str, set[str]] = {name: set() for name in child_graph}
for parent, children in child_graph.items():
for child in children:
if child in existing or dangling == 'include':
igraph.setdefault(child, set()).add(parent)
if dangling == 'error':
raw = self.child_graph(dangling='include')
dangling_refs = set().union(*(children - existing for children in raw.values()))
if dangling_refs:
raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph')
return igraph
def subtree(
self,
tops: str | Sequence[str],
) -> ILibraryView:
) -> Self:
if isinstance(tops, str):
tops = (tops,)
keep = cast('set[str]', self.referenced_patterns(tops) - {None})
graph = self.child_graph(dangling='include')
keep = self._referenced_patterns_from_graph(graph, tops=tops)
keep &= set(self)
keep |= set(tops)
return LibraryView({name: self[name] for name in keep})
new = type(self)()
new._layers = [
_SourceLayer(
library=layer.library,
source_target_map=dict(layer.source_target_map),
child_graph={name: set(children) for name, children in layer.child_graph.items()},
)
for layer in self._layers
]
new._order = [name for name in self._order if name in keep and name in self._entries]
new._entries = {name: self._entries[name] for name in new._order}
new._target_remap = dict(self._target_remap)
return new
def find_refs_local(
self,
@ -461,6 +408,7 @@ class OverlayLibrary(ILibrary):
parent_graph: dict[str, set[str]] | None = None,
dangling: dangling_mode_t = 'error',
) -> dict[str, list[NDArray[numpy.float64]]]:
_validate_dangling_mode(dangling)
instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list)
if parent_graph is None:
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
@ -475,57 +423,34 @@ class OverlayLibrary(ILibrary):
return instances
for parent in parent_graph.get(name, set()):
pat = self._materialize_pattern(parent, persist=False)
pat = self.materialize(parent, persist=False)
for ref in pat.refs.get(name, []):
instances[parent].append(ref.as_transforms())
return instances
def find_refs_global(
self,
name: str,
order: list[str] | None = None,
parent_graph: dict[str, set[str]] | None = None,
dangling: dangling_mode_t = 'error',
) -> dict[tuple[str, ...], NDArray[numpy.float64]]:
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
if order is None:
order = self.child_order(dangling=graph_mode)
if parent_graph is None:
parent_graph = self.parent_graph(dangling=graph_mode)
if name not in self:
if name not in parent_graph:
return {}
if dangling == 'error':
raise self._dangling_refs_error({name}, f'finding global refs for {name!r}')
if dangling == 'ignore':
return {}
self_keys = set(self.keys())
transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]]
transforms = defaultdict(list)
for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items():
transforms[parent] = [((name,), numpy.concatenate(vals))]
for next_name in order:
if next_name not in transforms:
continue
if not parent_graph.get(next_name, set()) & self_keys:
continue
outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling)
inners = transforms.pop(next_name)
for parent, outer in outers.items():
outer_tf = numpy.concatenate(outer)
for path, inner in inners:
combined = apply_transforms(outer_tf, inner)
transforms[parent].append(((next_name,) + path, combined))
result = {}
for parent, targets in transforms.items():
for path, instances in targets:
result[(parent,) + path] = instances
return result
def source_order(self) -> tuple[str, ...]:
return tuple(name for name in self._order if name in self._entries)
def borrowed_sources(self) -> tuple[ILibraryView, ...]:
return tuple(layer.library for layer in self._layers)
def can_copy_raw_struct(self, name: str) -> bool:
entry = self._entries.get(name)
if not isinstance(entry, _SourceEntry) or name != entry.source_name:
return False
layer = self._layers[entry.layer_index]
can_copy = getattr(layer.library, 'can_copy_raw_struct', None)
if not callable(can_copy) or not can_copy(entry.source_name):
return False
children = layer.child_graph.get(entry.source_name, set())
return all(self._effective_target(layer, child) == child for child in children)
def raw_struct_bytes(self, name: str) -> bytes:
entry = self._entries.get(name)
if not isinstance(entry, _SourceEntry):
raise TypeError('raw_struct_bytes')
layer = self._layers[entry.layer_index]
reader = getattr(layer.library, 'raw_struct_bytes', None)
if not callable(reader):
raise TypeError('raw_struct_bytes')
return cast('bytes', reader(entry.source_name))

137
masque/library/utils.py
View file

@ -1,8 +1,11 @@
"""Shared types and helpers for library implementations."""
from __future__ import annotations
from abc import ABC
from typing import TYPE_CHECKING, Literal, Protocol, TypeAlias
from collections.abc import Callable, Container, Mapping, MutableMapping, Sequence
from collections.abc import Callable, Collection, Iterator, Mapping, MutableMapping, Sequence
import logging
import re
from ..error import LibraryError
@ -11,7 +14,80 @@ if TYPE_CHECKING:
from numpy.typing import NDArray
from ..pattern import Pattern
from .base import ILibraryView
logger = logging.getLogger(__name__)
SINGLE_USE_PREFIX = '_'
"""
Names starting with this prefix are assumed to refer to single-use patterns,
which may be renamed automatically by `ILibrary.add()` (via
`rename_theirs=_rename_patterns()` )
"""
# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere?
class INameView(Collection[str], ABC):
"""
Read-only collection of reserved names with a shared name allocator.
Name views support membership, iteration, `len()`, and `get_name()`. They
do not provide pattern lookup or the other operations of a library mapping.
"""
def get_name(
self,
name: str = SINGLE_USE_PREFIX * 2,
sanitize: bool = True,
max_length: int = 32,
quiet: bool | None = None,
) -> str:
"""
Find a unique name.
This function may be overridden in a subclass or monkey-patched to fit
the caller's requirements.
Args:
name: Preferred name. Default is `SINGLE_USE_PREFIX * 2`.
sanitize: Allow only alphanumeric characters and _?$, replacing
invalid characters with underscores.
max_length: Truncate names longer than this.
quiet: Suppress log messages when `True`. The default suppresses
messages only when `name` starts with `SINGLE_USE_PREFIX`.
Returns:
A name unique within this view.
"""
if quiet is None:
quiet = name.startswith(SINGLE_USE_PREFIX)
if sanitize:
sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name)
else:
sanitized_name = name
suffixed_name = sanitized_name
if sanitized_name in self:
ii = sum(1 for nn in self if nn.startswith(sanitized_name))
else:
ii = 0
while suffixed_name in self or suffixed_name == '':
suffixed_name = sanitized_name + b64suffix(ii)
ii += 1
if len(suffixed_name) > max_length:
if name == '':
raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}')
cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True)
else:
cropped_name = suffixed_name
if not quiet:
logger.info(f'Requested name "{name}" changed to "{cropped_name}"')
return cropped_name
class visitor_function_t(Protocol):
@ -34,16 +110,9 @@ Tree: TypeAlias = MutableMapping[str, 'Pattern']
dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include']
""" How helpers should handle refs whose targets are not present in the library. """
SINGLE_USE_PREFIX = '_'
"""
Names starting with this prefix are assumed to refer to single-use patterns,
which may be renamed automatically by `ILibrary.add()` (via
`rename_theirs=_rename_patterns()` )
"""
# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere?
def _rename_patterns(lib: ILibraryView, name: str) -> str:
def _rename_patterns(lib: INameView, name: str) -> str:
"""
The default `rename_theirs` function for `ILibrary.add`.
@ -67,12 +136,41 @@ def _rename_patterns(lib: ILibraryView, name: str) -> str:
return lib.get_name(SINGLE_USE_PREFIX + stem)
def _validate_dangling_mode(dangling: dangling_mode_t) -> None:
if dangling not in ('error', 'ignore', 'include'):
raise ValueError(
f'Unknown dangling-reference mode {dangling!r}; '
'expected one of "error", "ignore", or "include"'
)
class _ProspectiveNames(INameView):
"""Target names plus names reserved earlier in an addition plan."""
def __init__(
self,
target: INameView,
reserved: set[str],
) -> None:
self._target = target
self._reserved = reserved
def __iter__(self) -> Iterator[str]:
yield from self._target
yield from self._reserved
def __len__(self) -> int:
return len(self._target) + len(self._reserved)
def __contains__(self, key: object) -> bool:
return key in self._reserved or key in self._target
def _plan_source_names(
target: ILibraryView,
target: INameView,
source_order: Sequence[str],
existing_names: Container[str],
*,
rename_theirs: Callable[[ILibraryView, str], str] | None = None,
rename_theirs: Callable[[INameView, str], str] | None = None,
rename_when: Literal['conflict', 'always'] = 'conflict',
) -> dict[str, str]:
if rename_when not in ('conflict', 'always'):
@ -81,21 +179,22 @@ def _plan_source_names(
raise TypeError('rename_theirs is required when rename_when="always"')
source_to_visible: dict[str, str] = {}
visible_names: set[str] = set()
reserved: set[str] = set()
prospective = _ProspectiveNames(target, reserved)
for name in source_order:
visible = name
if rename_when == 'always':
assert rename_theirs is not None
visible = rename_theirs(target, name)
elif visible in existing_names or visible in visible_names:
visible = rename_theirs(prospective, name)
elif visible in prospective:
if rename_theirs is None:
raise LibraryError(f'Conflicting name while adding source: {name!r}')
visible = rename_theirs(target, name)
if visible in existing_names or visible in visible_names:
visible = rename_theirs(prospective, name)
if visible in prospective:
raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}')
source_to_visible[name] = visible
visible_names.add(visible)
reserved.add(visible)
return source_to_visible