From 583bd5bd77ff20dfcb01738cebce1af95c53d912 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 10:34:50 -0700 Subject: [PATCH 01/10] [planner] avoid rotation_matrix_2d since float inaccuracy breaks caching --- masque/builder/planner/planner.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/masque/builder/planner/planner.py b/masque/builder/planner/planner.py index a9257ce..5cc83b1 100644 --- a/masque/builder/planner/planner.py +++ b/masque/builder/planner/planner.py @@ -24,7 +24,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from itertools import combinations -from math import isclose as math_isclose +from math import cos, isclose as math_isclose, sin from typing import Any, Literal import numpy @@ -33,7 +33,7 @@ from numpy.typing import ArrayLike from ...error import BuildError, PortError from ...ports import Port -from ...utils import PTypeMatch, SupportsBool, ptype_match, ptypes_compatible, rotation_matrix_2d +from ...utils import PTypeMatch, SupportsBool, ptype_match, ptypes_compatible from ..tools import ( BendOffer, PrimitiveKind, @@ -450,17 +450,21 @@ class Solver: points back into the primitive, so each step advances orientation by the primitive output rotation plus pi. """ - offset = numpy.zeros(2) + x = 0.0 + y = 0.0 angle = 0.0 ptype: str | None = None for step in steps: out_port = step.out_port if out_port.rotation is None: raise BuildError('Primitive endpoints must have rotation') - offset += rotation_matrix_2d(angle) @ out_port.offset + angle_cos = cos(angle) + angle_sin = sin(angle) + x += angle_cos * float(out_port.x) - angle_sin * float(out_port.y) + y += angle_sin * float(out_port.x) + angle_cos * float(out_port.y) angle += out_port.rotation + pi ptype = out_port.ptype - return Port(offset, rotation=angle - pi, ptype=ptype) + return Port((x, y), rotation=angle - pi, ptype=ptype) def current_ptype(self, steps: Sequence[SelectedPrimitive]) -> str | None: return self.request.in_ptype if not steps else self.compose_endpoint(steps).ptype From b6c222cdc2c91dc4fa28f382127b5e62ca7da0fe Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 10:36:00 -0700 Subject: [PATCH 02/10] [planner] cache primitive offers --- masque/builder/planner/planner.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/masque/builder/planner/planner.py b/masque/builder/planner/planner.py index 5cc83b1..2b68c45 100644 --- a/masque/builder/planner/planner.py +++ b/masque/builder/planner/planner.py @@ -288,6 +288,10 @@ class Solver: def __init__(self, request: RouteRequest) -> None: self.request = request self.eval_cache: dict[tuple[int, float, str | None, str | None, str, str, PrimitiveKind | None], SelectedPrimitive] = {} + self.offer_cache: dict[ + tuple[PrimitiveKind, str | None, str | None, tuple[tuple[str, Any], ...]], + tuple[PrimitiveOffer, ...], + ] = {} self.order = 0 def solve(self) -> Candidate: @@ -384,7 +388,18 @@ class Solver: kwargs.pop('out_ptype', None) if extra: kwargs.update(extra) - return self.request.tool.primitive_offers(kind, in_ptype=in_ptype, out_ptype=out_ptype, **kwargs) + try: + cache_key = (kind, in_ptype, out_ptype, tuple(sorted(kwargs.items()))) + hash(cache_key) + except TypeError: + return self.request.tool.primitive_offers(kind, in_ptype=in_ptype, out_ptype=out_ptype, **kwargs) + + cached = self.offer_cache.get(cache_key) + if cached is not None: + return cached + offers = self.request.tool.primitive_offers(kind, in_ptype=in_ptype, out_ptype=out_ptype, **kwargs) + self.offer_cache[cache_key] = offers + return offers def evaluate( self, From 54c4cd9a4a930deb7b382e0afcae2725cf9d19a4 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 11:00:05 -0700 Subject: [PATCH 03/10] [planner] try bends=2 before bends=(2, 4) --- masque/builder/planner/planner.py | 42 +++++++++++---- masque/test/test_pather_trace_into.py | 77 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 11 deletions(-) diff --git a/masque/builder/planner/planner.py b/masque/builder/planner/planner.py index 2b68c45..ce9a62d 100644 --- a/masque/builder/planner/planner.py +++ b/masque/builder/planner/planner.py @@ -50,6 +50,7 @@ from .interface import ( PreparedRouteResult, RoutePlanningError, RoutePortContext, + route_error_is_fatal, ) @@ -908,6 +909,12 @@ class RoutingPlanner: TRACE_INTO_MAX_BENDS: int = 4 + def trace_into_bend_budgets(self, family: PrimitiveKind) -> tuple[int, ...]: + """Return staged trace_into bend budgets for the route rotation parity.""" + if family == 'bend': + return tuple(budget for budget in (1, 3) if budget <= self.TRACE_INTO_MAX_BENDS) + return tuple(budget for budget in (2, 4) if budget <= self.TRACE_INTO_MAX_BENDS) + def plan_leg( self, family: PrimitiveKind, @@ -1197,17 +1204,30 @@ class RoutingPlanner: desired.rotation = port_dst.rotation - pi desired.ptype = out_ptype family, length, jog, ccw = self.trace_into_spec(context_src.port, desired) - leg = self.plan_leg( - family, - context_src, - length=length, - jog=jog, - ccw=ccw, - plug_into=portspec_dst if plug_destination else None, - constrain_jog=family == 'bend', - max_bends=self.TRACE_INTO_MAX_BENDS, - **(dict(kwargs) | {'out_ptype': out_ptype}), - ) + leg = None + last_error: Exception | None = None + for max_bends in self.trace_into_bend_budgets(family): + try: + leg = self.plan_leg( + family, + context_src, + length=length, + jog=jog, + ccw=ccw, + plug_into=portspec_dst if plug_destination else None, + constrain_jog=family == 'bend', + max_bends=max_bends, + **(dict(kwargs) | {'out_ptype': out_ptype}), + ) + break + except (BuildError, NotImplementedError) as err: + if route_error_is_fatal(err): + raise + last_error = err + if leg is None: + if last_error is not None: + raise last_error + raise BuildError('No legal primitive offer for trace_into route') renames = ((thru, context_src.portspec),) if thru is not None else () return self.prepared_result_from_legs( (leg,), diff --git a/masque/test/test_pather_trace_into.py b/masque/test/test_pather_trace_into.py index 7c5a362..ac51992 100644 --- a/masque/test/test_pather_trace_into.py +++ b/masque/test/test_pather_trace_into.py @@ -6,6 +6,8 @@ from numpy import pi from numpy.testing import assert_equal from masque import Library, PathTool, Port, Pather +from masque.builder.planner import PreparedRouteResult, RoutePlanningError, RoutePortContext, RoutingPlanner +from masque.builder.planner.planner import Candidate, RouteLeg from masque.builder.tools import BendOffer, PrimitiveOffer, StraightOffer, Tool from masque.error import BuildError, PortError @@ -260,3 +262,78 @@ def test_pather_trace_into_rejects_reserved_route_kwargs( assert p.pattern.ports['B'].rotation is not None assert numpy.isclose(p.pattern.ports['B'].rotation, dst.rotation) assert len(p._paths['A']) == 0 + + +class TraceIntoBudgetPlanner(RoutingPlanner): + def __init__(self, successes: set[int], fatal_at: set[int] | None = None) -> None: + self.successes = successes + self.fatal_at = set() if fatal_at is None else fatal_at + self.attempts: list[int | None] = [] + + def plan_leg( + self, + family: Any, + context: RoutePortContext, + *, + length: float | None = None, + jog: float | None = None, + ccw: Any = None, + plug_into: str | None = None, + constrain_jog: bool = False, + max_bends: int | None = None, + **kwargs: Any, + ) -> RouteLeg: + _ = family, length, jog, ccw, constrain_jog, kwargs + self.attempts.append(max_bends) + if max_bends in self.fatal_at: + raise RoutePlanningError('fatal', fatal=True) + if max_bends not in self.successes: + raise BuildError('try next budget') + return RouteLeg( + portspec=context.portspec, + start_port=context.port.copy(), + tool=context.tool, + candidate=Candidate((), context.port.copy(), 0.0, 0, 0.0), + plug_into=plug_into, + ) + + def prepared_result_from_legs( + self, + legs: Any, + *, + renames: tuple[tuple[str, str], ...] = (), + ) -> PreparedRouteResult: + _ = legs, renames + return PreparedRouteResult(()) + + +@pytest.mark.parametrize( + ('dst', 'successes', 'attempts'), + [ + (Port((-10, 0), rotation=pi, ptype='wire'), {2}, [2]), + (Port((-10, 0), rotation=pi, ptype='wire'), {4}, [2, 4]), + (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {1}, [1]), + (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {3}, [1, 3]), + ], +) +def test_trace_into_uses_staged_bend_budgets( + dst: Port, + successes: set[int], + attempts: list[int], + ) -> None: + planner = TraceIntoBudgetPlanner(successes) + context = RoutePortContext('src', Port((0, 0), rotation=0, ptype='wire'), PathTool(layer='M1', width=1, ptype='wire')) + + planner.plan_trace_into(context, 'dst', dst, out_ptype=None, plug_destination=True, thru=None) + + assert planner.attempts == attempts + + +def test_trace_into_staged_bend_budget_stops_on_fatal_error() -> None: + planner = TraceIntoBudgetPlanner({4}, fatal_at={2}) + context = RoutePortContext('src', Port((0, 0), rotation=0, ptype='wire'), PathTool(layer='M1', width=1, ptype='wire')) + + with pytest.raises(RoutePlanningError, match='fatal'): + planner.plan_trace_into(context, 'dst', Port((-10, 0), rotation=pi, ptype='wire'), out_ptype=None, plug_destination=True, thru=None) + + assert planner.attempts == [2] From f864ebbeab02ef86707d8f414d29bb6f83724f85 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 11:09:52 -0700 Subject: [PATCH 04/10] [planner] make solver carry state across increasing bend counts --- masque/builder/planner/planner.py | 205 +++++++++++++++++--------- masque/test/test_pather_trace_into.py | 82 ++++++----- 2 files changed, 186 insertions(+), 101 deletions(-) diff --git a/masque/builder/planner/planner.py b/masque/builder/planner/planner.py index ce9a62d..b684af9 100644 --- a/masque/builder/planner/planner.py +++ b/masque/builder/planner/planner.py @@ -293,16 +293,24 @@ class Solver: tuple[PrimitiveKind, str | None, str | None, tuple[tuple[str, Any], ...]], tuple[PrimitiveOffer, ...], ] = {} + self.seen_candidate_keys: set[tuple[Any, ...]] = set() self.order = 0 - def solve(self) -> Candidate: - """ - Enumerate, finalize, deduplicate, and rank legal candidates. + @staticmethod + def route_bend_count(steps: Sequence[SelectedPrimitive]) -> int: + """Return the route bend budget consumed by non-adapter primitives.""" + count = 0 + for step in steps: + if step.role == 'adapter': + continue + if step.route_kind == 'bend': + count += 1 + elif step.route_kind in ('s', 'u'): + count += 2 + return count - Non-fatal candidate errors are accumulated so the failure message can - preserve useful Tool feedback. Fatal offer-contract errors stop the - solve immediately. - """ + def candidate_key(self, candidate: Candidate) -> tuple[Any, ...]: + """Return a deterministic key for duplicate solved candidates.""" def endpoint_key(port: Port) -> tuple[float, float, float | None, str | None]: return ( round(float(port.x), 9), @@ -311,36 +319,51 @@ class Solver: port.ptype, ) - def candidate_key(candidate: Candidate) -> tuple[Any, ...]: - def offer_key(offer: PrimitiveOffer) -> tuple[Any, ...]: - return ( - type(offer).__qualname__, - offer.in_ptype, - offer.out_ptype, - round(float(offer.priority_bias), 9), - tuple(round(float(value), 9) for value in offer.parameter_domain), - getattr(offer, 'ccw', None), - id(offer.endpoint_planner), - id(offer.commit_planner), - ) - + def offer_key(offer: PrimitiveOffer) -> tuple[Any, ...]: return ( - endpoint_key(candidate.end_port), - tuple(( - offer_key(step.offer), - step.role, - step.route_kind, - round(float(step.parameter), 9), - endpoint_key(step.out_port), - ) for step in candidate.steps), + type(offer).__qualname__, + offer.in_ptype, + offer.out_ptype, + round(float(offer.priority_bias), 9), + tuple(round(float(value), 9) for value in offer.parameter_domain), + getattr(offer, 'ccw', None), + id(offer.endpoint_planner), + id(offer.commit_planner), ) + return ( + endpoint_key(candidate.end_port), + tuple(( + offer_key(step.offer), + step.role, + step.route_kind, + round(float(step.parameter), 9), + endpoint_key(step.out_port), + ) for step in candidate.steps), + ) + + def solve( + self, + *, + min_bends: int = 0, + max_bends: int | None = None, + ) -> Candidate: + """ + Enumerate, finalize, deduplicate, and rank legal candidates. + + Non-fatal candidate errors are accumulated so the failure message can + preserve useful Tool feedback. Fatal offer-contract errors stop the + solve immediately. + """ + if max_bends is None: + max_bends = self.request.bend_budget candidates: list[Candidate] = [] errors: list[Exception] = [] - seen: set[tuple[Any, ...]] = set() - for steps in self.enumerate_grammar(): + for steps in self.enumerate_grammar(max_bends): if not steps: continue + if self.route_bend_count(steps) < min_bends: + continue if any(first.role == 'adapter' and second.role == 'adapter' for first, second in zip(steps, steps[1:], strict=False)): continue try: @@ -349,10 +372,10 @@ class Solver: raise_if_fatal(err) errors.append(err) continue - key = candidate_key(candidate) - if key in seen: + key = self.candidate_key(candidate) + if key in self.seen_candidate_keys: continue - seen.add(key) + self.seen_candidate_keys.add(key) candidates.append(candidate) if not candidates: @@ -676,13 +699,13 @@ class Solver: options.extend((turn, 2) for turn in self.su_primitive_options(steps, 'u', jog)) return tuple(options) - def enumerate_grammar(self) -> Iterable[tuple[SelectedPrimitive, ...]]: + def enumerate_grammar(self, max_bends: int) -> Iterable[tuple[SelectedPrimitive, ...]]: """Yield raw primitive sequences allowed by the bounded route grammar.""" residual_jog = 0.0 if self.request.jog is None else float(self.request.jog) base: tuple[SelectedPrimitive, ...] = () for prefix_adapter in self.adapter_options(base, residual_jog=residual_jog): prefix = (*base, *prefix_adapter) - yield from self.enumerate_segments(prefix, self.request.bend_budget, residual_jog=residual_jog) + yield from self.enumerate_segments(prefix, max_bends, residual_jog=residual_jog) def enumerate_segments( self, @@ -909,11 +932,63 @@ class RoutingPlanner: TRACE_INTO_MAX_BENDS: int = 4 - def trace_into_bend_budgets(self, family: PrimitiveKind) -> tuple[int, ...]: - """Return staged trace_into bend budgets for the route rotation parity.""" + def trace_into_bend_bands(self, family: PrimitiveKind) -> tuple[tuple[int, int], ...]: + """Return non-overlapping trace_into bend-budget bands for staged solving.""" + max_bends = self.TRACE_INTO_MAX_BENDS if family == 'bend': - return tuple(budget for budget in (1, 3) if budget <= self.TRACE_INTO_MAX_BENDS) - return tuple(budget for budget in (2, 4) if budget <= self.TRACE_INTO_MAX_BENDS) + return tuple(band for band in ((1, 1), (3, 3)) if band[1] <= max_bends) + bands: list[tuple[int, int]] = [] + if max_bends >= 0: + bands.append((0, 2 if max_bends >= 2 else 0)) + if max_bends >= 4: + bands.append((4, 4)) + return tuple(bands) + + def route_request( + self, + family: PrimitiveKind, + context: RoutePortContext, + *, + length: float | None = None, + jog: float | None = None, + ccw: SupportsBool | None = None, + constrain_jog: bool = False, + max_bends: int | None = None, + **kwargs: Any, + ) -> RouteRequest: + """Build normalized solver input for one route leg.""" + return RouteRequest( + family=family, + tool=context.tool, + in_ptype=context.port.ptype, + route_kwargs=kwargs, + length=length, + jog=jog, + ccw=ccw, + out_ptype=kwargs.get('out_ptype'), + constrain_jog=constrain_jog, + max_bends=max_bends, + ) + + def solver_for_request(self, request: RouteRequest) -> Solver: + """Construct the solver for a route request.""" + return Solver(request) + + def route_leg_from_candidate( + self, + context: RoutePortContext, + candidate: Candidate, + *, + plug_into: str | None, + ) -> RouteLeg: + """Attach a solved candidate to its source Pather context.""" + return RouteLeg( + portspec=context.portspec, + start_port=context.port.copy(), + tool=context.tool, + candidate=candidate, + plug_into=plug_into, + ) def plan_leg( self, @@ -929,31 +1004,23 @@ class RoutingPlanner: **kwargs: Any, ) -> RouteLeg: """Solve one route leg and attach it to its source Pather context.""" - request = RouteRequest( + request = self.route_request( family=family, - tool=context.tool, - in_ptype=context.port.ptype, - route_kwargs=kwargs, + context=context, length=length, jog=jog, ccw=ccw, - out_ptype=kwargs.get('out_ptype'), constrain_jog=constrain_jog, max_bends=max_bends, + **kwargs, ) try: - candidate = Solver(request).solve() + candidate = self.solver_for_request(request).solve() except BuildError as err: if family == 'u' and length is None and not getattr(err, 'fatal', False): raise BuildError('No legal primitive offer for omitted-length U-turn') from err raise - return RouteLeg( - portspec=context.portspec, - start_port=context.port.copy(), - tool=context.tool, - candidate=candidate, - plug_into=plug_into, - ) + return self.route_leg_from_candidate(context, candidate, plug_into=plug_into) def prepared_route_action_from_leg( self, @@ -1204,30 +1271,36 @@ class RoutingPlanner: desired.rotation = port_dst.rotation - pi desired.ptype = out_ptype family, length, jog, ccw = self.trace_into_spec(context_src.port, desired) - leg = None + request = self.route_request( + family, + context_src, + length=length, + jog=jog, + ccw=ccw, + constrain_jog=family == 'bend', + max_bends=self.TRACE_INTO_MAX_BENDS, + **(dict(kwargs) | {'out_ptype': out_ptype}), + ) + solver = self.solver_for_request(request) + candidate = None last_error: Exception | None = None - for max_bends in self.trace_into_bend_budgets(family): + for min_bends, max_bends in self.trace_into_bend_bands(family): try: - leg = self.plan_leg( - family, - context_src, - length=length, - jog=jog, - ccw=ccw, - plug_into=portspec_dst if plug_destination else None, - constrain_jog=family == 'bend', - max_bends=max_bends, - **(dict(kwargs) | {'out_ptype': out_ptype}), - ) + candidate = solver.solve(min_bends=min_bends, max_bends=max_bends) break except (BuildError, NotImplementedError) as err: if route_error_is_fatal(err): raise last_error = err - if leg is None: + if candidate is None: if last_error is not None: raise last_error raise BuildError('No legal primitive offer for trace_into route') + leg = self.route_leg_from_candidate( + context_src, + candidate, + plug_into=portspec_dst if plug_destination else None, + ) renames = ((thru, context_src.portspec),) if thru is not None else () return self.prepared_result_from_legs( (leg,), diff --git a/masque/test/test_pather_trace_into.py b/masque/test/test_pather_trace_into.py index ac51992..8f66b28 100644 --- a/masque/test/test_pather_trace_into.py +++ b/masque/test/test_pather_trace_into.py @@ -7,7 +7,7 @@ from numpy.testing import assert_equal from masque import Library, PathTool, Port, Pather from masque.builder.planner import PreparedRouteResult, RoutePlanningError, RoutePortContext, RoutingPlanner -from masque.builder.planner.planner import Candidate, RouteLeg +from masque.builder.planner.planner import Candidate, RouteRequest from masque.builder.tools import BendOffer, PrimitiveOffer, StraightOffer, Tool from masque.error import BuildError, PortError @@ -264,38 +264,37 @@ def test_pather_trace_into_rejects_reserved_route_kwargs( assert len(p._paths['A']) == 0 -class TraceIntoBudgetPlanner(RoutingPlanner): - def __init__(self, successes: set[int], fatal_at: set[int] | None = None) -> None: +class TraceIntoBudgetSolver: + def __init__(self, successes: set[tuple[int, int]], fatal_at: set[tuple[int, int]] | None = None) -> None: self.successes = successes self.fatal_at = set() if fatal_at is None else fatal_at - self.attempts: list[int | None] = [] + self.attempts: list[tuple[int, int]] = [] - def plan_leg( + def solve( self, - family: Any, - context: RoutePortContext, *, - length: float | None = None, - jog: float | None = None, - ccw: Any = None, - plug_into: str | None = None, - constrain_jog: bool = False, + min_bends: int = 0, max_bends: int | None = None, - **kwargs: Any, - ) -> RouteLeg: - _ = family, length, jog, ccw, constrain_jog, kwargs - self.attempts.append(max_bends) - if max_bends in self.fatal_at: + ) -> Candidate: + assert max_bends is not None + band = (min_bends, max_bends) + self.attempts.append(band) + if band in self.fatal_at: raise RoutePlanningError('fatal', fatal=True) - if max_bends not in self.successes: + if band not in self.successes: raise BuildError('try next budget') - return RouteLeg( - portspec=context.portspec, - start_port=context.port.copy(), - tool=context.tool, - candidate=Candidate((), context.port.copy(), 0.0, 0, 0.0), - plug_into=plug_into, - ) + return Candidate((), Port((0, 0), rotation=0, ptype='wire'), 0.0, 0, 0.0) + + +class TraceIntoBudgetPlanner(RoutingPlanner): + def __init__(self, successes: set[tuple[int, int]], fatal_at: set[tuple[int, int]] | None = None) -> None: + self.solver = TraceIntoBudgetSolver(successes, fatal_at=fatal_at) + self.solver_requests = 0 + + def solver_for_request(self, request: RouteRequest) -> Any: + _ = request + self.solver_requests += 1 + return self.solver def prepared_result_from_legs( self, @@ -310,30 +309,43 @@ class TraceIntoBudgetPlanner(RoutingPlanner): @pytest.mark.parametrize( ('dst', 'successes', 'attempts'), [ - (Port((-10, 0), rotation=pi, ptype='wire'), {2}, [2]), - (Port((-10, 0), rotation=pi, ptype='wire'), {4}, [2, 4]), - (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {1}, [1]), - (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {3}, [1, 3]), + (Port((-10, 0), rotation=pi, ptype='wire'), {(0, 2)}, [(0, 2)]), + (Port((-10, 0), rotation=pi, ptype='wire'), {(4, 4)}, [(0, 2), (4, 4)]), + (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {(1, 1)}, [(1, 1)]), + (Port((-10, -10), rotation=3 * pi / 2, ptype='wire'), {(3, 3)}, [(1, 1), (3, 3)]), ], ) -def test_trace_into_uses_staged_bend_budgets( +def test_trace_into_reuses_solver_across_staged_bend_bands( dst: Port, - successes: set[int], - attempts: list[int], + successes: set[tuple[int, int]], + attempts: list[tuple[int, int]], ) -> None: planner = TraceIntoBudgetPlanner(successes) context = RoutePortContext('src', Port((0, 0), rotation=0, ptype='wire'), PathTool(layer='M1', width=1, ptype='wire')) planner.plan_trace_into(context, 'dst', dst, out_ptype=None, plug_destination=True, thru=None) - assert planner.attempts == attempts + assert planner.solver.attempts == attempts + assert planner.solver_requests == 1 def test_trace_into_staged_bend_budget_stops_on_fatal_error() -> None: - planner = TraceIntoBudgetPlanner({4}, fatal_at={2}) + planner = TraceIntoBudgetPlanner({(4, 4)}, fatal_at={(0, 2)}) context = RoutePortContext('src', Port((0, 0), rotation=0, ptype='wire'), PathTool(layer='M1', width=1, ptype='wire')) with pytest.raises(RoutePlanningError, match='fatal'): planner.plan_trace_into(context, 'dst', Port((-10, 0), rotation=pi, ptype='wire'), out_ptype=None, plug_destination=True, thru=None) - assert planner.attempts == [2] + assert planner.solver.attempts == [(0, 2)] + assert planner.solver_requests == 1 + + +def test_trace_into_bend_bands_respect_max_bends() -> None: + class OneBendPlanner(RoutingPlanner): + TRACE_INTO_MAX_BENDS = 1 + + planner = OneBendPlanner() + + assert planner.trace_into_bend_bands('straight') == ((0, 0),) + assert planner.trace_into_bend_bands('s') == ((0, 0),) + assert planner.trace_into_bend_bands('bend') == ((1, 1),) From 84664303f1a66da1de31a0b8311590dd50b49a54 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 11:27:29 -0700 Subject: [PATCH 05/10] [planner] early exit if rotation doesn't match --- masque/builder/planner/planner.py | 18 ++++++++ masque/test/test_pather_primitive_offers.py | 47 +++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/masque/builder/planner/planner.py b/masque/builder/planner/planner.py index b684af9..f7c3cca 100644 --- a/masque/builder/planner/planner.py +++ b/masque/builder/planner/planner.py @@ -857,6 +857,22 @@ class Solver: return False return self.request.out_ptype is None or ptypes_compatible(end_port.ptype, self.request.out_ptype) + def rotation_matches_request(self, steps: Sequence[SelectedPrimitive]) -> bool: + """ + Return true when a sequence can produce the requested output rotation. + + Primitive parameters do not affect the rotation contract, so + rotation-impossible candidates can be rejected before parameter solving. + """ + angle = 0.0 + for step in steps: + step_rotation = step.out_port.rotation + if step_rotation is None: + raise BuildError('Primitive endpoints must have rotation') + angle += float(step_rotation) + pi + rotation_delta = ((angle - pi) - self.request.out_rotation) % (2 * pi) + return is_close(rotation_delta, 0) or is_close(rotation_delta, 2 * pi) + def finalize(self, steps: Sequence[SelectedPrimitive]) -> Candidate: """ Try all small solve sets for one raw sequence and return the first match. @@ -874,6 +890,8 @@ class Solver: raise BuildError(f'{self.request.route_name} route requires a jog constraint') constraints.append(('y', float(self.request.jog))) route_constraints = tuple(constraints) + if not self.rotation_matches_request(steps): + raise BuildError(f'{self.request.route_name} composed primitive route is unsupported') adjustable = self.adjustable_indices(steps) solve_sets: list[tuple[int, ...]] = [()] max_solve = min(len(route_constraints), len(adjustable)) diff --git a/masque/test/test_pather_primitive_offers.py b/masque/test/test_pather_primitive_offers.py index ab8facf..06916ae 100644 --- a/masque/test/test_pather_primitive_offers.py +++ b/masque/test/test_pather_primitive_offers.py @@ -422,6 +422,53 @@ def test_pather_selects_lowest_cost_offer() -> None: assert numpy.allclose(p.ports['A'].offset, (-7, 0)) +def test_solver_rejects_rotation_impossible_candidates_before_parameter_solving() -> None: + invalid_parameters: list[float] = [] + + class RotationOfferTool(PlanningOnlyTool): + def primitive_offers( + self, + kind: Literal['straight', 'bend', 's', 'u'], + *, + in_ptype: str | None = None, + out_ptype: str | None = None, + **kwargs: Any, + ) -> tuple[PrimitiveOffer, ...]: + _ = kwargs + if kind != 'straight': + return () + + def invalid_endpoint(length: float) -> Port: + invalid_parameters.append(length) + return Port((length, 0), rotation=0, ptype=out_ptype or in_ptype) + + def valid_endpoint(length: float) -> Port: + return Port((length, 0), rotation=pi, ptype=out_ptype or in_ptype) + + return ( + StraightOffer( + in_ptype=in_ptype, + out_ptype=out_ptype, + endpoint_planner=invalid_endpoint, + commit_planner=lambda length: {'kind': 'invalid', 'length': length}, + ), + StraightOffer( + in_ptype=in_ptype, + out_ptype=out_ptype, + endpoint_planner=valid_endpoint, + commit_planner=lambda length: {'kind': 'valid', 'length': length}, + ), + ) + + p = Pather(Library(), tools=RotationOfferTool(), auto_render=False) + p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') + + p.straight('A', 7) + + assert p._paths['A'][0].data == {'kind': 'valid', 'length': 7} + assert invalid_parameters == [0.0, 0.0] + + def test_pather_commits_only_selected_offer() -> None: committed: list[str] = [] From 9a39a436b23159f72a3bc1266e79ed20ee43ac48 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 12:28:05 -0700 Subject: [PATCH 06/10] [Pather] move render check from __del__ to __exit__ and use render= enum instead of auto_render --- examples/tutorial/README.md | 2 +- examples/tutorial/port_pather.py | 4 +- examples/tutorial/renderpather.py | 4 +- masque/builder/pather.py | 130 ++++++++++++++------ masque/builder/tools.py | 2 +- masque/test/test_autotool_planning.py | 30 ++--- masque/test/test_pather_autotool.py | 4 +- masque/test/test_pather_constraints.py | 32 ++--- masque/test/test_pather_core.py | 34 ++--- masque/test/test_pather_place_plug.py | 12 +- masque/test/test_pather_primitive_offers.py | 12 +- masque/test/test_pather_rendering.py | 123 ++++++++++++++++-- masque/test/test_pather_trace_into.py | 8 +- 13 files changed, 274 insertions(+), 123 deletions(-) diff --git a/examples/tutorial/README.md b/examples/tutorial/README.md index c03e4a0..aab8ad2 100644 --- a/examples/tutorial/README.md +++ b/examples/tutorial/README.md @@ -29,7 +29,7 @@ Contents * Define a custom `Tool` that exposes primitive routing offers * Use primitive offers to automatically transition between path types - [renderpather](renderpather.py) - * Use `Pather(auto_render=False)` and `PathTool` to build a layout similar to the one in [pather](pather.py), + * Use `Pather(render='deferred')` and `PathTool` to build a layout similar to the one in [pather](pather.py), but using `Path` shapes instead of `Polygon`s. - [port_pather](port_pather.py) * Use `PortPather` and the `.at()` syntax for more concise routing diff --git a/examples/tutorial/port_pather.py b/examples/tutorial/port_pather.py index 48e948e..8dabf82 100644 --- a/examples/tutorial/port_pather.py +++ b/examples/tutorial/port_pather.py @@ -13,7 +13,7 @@ def main() -> None: library, M1_tool, M2_tool = prepare_tools() # Create a deferred Pather and place some initial pads (same as Pather tutorial) - rpather = Pather(library, tools=M2_tool, auto_render=False) + rpather = Pather(library, tools=M2_tool, render='deferred') rpather.place('pad', offset=(18_000, 30_000), port_map={'wire_port': 'VCC'}) rpather.place('pad', offset=(18_000, 60_000), port_map={'wire_port': 'GND'}) @@ -157,7 +157,7 @@ def main() -> None: # # Rendering and Saving # - # Since we deferred auto-rendering, we must call .render() to generate the geometry. + # Since routing is deferred, we must call .render() to generate the geometry. rpather.render() library['PortPather_Tutorial'] = rpather.pattern diff --git a/examples/tutorial/renderpather.py b/examples/tutorial/renderpather.py index 01fd352..d048d65 100644 --- a/examples/tutorial/renderpather.py +++ b/examples/tutorial/renderpather.py @@ -13,7 +13,7 @@ def main() -> None: # # To illustrate deferred routing with `Pather`, we use `PathTool` instead # of `AutoTool`. `PathTool` lacks some sophistication (e.g. no automatic transitions) - # but when used with `Pather(auto_render=False)`, it can consolidate multiple routing steps into + # but when used with `Pather(render='deferred')`, it can consolidate multiple routing steps into # a single `Path` shape. # # We'll try to nearly replicate the layout from the `Pather` tutorial; see `pather.py` @@ -39,7 +39,7 @@ def main() -> None: # and what port type to present. M1_ptool = PathTool(layer='M1', width=M1_WIDTH, ptype='m1wire') M2_ptool = PathTool(layer='M2', width=M2_WIDTH, ptype='m2wire') - rpather = Pather(tools=M2_ptool, library=library, auto_render=False) + rpather = Pather(tools=M2_ptool, library=library, render='deferred') # As in the pather tutorial, we make some pads and labels... rpather.place('pad', offset=(18_000, 30_000), port_map={'wire_port': 'VCC'}) diff --git a/masque/builder/pather.py b/masque/builder/pather.py index 1206d60..89bff0e 100644 --- a/masque/builder/pather.py +++ b/masque/builder/pather.py @@ -3,7 +3,7 @@ Unified Pattern assembly and routing (`Pather`). `Pather` is the public object that owns layout state. It holds the working Pattern, Library, per-port Tool assignments, pending `RenderStep`s, and routing -side effects such as plug consumption, port renames, and auto-render. The +side effects such as plug consumption, port renames, and automatic rendering. The planner package is intentionally internal: custom route generators should extend `Tool.primitive_offers()` and `Tool.render()` rather than depending on planner classes or search details. @@ -16,7 +16,7 @@ Routing is split into four ownership phases: `RenderStep.data`, and returns prepared actions, - application: `Pather` appends the prepared steps to its pending queue, replaces live output ports, consumes plug destinations, applies deferred - trace-thru renames, and batches auto-render around the whole route, + trace-thru renames, and batches immediate rendering around the whole route, - rendering: pending steps are grouped by live port, Tool, and continuity before `Tool.render()` builds geometry for each compatible batch. @@ -27,7 +27,7 @@ prepared actions are applied, later commit, plug, rename, render, or insertion failures may leave partial output; that mutation boundary belongs to Pather, not to Tool implementations or the route solver. """ -from typing import Self, Any, overload +from typing import Self, Any, Literal, overload from collections.abc import Iterator, Iterable, Mapping, MutableMapping, Sequence import copy import logging @@ -36,6 +36,7 @@ from functools import wraps from pprint import pformat from contextlib import contextmanager from itertools import chain +from types import TracebackType import numpy from numpy import pi @@ -64,6 +65,8 @@ from .logging import PatherLogger logger = logging.getLogger(__name__) +RenderPolicy = Literal['auto', 'immediate', 'deferred', 'warn', 'error', 'ignore'] +RENDER_POLICIES: tuple[RenderPolicy, ...] = ('auto', 'immediate', 'deferred', 'warn', 'error', 'ignore') class Pather(PortList): @@ -76,9 +79,9 @@ class Pather(PortList): pattern, and a set of `Tool`s for generating routing segments. Routing operations (`trace`, `jog`, `uturn`, etc.) select primitive offers - from the active `Tool` and compose them into deferred `RenderStep`s. - Set `auto_render=False` to defer geometry generation until an explicit call - to `render()`. + from the active `Tool` and compose them into `RenderStep`s. By default, + geometry is rendered after each route, unless the `Pather` is used as a + context manager. Examples: Creating a Pather =========================== @@ -94,12 +97,12 @@ class Pather(PortList): connects port 'A' of the current pattern to port 'C' of `subdevice`. - `pather.trace('my_port', ccw=True, length=100)` plans a 100-unit bend - starting at 'my_port'. Geometry is added immediately by default. - Set `auto_render=False` to defer and call `pather.render()` later. + starting at 'my_port'. If the `Pather` is used as a context manager, + geometry is generated on clean context exit. """ __slots__ = ( 'pattern', 'library', 'tools', 'planner', '_paths', - '_dead', '_logger', '_auto_render', '_auto_render_append' + '_dead', '_logger', '_render_policy', '_render_append', '_context_depth' ) pattern: Pattern @@ -128,8 +131,14 @@ class Pather(PortList): _logger: PatherLogger """ Handles diagnostic logging of operations """ - _auto_render: bool - """ If True, routing operations call render() immediately """ + _render_policy: RenderPolicy + """ Routing render behavior """ + + _render_append: bool + """ If True, automatic render calls append geometry instead of adding references """ + + _context_depth: int + """ Number of active context-manager entries """ _paths: defaultdict[str, list[RenderStep]] """ Per-port pending render steps, consumed by `render()` """ @@ -168,8 +177,8 @@ class Pather(PortList): tools: Tool | MutableMapping[str | None, Tool] | None = None, name: str | None = None, debug: bool = False, - auto_render: bool = True, - auto_render_append: bool = True, + render: RenderPolicy = 'auto', + render_append: bool = True, planner: RoutingPlanner | None = None, ) -> None: """ @@ -181,16 +190,26 @@ class Pather(PortList): tools: Tool(s) to use for routing segments. name: If specified, `library[name]` is set to `self.pattern`. debug: If True, enables detailed logging. - auto_render: If True, enables immediate rendering of routing steps. - auto_render_append: If `auto_render` is True, determines whether - to append geometry or add a reference. + render: Routing render policy. `'auto'` renders after each route + outside a context manager and defers until clean context exit + inside one. Use `'immediate'` to always render after each route, + `'deferred'` to keep paths pending until `render()` or clean + context exit, `'warn'` to log pending paths on clean context + exit, `'error'` to reject pending paths on clean context exit, + or `'ignore'` to leave pending paths silent. + render_append: If an automatic render is triggered, determines + whether to append geometry or add a reference. planner: Optional stateless route-selection planner. If omitted, a new `RoutingPlanner` is used. """ + if render not in RENDER_POLICIES: + raise BuildError(f'Invalid render policy {render!r}; expected one of {RENDER_POLICIES}') + self._dead = False self._logger = PatherLogger(debug=debug) - self._auto_render = auto_render - self._auto_render_append = auto_render_append + self._render_policy = render + self._render_append = render_append + self._context_depth = 0 self.library = library self.pattern = pattern if pattern is not None else Pattern() self.planner = RoutingPlanner() if planner is None else planner @@ -213,9 +232,40 @@ class Pather(PortList): if name is not None: library[name] = self.pattern - def __del__(self) -> None: - if any(self._paths.values()): - logger.warning(f'Pather {self} had pending render steps', stack_info=True) + def __enter__(self) -> Self: + self._context_depth += 1 + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + _ = exc_value, traceback + self._context_depth -= 1 + if exc_type is not None or self._context_depth != 0 or not any(self._paths.values()): + return False + + if self._render_policy in ('auto', 'deferred'): + self.render(append=self._render_append) + elif self._render_policy == 'warn': + logger.warning( + 'Pather context exited with %s; call render() or use render="deferred"', + self._pending_render_summary(), + ) + elif self._render_policy == 'error': + raise BuildError(f'Pather context exited with {self._pending_render_summary()}') + return False + + def _pending_render_summary(self) -> str: + ports = [(portspec, len(steps)) for portspec, steps in self._paths.items() if steps] + port_count = len(ports) + step_count = sum(count for _portspec, count in ports) + return ( + f'{step_count} pending render step{"s" if step_count != 1 else ""} ' + f'on {port_count} port{"s" if port_count != 1 else ""}' + ) def __repr__(self) -> str: s = f'' @@ -397,30 +447,28 @@ class Pather(PortList): Apply every action and deferred rename in a prepared route result. Route actions may contain several primitive render steps and port - mutations. Temporarily disabling auto-render prevents each step from - becoming its own rendered cell. + mutations. Immediate rendering happens once after the whole prepared + result has been applied. """ - saved_auto_render = self._auto_render - self._auto_render = False - try: - for action in result.actions: - if not action.render_steps: - raise BuildError('Prepared route action has no render steps') + for action in result.actions: + if not action.render_steps: + raise BuildError('Prepared route action has no render steps') - if not self._dead: - self._paths[action.portspec].extend(action.render_steps) + if not self._dead: + self._paths[action.portspec].extend(action.render_steps) - self.pattern.ports[action.portspec] = action.final_port.copy() + self.pattern.ports[action.portspec] = action.final_port.copy() - if action.plug_into is not None: - self.plugged({action.portspec: action.plug_into}) - for old_name, new_name in result.renames: - self.rename_ports({old_name: new_name}) - self._auto_render = saved_auto_render - if saved_auto_render and any(self._paths.values()): - self.render(append = self._auto_render_append) - finally: - self._auto_render = saved_auto_render + if action.plug_into is not None: + self.plugged({action.portspec: action.plug_into}) + for old_name, new_name in result.renames: + self.rename_ports({old_name: new_name}) + render_immediately = ( + self._render_policy == 'immediate' + or (self._render_policy == 'auto' and self._context_depth == 0) + ) + if render_immediately and any(self._paths.values()): + self.render(append=self._render_append) def _apply_dead_fallback( self, diff --git a/masque/builder/tools.py b/masque/builder/tools.py index e4569bd..24ca9d1 100644 --- a/masque/builder/tools.py +++ b/masque/builder/tools.py @@ -560,7 +560,7 @@ class RenderStep: """ A single deferred routing operation. - `Pather(auto_render=False)` stores these records while routing and later + `Pather(render='deferred')` stores these records while routing and later passes batches of compatible steps to `Tool.render()` when `Pather.render()` is called. """ diff --git a/masque/test/test_autotool_planning.py b/masque/test/test_autotool_planning.py index db7e8f4..1ae17b9 100644 --- a/masque/test/test_autotool_planning.py +++ b/masque/test/test_autotool_planning.py @@ -122,7 +122,7 @@ def test_autotool_straight_offer_supports_requested_output_transition( ) -> None: _p, tool, lib = autotool_setup - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["start"] = Port((0, 0), pi, ptype="wire_m1") p.straight("start", 10, out_ptype="wire_m2") @@ -158,7 +158,7 @@ def test_pather_straight_topology_allows_transition_offset_cancellation() -> Non .add_transition(lib.abstract("trans_out"), "EXT", "CORE") ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), pi, ptype="ext_in") p.trace("A", None, length=10, out_ptype="ext_out") @@ -550,7 +550,7 @@ def test_autotool_s_offer_bbox_matches_rendered_sbend() -> None: assert_offer_bbox_matches_rendered_offer(tool, offer, 4, lib) - pather = Pather(lib, tools=tool, auto_render=False) + pather = Pather(lib, tools=tool, render='deferred') pather.ports["A"] = Port((0, 0), 0, ptype="ext") pather.jog("A", 4) @@ -581,7 +581,7 @@ def test_autotool_s_offer_uses_absolute_jog_range_for_both_signs() -> None: assert valid_positive assert valid_negative - pather = Pather(Library(), tools=tool, auto_render=False) + pather = Pather(Library(), tools=tool, render='deferred') pather.ports["A"] = Port((0, 0), 0, ptype="core") pather.jog("A", -4) assert_allclose(pather.ports["A"].offset, [-10, 4]) @@ -623,7 +623,7 @@ def test_autotool_s_offer_singleton_jog_range_includes_both_signs() -> None: assert [offer.endpoint_at(-4).y for offer in offers if offer.jog_domain == (-4.0, -4.0)] == [-4] for jog, expected_y in ((4, -4), (-4, 4)): - pather = Pather(Library(), tools=tool, auto_render=False) + pather = Pather(Library(), tools=tool) pather.ports["A"] = Port((0, 0), 0, ptype="core") pather.jog("A", jog) assert_allclose(pather.ports["A"].offset, [-10, expected_y]) @@ -669,7 +669,7 @@ def test_autotool_uturn_offer_mirror_exposes_both_signs() -> None: def test_pather_autotool_uses_prebaked_uturn_offer() -> None: tool, lib = make_uturn_tool() - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="core") p.uturn("A", 4, length=10) @@ -695,7 +695,7 @@ def test_pather_autotool_omitted_uturn_composes_l_offers( multi_bend_tool: tuple[AutoTool, Library], ) -> None: tool, lib = multi_bend_tool - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="wire") p.uturn("A", 20) @@ -714,7 +714,7 @@ def test_pather_autotool_explicit_uturn_composes_l_offers( multi_bend_tool: tuple[AutoTool, Library], ) -> None: tool, lib = multi_bend_tool - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="wire") p.uturn("A", 20, length=10) @@ -739,7 +739,7 @@ def test_autotool_offer_bbox_rejects_invalid_parameter(multi_bend_tool: tuple[Au def test_pather_autotool_uses_l_offer_domains(multi_bend_tool: tuple[AutoTool, Library]) -> None: tool, lib = multi_bend_tool - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="wire") p.trace("A", True, length=15) @@ -760,7 +760,7 @@ def test_autotool_generated_primitives_do_not_capture_route_kwargs() -> None: return make_straight(length, ptype="wire") tool = AutoTool().add_straight("wire", make_marked_straight, "A", length_range=(0, 1e8)) - p = Pather(Library(), tools=tool, auto_render=False) + p = Pather(Library(), tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="wire") p.straight("A", 5, marker="route") @@ -790,7 +790,7 @@ def test_autotool_bend_offer_supports_requested_output_transition(ccw: bool) -> .add_transition(lib.abstract("out_trans"), "EXT", "CORE") ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="core") p.trace("A", ccw, length=10, out_ptype="ext") @@ -824,7 +824,7 @@ def test_autotool_bend_offer_supports_bend_input_transition(ccw: bool) -> None: .add_transition(lib.abstract("bend_trans"), "MID", "CORE") ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="core") p.trace("A", ccw, length=10, out_ptype="mid") @@ -857,7 +857,7 @@ def test_pather_accepts_bend_offer_with_zero_lateral_endpoint() -> None: .add_transition(lib.abstract("bend_trans"), "MID", "CORE") ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="core") p.trace("A", True, length=10, out_ptype="mid") @@ -897,7 +897,7 @@ def test_autotool_bend_offer_supports_bend_and_output_transitions(ccw: bool) -> .add_transition(lib.abstract("out_trans"), "EXT", "MID") ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="core") p.trace("A", ccw, length=12, out_ptype="ext") @@ -955,7 +955,7 @@ def test_pather_autotool_pure_sbend_with_transition_dx() -> None: assert isinstance(s_data, AutoTool.GeneratedData) assert s_offer.out_ptype == "core" - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.ports["A"] = Port((0, 0), 0, ptype="ext") p.jog("A", 4) diff --git a/masque/test/test_pather_autotool.py b/masque/test/test_pather_autotool.py index 98be1a1..e5928ab 100644 --- a/masque/test/test_pather_autotool.py +++ b/masque/test/test_pather_autotool.py @@ -70,7 +70,7 @@ def test_autotool_uturn() -> None: .add_bend(lib.abstract('bend'), 'in', 'out', clockwise=True) ) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), 0) p.at('A').uturn(offset=-2000, length=1000) @@ -82,7 +82,7 @@ def test_autotool_uturn() -> None: def test_deferred_render_autotool_double_L(multi_bend_tool: tuple[AutoTool, Library]) -> None: tool, lib = multi_bend_tool - rp = Pather(lib, tools=tool, auto_render=False) + rp = Pather(lib, tools=tool) rp.ports["A"] = Port((0,0), 0, ptype="wire") rp.jog("A", 10, length=20) diff --git a/masque/test/test_pather_constraints.py b/masque/test/test_pather_constraints.py index e1d77ba..fab1c68 100644 --- a/masque/test/test_pather_constraints.py +++ b/masque/test/test_pather_constraints.py @@ -117,7 +117,7 @@ class CountingPathTool(PathTool): def test_pather_jog_failed_two_bend_route_is_atomic() -> None: lib = Library() tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool) + p = Pather(lib, tools=tool, render='immediate') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(BuildError, match='S-bend'): @@ -130,7 +130,7 @@ def test_pather_jog_failed_two_bend_route_is_atomic() -> None: def test_pather_jog_accepts_sub_width_offset_when_length_is_sufficient() -> None: lib = Library() tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool) + p = Pather(lib, tools=tool, render='immediate') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.jog('A', 1.5, length=5) @@ -139,10 +139,10 @@ def test_pather_jog_accepts_sub_width_offset_when_length_is_sufficient() -> None assert p.pattern.ports['A'].rotation == 0 assert len(p._paths['A']) == 0 -def test_pather_auto_render_batches_multi_step_selected_route_once() -> None: +def test_pather_immediate_render_batches_multi_step_selected_route_once() -> None: lib = Library() tool = CountingPathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool, auto_render=True) + p = Pather(lib, tools=tool, render='immediate') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.jog('A', 4, length=10) @@ -169,7 +169,7 @@ def test_pather_jog_length_solved_from_single_position_bound() -> None: def test_pather_positional_bound_requires_port_rotation() -> None: - p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire'), auto_render=False) + p = Pather(Library(), tools=PathTool(layer='M1', width=1, ptype='wire'), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=None, ptype='wire') with pytest.raises(BuildError, match='Ports must have rotation'): @@ -179,7 +179,7 @@ def test_pather_positional_bound_requires_port_rotation() -> None: def test_pather_jog_omitted_length_uses_minimum_length_route() -> None: lib = Library() tool = PathTool(layer='M1', width=1, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.jog('A', 2) @@ -195,7 +195,7 @@ def test_pather_jog_omitted_length_uses_minimum_length_route() -> None: def test_pather_trace_omitted_length_uses_minimum_offer() -> None: lib = Library() tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.trace('A', None) @@ -211,7 +211,7 @@ def test_pather_trace_omitted_length_uses_minimum_offer() -> None: def test_pather_trace_to_without_bound_uses_single_port_trace_minimum() -> None: lib = Library() tool = PathTool(layer='M1', width=2, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.trace_to('A', False) @@ -229,7 +229,7 @@ def test_pather_trace_to_rejects_conflicting_position_bounds() -> None: with pytest.raises(BuildError, match='exactly one positional bound'): p.trace_to('A', None, **kwargs) - p = Pather(Library(), tools=tool) + p = Pather(Library(), tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(BuildError, match='length cannot be combined'): p.trace_to('A', None, x=-5, length=3) @@ -261,7 +261,7 @@ def test_planner_constrained_bend_requires_jog() -> None: def test_pather_trace_each_plans_all_ports_before_mutation() -> None: tool = FirstPortOnlyTraceTool() - p = Pather(Library(), tools=tool, auto_render=False) + p = Pather(Library(), tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.pattern.ports['B'] = Port((-2, 5), rotation=0, ptype='blocked') @@ -277,7 +277,7 @@ def test_pather_trace_each_plans_all_ports_before_mutation() -> None: def test_pather_bundle_trace_plans_all_ports_before_mutation_or_render() -> None: tool = FirstPortOnlyTraceTool() - p = Pather(Library(), tools=tool, auto_render=True) + p = Pather(Library(), tools=tool, render='immediate') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.pattern.ports['B'] = Port((0, 4), rotation=0, ptype='blocked') @@ -343,7 +343,7 @@ def test_pather_route_commit_failure_is_atomic_for_multi_port_trace() -> None: return tree tool = CommitFailureTool() - p = Pather(Library(), tools=tool, auto_render=True) + p = Pather(Library(), tools=tool, render='immediate') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.pattern.ports['B'] = Port((0, 4), rotation=0, ptype='bad') @@ -433,7 +433,7 @@ def test_pather_uturn_does_not_use_direct_planl_fallback() -> None: jog = -1 return Port((length, jog), rotation=rotation, ptype=in_ptype or 'wire'), {'ccw': ccw, 'length': length} - p = Pather(Library(), tools=PlanLOnlyTool(), auto_render=False) + p = Pather(Library(), tools=PlanLOnlyTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(BuildError, match='No legal primitive offer for omitted-length U-turn'): @@ -476,7 +476,7 @@ def test_pather_su_topology_rejects_out_ptype_sensitive_planl_jog() -> None: ptype = out_ptype or in_ptype or 'wire' return Port((length, jog), rotation=rotation, ptype=ptype), {'ccw': ccw, 'length': length} - p = Pather(Library(), tools=OutPtypeSensitiveTool(), auto_render=False) + p = Pather(Library(), tools=OutPtypeSensitiveTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises((BuildError, NotImplementedError)): @@ -509,7 +509,7 @@ def test_pather_two_l_planl_only_uturn_is_not_supported() -> None: jog = -1 return Port((length, jog), rotation=rotation, ptype=in_ptype or 'wire'), {'ccw': ccw, 'length': length} - p = Pather(Library(), tools=PlanLOnlyTool(), auto_render=False) + p = Pather(Library(), tools=PlanLOnlyTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises((BuildError, NotImplementedError)): @@ -543,7 +543,7 @@ def test_pather_two_l_planl_only_jog_is_not_supported() -> None: jog = -1 return Port((length, jog), rotation=rotation, ptype=in_ptype or 'wire'), {'ccw': ccw, 'length': length} - p = Pather(Library(), tools=PlanLOnlyTool(), auto_render=False) + p = Pather(Library(), tools=PlanLOnlyTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises((BuildError, NotImplementedError)): diff --git a/masque/test/test_pather_core.py b/masque/test/test_pather_core.py index 96162af..ddf869b 100644 --- a/masque/test/test_pather_core.py +++ b/masque/test/test_pather_core.py @@ -62,7 +62,7 @@ def test_builder_tool_symbol_exports() -> None: def test_pather_pending_render_steps_are_private() -> None: - p = Pather(Library(), tools=PathTool(layer=(1, 0), width=1), auto_render=False) + p = Pather(Library(), tools=PathTool(layer=(1, 0), width=1)) assert not hasattr(p, 'paths') assert hasattr(p, '_paths') @@ -81,7 +81,7 @@ def test_pather_accepts_and_reuses_planner_instance() -> None: p = Pather( Library(), tools=PathTool(layer=(1, 0), width=1), - auto_render=False, + render='deferred', planner=planner, ) p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') @@ -148,7 +148,7 @@ def test_pather_dead_ports() -> None: def test_pather_trace_basic() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) # Routing extends opposite the port's inward-facing rotation. p.pattern.ports['A'] = Port((0, 0), rotation=0) @@ -164,7 +164,7 @@ def test_pather_trace_basic() -> None: def test_pather_trace_to() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) @@ -177,7 +177,7 @@ def test_pather_trace_to() -> None: def test_pather_bundle_trace() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((0, 2000), rotation=0) @@ -194,13 +194,13 @@ def test_pather_bundle_trace() -> None: def test_portpather_default_spacing_matches_explicit_spacing() -> None: lib_default = Library() tool_default = PathTool(layer='M1', width=1000) - p_default = Pather(lib_default, tools=tool_default, auto_render=False) + p_default = Pather(lib_default, tools=tool_default) p_default.pattern.ports['A'] = Port((0, 0), rotation=0) p_default.pattern.ports['B'] = Port((0, 2000), rotation=0) lib_explicit = Library() tool_explicit = PathTool(layer='M1', width=1000) - p_explicit = Pather(lib_explicit, tools=tool_explicit, auto_render=False) + p_explicit = Pather(lib_explicit, tools=tool_explicit) p_explicit.pattern.ports['A'] = Port((0, 0), rotation=0) p_explicit.pattern.ports['B'] = Port((0, 2000), rotation=0) @@ -211,11 +211,11 @@ def test_portpather_default_spacing_matches_explicit_spacing() -> None: assert_allclose(p_default.pattern.ports['B'].offset, p_explicit.pattern.ports['B'].offset) def test_portpather_default_spacing_reused_and_overridden() -> None: - p_default = Pather(Library(), tools=PathTool(layer='M1', width=1000), auto_render=False) + p_default = Pather(Library(), tools=PathTool(layer='M1', width=1000)) p_default.pattern.ports['A'] = Port((0, 0), rotation=0) p_default.pattern.ports['B'] = Port((0, 2000), rotation=0) - p_explicit = Pather(Library(), tools=PathTool(layer='M1', width=1000), auto_render=False) + p_explicit = Pather(Library(), tools=PathTool(layer='M1', width=1000)) p_explicit.pattern.ports['A'] = Port((0, 0), rotation=0) p_explicit.pattern.ports['B'] = Port((0, 2000), rotation=0) @@ -227,11 +227,11 @@ def test_portpather_default_spacing_reused_and_overridden() -> None: assert_allclose(p_default.pattern.ports['A'].offset, p_explicit.pattern.ports['A'].offset) assert_allclose(p_default.pattern.ports['B'].offset, p_explicit.pattern.ports['B'].offset) - p_override = Pather(Library(), tools=PathTool(layer='M1', width=1000), auto_render=False) + p_override = Pather(Library(), tools=PathTool(layer='M1', width=1000)) p_override.pattern.ports['A'] = Port((0, 0), rotation=0) p_override.pattern.ports['B'] = Port((0, 2000), rotation=0) - p_override_explicit = Pather(Library(), tools=PathTool(layer='M1', width=1000), auto_render=False) + p_override_explicit = Pather(Library(), tools=PathTool(layer='M1', width=1000)) p_override_explicit.pattern.ports['A'] = Port((0, 0), rotation=0) p_override_explicit.pattern.ports['B'] = Port((0, 2000), rotation=0) @@ -244,7 +244,7 @@ def test_portpather_default_spacing_reused_and_overridden() -> None: def test_portpather_default_spacing_not_injected_for_straight_bundle() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((0, 2000), rotation=0) @@ -256,7 +256,7 @@ def test_portpather_default_spacing_not_injected_for_straight_bundle() -> None: def test_portpather_default_spacing_vector_revalidated_after_selection_change() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((0, 2000), rotation=0) p.pattern.ports['C'] = Port((0, 4000), rotation=0) @@ -270,7 +270,7 @@ def test_portpather_default_spacing_vector_revalidated_after_selection_change() def test_pather_each_bound() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((-1000, 2000), rotation=0) @@ -390,7 +390,7 @@ def test_rename() -> None: def test_pather_dead_fallback_preserves_out_ptype() -> None: lib = Library() tool = PathTool(layer='M1', width=1000, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.set_dead() @@ -424,7 +424,7 @@ def test_pather_dead_fallback_does_not_hide_fatal_route_errors() -> None: commit_planner=lambda length: {'length': length}, ),) - p = Pather(Library(), tools=MismatchedEndpointTool(layer='M1', width=1000, ptype='wire'), auto_render=False) + p = Pather(Library(), tools=MismatchedEndpointTool(layer='M1', width=1000, ptype='wire')) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.set_dead() @@ -472,7 +472,7 @@ def test_pather_valid_candidate_does_not_hide_fatal_route_errors() -> None: ), ) - p = Pather(Library(), tools=PartlyMismatchedEndpointTool(layer='M1', width=1000, ptype='wire'), auto_render=False) + p = Pather(Library(), tools=PartlyMismatchedEndpointTool(layer='M1', width=1000, ptype='wire')) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(BuildError, match='does not match declared offer out_ptype'): diff --git a/masque/test/test_pather_place_plug.py b/masque/test/test_pather_place_plug.py index 11e9bd2..660d080 100644 --- a/masque/test/test_pather_place_plug.py +++ b/masque/test/test_pather_place_plug.py @@ -10,7 +10,7 @@ from masque.error import PortError, PatternError def test_pather_place_treeview_resolves_once() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool) + p = Pather(lib, tools=tool, render='deferred') tree = {'child': Pattern(ports={'B': Port((1, 0), pi)})} @@ -24,7 +24,7 @@ def test_pather_place_treeview_resolves_once() -> None: def test_pather_plug_treeview_resolves_once() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0) tree = {'child': Pattern(ports={'B': Port((0, 0), pi)})} @@ -39,7 +39,7 @@ def test_pather_plug_treeview_resolves_once() -> None: def test_pather_failed_plug_does_not_add_break_marker() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.annotations = {'k': [1]} p.pattern.ports['A'] = Port((0, 0), rotation=0) @@ -60,7 +60,7 @@ def test_pather_failed_plug_does_not_add_break_marker() -> None: def test_pather_place_reused_deleted_name_keeps_break_marker() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0) p.at('A').straight(5000) @@ -80,7 +80,7 @@ def test_pather_place_reused_deleted_name_keeps_break_marker() -> None: def test_pather_plug_reused_deleted_name_keeps_break_marker() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((0, 0), rotation=0) @@ -107,7 +107,7 @@ def test_pather_plug_reused_deleted_name_keeps_break_marker() -> None: def test_pather_failed_plugged_does_not_add_break_marker() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool, render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0) p.at('A').straight(5000) diff --git a/masque/test/test_pather_primitive_offers.py b/masque/test/test_pather_primitive_offers.py index 06916ae..7d99355 100644 --- a/masque/test/test_pather_primitive_offers.py +++ b/masque/test/test_pather_primitive_offers.py @@ -360,7 +360,7 @@ def test_pather_treats_notimplemented_offer_query_as_no_offers() -> None: _ = kind, in_ptype, out_ptype, kwargs raise NotImplementedError - p = Pather(Library(), tools=NoOfferTool(), auto_render=False) + p = Pather(Library(), tools=NoOfferTool()) p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(BuildError, match='No legal primitive offer for trace'): @@ -381,7 +381,7 @@ def test_pather_propagates_offer_query_errors(err_type: type[Exception]) -> None _ = kind, in_ptype, out_ptype, kwargs raise err_type('offer query failed') - p = Pather(Library(), tools=BrokenTool(), auto_render=False) + p = Pather(Library(), tools=BrokenTool()) p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') with pytest.raises(err_type): @@ -413,7 +413,7 @@ def test_pather_selects_lowest_cost_offer() -> None: StraightOffer(in_ptype=in_ptype, out_ptype=out_ptype, priority_bias=0, **offer_callbacks(low)), ) - p = Pather(Library(), tools=MultiOfferTool(), auto_render=False) + p = Pather(Library(), tools=MultiOfferTool(), render='deferred') p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 7) @@ -460,7 +460,7 @@ def test_solver_rejects_rotation_impossible_candidates_before_parameter_solving( ), ) - p = Pather(Library(), tools=RotationOfferTool(), auto_render=False) + p = Pather(Library(), tools=RotationOfferTool(), render='deferred') p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 7) @@ -503,7 +503,7 @@ def test_pather_commits_only_selected_offer() -> None: return (make('expensive', 100), make('cheap', 0)) - p = Pather(Library(), tools=RecordingTool(), auto_render=False) + p = Pather(Library(), tools=RecordingTool(), render='deferred') p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 5) @@ -553,7 +553,7 @@ def test_pather_routes_with_offer_only_s_and_u_tool() -> None: ),) return () - p = Pather(Library(), tools=OfferOnlyTool(), auto_render=False) + p = Pather(Library(), tools=OfferOnlyTool(), render='deferred') p.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.jog('A', 3) p.uturn('A', 4) diff --git a/masque/test/test_pather_rendering.py b/masque/test/test_pather_rendering.py index f539a51..c5ddd68 100644 --- a/masque/test/test_pather_rendering.py +++ b/masque/test/test_pather_rendering.py @@ -1,4 +1,5 @@ from typing import TYPE_CHECKING, cast +import logging import pytest import numpy @@ -20,10 +21,27 @@ if TYPE_CHECKING: def deferred_render_setup() -> tuple[Pather, PathTool, Library]: lib = Library() tool = PathTool(layer=(1, 0), width=2, ptype="wire") - rp = Pather(lib, tools=tool, auto_render=False) + rp = Pather(lib, tools=tool, render='deferred') rp.ports["start"] = Port((0, 0), pi / 2, ptype="wire") return rp, tool, lib + +def add_pending_straight(pather: Pather) -> None: + pather.ports["start"] = Port((0, 0), pi / 2, ptype="wire") + pather.straight("start", 10) + + +def route_in_context(pather: Pather) -> None: + with pather: + add_pending_straight(pather) + + +def fail_in_context(pather: Pather) -> None: + with pather: + add_pending_straight(pather) + raise RuntimeError('body failed') + + def test_deferred_render_stores_pending_paths_until_render(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: rp, tool, lib = deferred_render_setup rp.at("start").straight(10).straight(10) @@ -101,7 +119,7 @@ def test_portpather_translate_only_affects_future_steps(deferred_render_setup: t def test_deferred_render_dead_ports() -> None: lib = Library() tool = PathTool(layer=(1, 0), width=1) - rp = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool, auto_render=False) + rp = Pather(lib, ports={"in": Port((0, 0), 0)}, tools=tool, render='deferred') rp.set_dead() rp.straight("in", -10) @@ -113,6 +131,91 @@ def test_deferred_render_dead_ports() -> None: rp.render() assert not rp.pattern.has_shapes() + +def test_pather_default_auto_policy_renders_immediately_outside_context() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + p = Pather(lib, tools=tool) + p.ports["start"] = Port((0, 0), pi / 2, ptype="wire") + + p.straight("start", 10) + + assert p.pattern.has_shapes() + assert not any(p._paths.values()) + + +def test_pather_default_auto_policy_defers_until_clean_context_exit() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + + with Pather(lib, tools=tool) as p: + add_pending_straight(p) + assert not p.pattern.has_shapes() + assert len(p._paths["start"]) == 1 + + assert p.pattern.has_shapes() + assert not any(p._paths.values()) + + +def test_pather_context_warn_policy_logs_pending_paths(caplog: pytest.LogCaptureFixture) -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + + with caplog.at_level(logging.WARNING, logger="masque.builder.pather"), Pather(lib, tools=tool, render='warn') as p: + add_pending_straight(p) + + records = [ + record + for record in caplog.records + if 'Pather context exited with 1 pending render step on 1 port' in record.getMessage() + ] + assert len(records) == 1 + assert records[0].stack_info is None + assert len(p._paths["start"]) == 1 + assert not p.pattern.has_shapes() + + +def test_pather_context_error_policy_rejects_pending_paths() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + p = Pather(lib, tools=tool, render='error') + + with pytest.raises(BuildError, match='1 pending render step on 1 port'): + route_in_context(p) + + assert len(p._paths["start"]) == 1 + assert not p.pattern.has_shapes() + + +def test_pather_context_ignore_policy_leaves_pending_paths_silent(caplog: pytest.LogCaptureFixture) -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + + with caplog.at_level(logging.WARNING, logger="masque.builder.pather"), Pather(lib, tools=tool, render='ignore') as p: + add_pending_straight(p) + + assert not caplog.records + assert len(p._paths["start"]) == 1 + assert not p.pattern.has_shapes() + + +def test_pather_context_policy_does_not_mask_body_exception() -> None: + lib = Library() + tool = PathTool(layer=(1, 0), width=2, ptype="wire") + p = Pather(lib, tools=tool, render='error') + + with pytest.raises(RuntimeError, match='body failed'): + fail_in_context(p) + + assert len(p._paths["start"]) == 1 + assert not p.pattern.has_shapes() + + +def test_pather_rejects_invalid_render_policy() -> None: + with pytest.raises(BuildError, match='Invalid render policy'): + Pather(Library(), tools=PathTool(layer=(1, 0), width=2), render='later') # type: ignore[arg-type] + + def test_deferred_render_rename_port(deferred_render_setup: tuple[Pather, PathTool, Library]) -> None: rp, tool, lib = deferred_render_setup rp.at("start").straight(10) @@ -174,7 +277,7 @@ def test_pathtool_s_offer_render_geometry_matches_ports() -> None: def test_deferred_render_uturn_fallback() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - rp = Pather(lib, tools=tool, auto_render=False) + rp = Pather(lib, tools=tool, render='deferred') rp.pattern.ports['A'] = Port((0, 0), rotation=0) rp.at('A').uturn(offset=10000, length=5000) @@ -213,7 +316,7 @@ def test_pather_render_auto_renames_single_use_tool_children() -> None: return tree lib = Library() - p = Pather(lib, tools=FullTreeTool(), auto_render=False) + p = Pather(lib, tools=FullTreeTool()) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -258,7 +361,7 @@ def test_custom_tool_render_preserves_segment_subtrees() -> None: return batch[0].data lib = Library() - p = Pather(lib, tools=TraceTreeTool(), auto_render=False) + p = Pather(lib, tools=TraceTreeTool()) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -293,7 +396,7 @@ def test_pather_render_rejects_missing_single_use_tool_refs() -> None: lib = Library() lib['_seg'] = Pattern(annotations={'stale': [1]}) - p = Pather(lib, tools=MissingSingleUseTool(), auto_render=False) + p = Pather(lib, tools=MissingSingleUseTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -329,7 +432,7 @@ def test_pather_render_allows_missing_non_single_use_tool_refs() -> None: lib = Library() lib['shared'] = Pattern(annotations={'shared': [1]}) - p = Pather(lib, tools=SharedRefTool(), auto_render=False) + p = Pather(lib, tools=SharedRefTool()) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -362,7 +465,7 @@ def test_pather_render_rejects_output_port_that_misses_planned_endpoint(append: return tree lib = Library() - p = Pather(lib, tools=WrongEndpointTool(), auto_render=False) + p = Pather(lib, tools=WrongEndpointTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -396,7 +499,7 @@ def test_pather_render_rejects_output_port_with_wrong_ptype(append: bool) -> Non return tree lib = Library() - p = Pather(lib, tools=WrongPtypeTool(), auto_render=False) + p = Pather(lib, tools=WrongPtypeTool(), render='deferred') p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.straight('A', 10) @@ -409,7 +512,7 @@ def test_pather_render_rejects_output_port_with_wrong_ptype(append: bool) -> Non def test_deferred_render_rename_to_none_keeps_pending_geometry_without_port() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - rp = Pather(lib, tools=tool, auto_render=False) + rp = Pather(lib, tools=tool, render='deferred') rp.pattern.ports['A'] = Port((0, 0), rotation=0) rp.at('A').straight(5000) diff --git a/masque/test/test_pather_trace_into.py b/masque/test/test_pather_trace_into.py index 8f66b28..ed967dd 100644 --- a/masque/test/test_pather_trace_into.py +++ b/masque/test/test_pather_trace_into.py @@ -16,7 +16,7 @@ from masque.error import BuildError, PortError def trace_into_setup() -> tuple[Pather, PathTool, Library]: lib = Library() tool = PathTool(layer=(1, 0), width=2, ptype="wire") - p = Pather(lib, tools=tool, auto_render=True, auto_render_append=False) + p = Pather(lib, tools=tool, render='immediate', render_append=False) return p, tool, lib @@ -71,7 +71,7 @@ def test_path_into_thru(trace_into_setup: tuple[Pather, PathTool, Library]) -> N def test_pather_trace_into_shapes() -> None: lib = Library() tool = PathTool(layer='M1', width=1000) - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0) p.pattern.ports['B'] = Port((-10000, 0), rotation=pi) @@ -175,7 +175,7 @@ def test_pather_trace_into_refines_output_adapter_route_before_plug() -> None: pat.add_port_pair(names=port_names, ptype=batch[-1].end_port.ptype if batch else 'm1wire') return tree - p = Pather(Library(), tools=TransitionTool(), auto_render=False) + p = Pather(Library(), tools=TransitionTool(), render='deferred') p.pattern.ports['src'] = Port((-65000, -11500), rotation=pi / 2, ptype='m1wire') p.pattern.ports['dst'] = Port((-100000, -100000), rotation=pi, ptype='m2wire') @@ -188,7 +188,7 @@ def test_pather_trace_into_refines_output_adapter_route_before_plug() -> None: def test_pather_trace_into_dead_updates_ports_without_geometry() -> None: lib = Library() tool = PathTool(layer='M1', width=1000, ptype='wire') - p = Pather(lib, tools=tool, auto_render=False) + p = Pather(lib, tools=tool) p.pattern.ports['A'] = Port((0, 0), rotation=0, ptype='wire') p.pattern.ports['B'] = Port((-10000, 0), rotation=pi, ptype='wire') p.set_dead() From 4c64352152e2a38b56a564882a0567d996b3b6b0 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 13:05:01 -0700 Subject: [PATCH 07/10] [OverlayLibrary / PortsLibraryView] move into masque.library --- masque/__init__.py | 2 + masque/file/gdsii_lazy.py | 27 +- masque/file/gdsii_lazy_arrow.py | 25 +- masque/file/gdsii_lazy_core.py | 550 +-------------------------- masque/library.py | 510 ++++++++++++++++++++++++- masque/test/test_gdsii_lazy.py | 10 +- masque/test/test_gdsii_lazy_arrow.py | 8 +- 7 files changed, 558 insertions(+), 574 deletions(-) diff --git a/masque/__init__.py b/masque/__init__.py index f64b180..b5dddde 100644 --- a/masque/__init__.py +++ b/masque/__init__.py @@ -63,6 +63,8 @@ from .library import ( ILibrary as ILibrary, LibraryView as LibraryView, Library as Library, + OverlayLibrary as OverlayLibrary, + PortsLibraryView as PortsLibraryView, BuildLibrary as BuildLibrary, BuildReport as BuildReport, CellProvenance as CellProvenance, diff --git a/masque/file/gdsii_lazy.py b/masque/file/gdsii_lazy.py index b587c89..a0b6e6f 100644 --- a/masque/file/gdsii_lazy.py +++ b/masque/file/gdsii_lazy.py @@ -8,7 +8,7 @@ This module provides the non-Arrow half of Masque's lazy GDS architecture: - cells are materialized on demand through the classic `gdsii` decoder whenever a caller indexes the lazy view - the source can be wrapped in `PortsLibraryView` or merged through - `OverlayLibrary`, both of which live in `gdsii_lazy_core` + `OverlayLibrary` The public surface intentionally parallels `gdsii_lazy_arrow` closely so that callers can swap between the classic and Arrow-backed implementations with @@ -17,9 +17,8 @@ minimal changes. from __future__ import annotations from dataclasses import dataclass -from typing import IO, Any, cast +from typing import IO, TYPE_CHECKING, Any, cast from collections import defaultdict -from collections.abc import Iterator, Sequence import gzip import io import logging @@ -28,17 +27,27 @@ import pathlib import klamath import numpy -from numpy.typing import NDArray from klamath import records from . import gdsii from .utils import is_gzipped -from .gdsii_lazy_core import OverlayLibrary, PortsLibraryView, _pattern_children, write, writefile +from .gdsii_lazy_core import write as write, writefile as writefile from ..error import LibraryError -from ..library import ILibraryView, LibraryView, dangling_mode_t -from ..pattern import Pattern +from ..library import ( + ILibraryView, + LibraryView, + PortsLibraryView, + dangling_mode_t, +) from ..utils import apply_transforms +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + from numpy.typing import NDArray + + from ..pattern import Pattern + logger = logging.getLogger(__name__) @@ -76,7 +85,7 @@ def _open_source_stream( with gzip.open(path, mode='rb') as stream: data = stream.read() return _SourceHandle(path=path, stream=io.BytesIO(data)) - stream = cast('IO[bytes]', gzip.open(path, mode='rb')) + stream = cast('IO[bytes]', gzip.open(path, mode='rb')) # noqa: SIM115 return _SourceHandle(path=path, stream=stream) if use_mmap: @@ -216,7 +225,7 @@ class GdsLibrarySource(ILibraryView): graph: dict[str, set[str]] = {} for name in self._cell_order: if name in self._cache: - graph[name] = _pattern_children(self._cache[name]) + graph[name] = {child for child, refs in self._cache[name].refs.items() if child is not None and refs} else: graph[name] = self._raw_children(name) diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii_lazy_arrow.py index f1dcc32..a53c1ff 100644 --- a/masque/file/gdsii_lazy_arrow.py +++ b/masque/file/gdsii_lazy_arrow.py @@ -7,25 +7,34 @@ keeps its current behavior and performance profile. from __future__ import annotations from dataclasses import dataclass -from typing import IO, Any, cast +from typing import IO, TYPE_CHECKING, Any, cast from collections import defaultdict -from collections.abc import Iterator, Sequence import gzip import logging import mmap import pathlib import numpy -from numpy.typing import NDArray -import pyarrow from . import gdsii_arrow from .utils import is_gzipped -from .gdsii_lazy_core import OverlayLibrary, PortsLibraryView, _pattern_children, write, writefile -from ..library import ILibraryView, LibraryView, dangling_mode_t -from ..pattern import Pattern +from .gdsii_lazy_core import write as write, writefile as writefile +from ..library import ( + ILibraryView, + LibraryView, + PortsLibraryView, + dangling_mode_t, +) from ..utils import apply_transforms +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + from numpy.typing import NDArray + import pyarrow + + from ..pattern import Pattern + logger = logging.getLogger(__name__) @@ -343,7 +352,7 @@ class ArrowLibrary(ILibraryView): graph: dict[str, set[str]] = {} for name in self._payload.cell_order: if name in self._cache: - graph[name] = _pattern_children(self._cache[name]) + graph[name] = {child for child, refs in self._cache[name].refs.items() if child is not None and refs} else: graph[name] = self._raw_children(name) diff --git a/masque/file/gdsii_lazy_core.py b/masque/file/gdsii_lazy_core.py index 88c4eed..d10a1a9 100644 --- a/masque/file/gdsii_lazy_core.py +++ b/masque/file/gdsii_lazy_core.py @@ -1,558 +1,34 @@ """ -Shared helpers for source-backed lazy GDS views. +GDS write helpers for source-backed lazy GDS views. -This module contains the reusable pieces that sit between lazy source readers -and ordinary mutable library usage: - -- `PortsLibraryView` layers a processed, ports-importing cache on top of a raw - source view without mutating the source itself -- `OverlayLibrary` exposes a mutable library surface that can mix source-backed - cells with overlay-owned materialized patterns -- the write helpers preserve source-backed copy-through behavior where - possible, falling back to normal pattern serialization when a cell has been - materialized or remapped - -Both the classic and Arrow-backed lazy GDS readers rely on these helpers. +The generic mutable overlay and ports-importing view live in `masque.library`. +This module preserves source-backed GDS copy-through behavior where possible, +falling back to normal pattern serialization when a cell has been materialized +or remapped. """ from __future__ import annotations -from dataclasses import dataclass -from typing import IO, Any, Literal, cast -from collections import defaultdict -from collections.abc import Callable, Iterator, Mapping, Sequence -import copy +from typing import IO, TYPE_CHECKING, Any, cast import gzip import logging import pathlib import klamath -import numpy -from numpy.typing import NDArray from . import gdsii from .utils import tmpfile from ..error import LibraryError -from ..library import ILibrary, ILibraryView, LibraryView, _plan_source_names, _source_rename_map, dangling_mode_t -from ..pattern import Pattern, map_targets -from ..utils import apply_transforms -from ..utils.ports2data import data_to_ports +from ..library import ILibraryView, OverlayLibrary, _SourceEntry, _materialize_detached_pattern + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + + from ..pattern import Pattern logger = logging.getLogger(__name__) -@dataclass -class _SourceLayer: - """ One imported source layer tracked by an `OverlayLibrary`. """ - library: ILibraryView - source_to_visible: dict[str, str] - visible_to_source: dict[str, str] - child_graph: dict[str, set[str]] - order: list[str] - - -@dataclass(frozen=True) -class _SourceEntry: - """ Reference to a single visible source-backed cell in an overlay. """ - layer_index: int - source_name: str - - -def _pattern_children(pat: Pattern) -> set[str]: - return {child for child, refs in pat.refs.items() if child is not None and refs} - - -def _remap_pattern_targets(pat: Pattern, remap: Callable[[str | None], str | None]) -> Pattern: - if not pat.refs: - return pat - pat.refs = map_targets(pat.refs, remap) - return pat - - -def _coerce_library_view(source: Mapping[str, Pattern] | ILibraryView) -> ILibraryView: - if isinstance(source, ILibraryView): - return source - return LibraryView(source) - - -def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: - func = getattr(view, '_materialize_pattern', None) - if callable(func): - return cast('Pattern', func(name, persist=False)) - return view[name].deepcopy() - - -class PortsLibraryView(ILibraryView): - """ - Read-only view which imports ports into cells on first materialization. - - The wrapped source remains untouched; this view owns a separate processed - cache so direct-copy workflows can continue to use the raw source view. - - Graph queries, source ordering, and copy-through capabilities are delegated - to the wrapped source whenever possible, while `__getitem__` and - `materialize_many()` return port-imported patterns. - """ - - def __init__( - self, - source: ILibraryView, - *, - layers: Sequence[gdsii.layer_t], - max_depth: int = 0, - skip_subcells: bool = True, - ) -> None: - self._source = source - self._layers = tuple(layers) - self._max_depth = max_depth - self._skip_subcells = skip_subcells - self._cache: dict[str, Pattern] = {} - self._lookups_in_progress: list[str] = [] - if hasattr(source, 'library_info'): - self.library_info = cast('dict[str, Any]', getattr(source, 'library_info')) - - def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) - - def __iter__(self) -> Iterator[str]: - return iter(self._source) - - def __len__(self) -> int: - return len(self._source) - - def __contains__(self, key: object) -> bool: - return key in self._source - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: - if name in self._cache: - return self._cache[name] - - if name in self._lookups_in_progress: - chain = ' -> '.join(self._lookups_in_progress + [name]) - raise LibraryError( - f'Detected circular reference or recursive lookup of "{name}".\n' - f'Lookup chain: {chain}\n' - 'This may be caused by an invalid (cyclical) reference, or buggy code.' - ) - - self._lookups_in_progress.append(name) - try: - pat = _materialize_detached_pattern(self._source, name) - pat = data_to_ports( - layers=self._layers, - library=self, - pattern=pat, - name=name, - max_depth=self._max_depth, - skip_subcells=self._skip_subcells, - ) - finally: - self._lookups_in_progress.pop() - - if persist: - self._cache[name] = pat - return pat - - def materialize_many( - self, - names: Sequence[str], - *, - persist: bool = True, - ) -> LibraryView: - mats = { - name: self._materialize_pattern(name, persist=persist) - for name in dict.fromkeys(names) - } - return LibraryView(mats) - - def source_order(self) -> tuple[str, ...]: - return self._source.source_order() - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._source.child_graph(dangling=dangling) - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._source.parent_graph(dangling=dangling) - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) - keep |= set(tops) - return self.materialize_many(tuple(keep), persist=True) - - def tops(self) -> list[str]: - return self._source.tops() - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - finder = getattr(self._source, 'find_refs_local', None) - if callable(finder): - return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) - return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - finder = getattr(self._source, 'find_refs_global', None) - if callable(finder): - return cast( - 'dict[tuple[str, ...], NDArray[numpy.float64]]', - finder(name, order=order, parent_graph=parent_graph, dangling=dangling), - ) - return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) - - def raw_struct_bytes(self, name: str) -> bytes: - reader = getattr(self._source, 'raw_struct_bytes', None) - if not callable(reader): - raise AttributeError('raw_struct_bytes') - return cast('bytes', reader(name)) - - def can_copy_raw_struct(self, name: str) -> bool: - can_copy = getattr(self._source, 'can_copy_raw_struct', None) - if not callable(can_copy): - return False - return bool(can_copy(name)) - - def close(self) -> None: - closer = getattr(self._source, 'close', None) - if callable(closer): - closer() - - def __enter__(self) -> PortsLibraryView: - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - -class OverlayLibrary(ILibrary): - """ - Mutable overlay over one or more source libraries. - - Source-backed cells remain lazy until accessed through `__getitem__`, at - which point that visible cell is promoted into an overlay-owned materialized - `Pattern`. - - This is the main mutable integration surface for lazy GDS content. It lets - callers: - - expose one or more source-backed libraries behind a normal `ILibrary` - interface - - add or replace cells with overlay-owned patterns - - rename visible source cells - - remap references without immediately rewriting untouched source structs - """ - - def __init__(self) -> None: - self._layers: list[_SourceLayer] = [] - self._entries: dict[str, Pattern | _SourceEntry] = {} - self._order: list[str] = [] - self._target_remap: dict[str, str] = {} - - def __iter__(self) -> Iterator[str]: - return (name for name in self._order if name in self._entries) - - def __len__(self) -> int: - return len(self._entries) - - def __contains__(self, key: object) -> bool: - return key in self._entries - - def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) - - def __setitem__( - self, - key: str, - value: Pattern | Callable[[], Pattern], - ) -> None: - if key in self._entries: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - pattern = value() if callable(value) else value - self._entries[key] = pattern - if key not in self._order: - self._order.append(key) - - def __delitem__(self, key: str) -> None: - if key not in self._entries: - raise KeyError(key) - del self._entries[key] - - def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: - self[key_self] = copy.deepcopy(other[key_other]) - - def add_source( - self, - source: Mapping[str, Pattern] | ILibraryView, - *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, - rename_when: Literal['conflict', 'always'] = 'conflict', - ) -> dict[str, str]: - """ - Add a source-backed library layer. - - Args: - rename_theirs: Function used to choose visible names for imported - source cells. - rename_when: If `'conflict'`, only conflicting names are renamed. - If `'always'`, every imported source name is passed through - `rename_theirs`. - """ - view = _coerce_library_view(source) - source_order = list(view.source_order()) - child_graph = view.child_graph(dangling='include') - - source_to_visible = _plan_source_names( - self, - source_order, - self._entries, - rename_theirs = rename_theirs, - rename_when = rename_when, - ) - visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()} - - layer = _SourceLayer( - library=view, - source_to_visible=source_to_visible, - visible_to_source=visible_to_source, - child_graph=child_graph, - order=[source_to_visible[name] for name in source_order], - ) - layer_index = len(self._layers) - self._layers.append(layer) - - for source_name, visible_name in source_to_visible.items(): - self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) - if visible_name not in self._order: - self._order.append(visible_name) - - return _source_rename_map(source_to_visible) - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> OverlayLibrary: - if old_name not in self._entries: - raise LibraryError(f'"{old_name}" does not exist in the library.') - if old_name == new_name: - return self - if new_name in self._entries: - raise LibraryError(f'"{new_name}" already exists in the library.') - - entry = self._entries.pop(old_name) - self._entries[new_name] = entry - if isinstance(entry, _SourceEntry): - layer = self._layers[entry.layer_index] - layer.source_to_visible[entry.source_name] = new_name - del layer.visible_to_source[old_name] - layer.visible_to_source[new_name] = entry.source_name - - idx = self._order.index(old_name) - self._order[idx] = new_name - - if move_references: - self.move_references(old_name, new_name) - return self - - def _resolve_target(self, target: str) -> str: - seen: set[str] = set() - current = target - while current in self._target_remap: - if current in seen: - raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') - seen.add(current) - current = self._target_remap[current] - return current - - def _set_target_remap(self, old_target: str, new_target: str) -> None: - resolved_new = self._resolve_target(new_target) - if resolved_new == old_target: - raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') - self._target_remap[old_target] = resolved_new - for key in list(self._target_remap): - self._target_remap[key] = self._resolve_target(self._target_remap[key]) - - def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: - if old_target == new_target: - return self - self._set_target_remap(old_target, new_target) - for entry in list(self._entries.values()): - if isinstance(entry, Pattern) and old_target in entry.refs: - entry.refs[new_target].extend(entry.refs[old_target]) - del entry.refs[old_target] - return self - - def _effective_target(self, layer: _SourceLayer, target: str) -> str: - visible = layer.source_to_visible.get(target, target) - return self._resolve_target(visible) - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: - if name not in self._entries: - raise KeyError(name) - entry = self._entries[name] - if isinstance(entry, Pattern): - return entry - - layer = self._layers[entry.layer_index] - source_pat = layer.library[entry.source_name].deepcopy() - remap = lambda target: None if target is None else self._effective_target(layer, target) - pat = _remap_pattern_targets(source_pat, remap) - if persist: - self._entries[name] = pat - return pat - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - graph: dict[str, set[str]] = {} - for name in self._order: - if name not in self._entries: - continue - entry = self._entries[name] - if isinstance(entry, Pattern): - graph[name] = _pattern_children(entry) - continue - layer = self._layers[entry.layer_index] - children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} - graph[name] = children - - existing = set(graph) - dangling_refs = set().union(*(children - existing for children in graph.values())) - if dangling == 'error': - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') - return graph - if dangling == 'ignore': - return {name: {child for child in children if child in existing} for name, children in graph.items()} - - for child in dangling_refs: - graph.setdefault(cast('str', child), set()) - return graph - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - return LibraryView({name: self[name] for name in keep}) - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) - if parent_graph is None: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return instances - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') - if dangling == 'ignore': - return instances - - for parent in parent_graph.get(name, set()): - pat = self._materialize_pattern(parent, persist=False) - for ref in pat.refs.get(name, []): - instances[parent].append(ref.as_transforms()) - return instances - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - result[(parent,) + path] = instances - return result - - def source_order(self) -> tuple[str, ...]: - return tuple(name for name in self._order if name in self._entries) - - def _iter_library_infos(library: Mapping[str, Pattern] | ILibraryView) -> Iterator[dict[str, Any]]: info = getattr(library, 'library_info', None) if isinstance(info, dict): @@ -598,7 +74,7 @@ def _can_copy_raw_cell(library: Mapping[str, Pattern] | ILibraryView, name: str) def _raw_struct_bytes(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bytes: reader = getattr(library, 'raw_struct_bytes', None) if not callable(reader): - raise AttributeError('raw_struct_bytes') + raise TypeError('raw_struct_bytes') return cast('bytes', reader(name)) diff --git a/masque/library.py b/masque/library.py index bcc23ce..0a8ae73 100644 --- a/masque/library.py +++ b/masque/library.py @@ -14,7 +14,7 @@ Classes include: - `AbstractView`: Provides a way to use []-indexing to generate abstracts for patterns in the linked library. Generated with `ILibraryView.abstract_view()`. """ -from typing import Self, TYPE_CHECKING, Any, cast, TypeAlias, Protocol, Literal +from typing import Self, Any, cast, TypeAlias, Protocol, Literal from collections.abc import Container, Iterator, Mapping, MutableMapping, Sequence, Callable import logging import re @@ -35,11 +35,7 @@ from .utils import layer_t, apply_transforms from .shapes import Shape, Polygon from .label import Label from .abstract import Abstract -from .pattern import map_layers - -if TYPE_CHECKING: - from .pattern import Pattern - +from .pattern import Pattern, map_layers, map_targets logger = logging.getLogger(__name__) @@ -903,7 +899,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): An `Abstract` or `Pattern` object. """ from .pattern import Pattern #noqa: PLC0415 - if not isinstance(other, (str, Abstract, Pattern)): + if not isinstance(other, str | Abstract | Pattern): # We got a TreeView; add it into self and grab its topcell as an Abstract other = self << other @@ -1491,6 +1487,500 @@ class Library(ILibrary): return tree, pat +@dataclass +class _SourceLayer: + """ One imported source layer tracked by an `OverlayLibrary`. """ + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + """ Reference to a single visible source-backed cell in an overlay. """ + layer_index: int + source_name: str + + +def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: + func = getattr(view, '_materialize_pattern', None) + if callable(func): + return cast('Pattern', func(name, persist=False)) + return view[name].deepcopy() + + +class PortsLibraryView(ILibraryView): + """ + Read-only view which imports ports into cells on first materialization. + + The wrapped source remains untouched; this view owns a separate processed + cache so direct-copy workflows can continue to use the raw source view. + + Graph queries, source ordering, and copy-through capabilities are delegated + to the wrapped source whenever possible, while `__getitem__` and + `materialize_many()` return port-imported patterns. + """ + + def __init__( + self, + source: ILibraryView, + *, + layers: Sequence[layer_t], + max_depth: int = 0, + skip_subcells: bool = True, + ) -> None: + self._source = source + self._layers = tuple(layers) + self._max_depth = max_depth + self._skip_subcells = skip_subcells + self._cache: dict[str, Pattern] = {} + self._lookups_in_progress: list[str] = [] + if hasattr(source, 'library_info'): + self.library_info = cast('dict[str, Any]', source.library_info) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._source) + + def __len__(self) -> int: + return len(self._source) + + def __contains__(self, key: object) -> bool: + return key in self._source + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + from .utils.ports2data import data_to_ports # noqa: PLC0415 + + if name in self._cache: + return self._cache[name] + + if name in self._lookups_in_progress: + chain = ' -> '.join(self._lookups_in_progress + [name]) + raise LibraryError( + f'Detected circular reference or recursive lookup of "{name}".\n' + f'Lookup chain: {chain}\n' + 'This may be caused by an invalid (cyclical) reference, or buggy code.' + ) + + self._lookups_in_progress.append(name) + try: + pat = _materialize_detached_pattern(self._source, name) + pat = data_to_ports( + layers=self._layers, + library=self, + pattern=pat, + name=name, + max_depth=self._max_depth, + skip_subcells=self._skip_subcells, + ) + finally: + self._lookups_in_progress.pop() + + if persist: + self._cache[name] = pat + return pat + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = { + name: self._materialize_pattern(name, persist=persist) + for name in dict.fromkeys(names) + } + return LibraryView(mats) + + def source_order(self) -> tuple[str, ...]: + return self._source.source_order() + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.child_graph(dangling=dangling) + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.parent_graph(dangling=dangling) + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + return self._source.tops() + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + finder = getattr(self._source, 'find_refs_local', None) + if callable(finder): + return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) + return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + finder = getattr(self._source, 'find_refs_global', None) + if callable(finder): + return cast( + 'dict[tuple[str, ...], NDArray[numpy.float64]]', + finder(name, order=order, parent_graph=parent_graph, dangling=dangling), + ) + return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) + + def raw_struct_bytes(self, name: str) -> bytes: + reader = getattr(self._source, 'raw_struct_bytes', None) + if not callable(reader): + raise TypeError('raw_struct_bytes') + return cast('bytes', reader(name)) + + def can_copy_raw_struct(self, name: str) -> bool: + can_copy = getattr(self._source, 'can_copy_raw_struct', None) + if not callable(can_copy): + return False + return bool(can_copy(name)) + + def close(self) -> None: + closer = getattr(self._source, 'close', None) + if callable(closer): + closer() + + def __enter__(self) -> 'PortsLibraryView': + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_when: Literal['conflict', 'always'] = 'conflict', + ) -> dict[str, str]: + """ + Add a source-backed library layer. + + Args: + rename_theirs: Function used to choose visible names for imported + source cells. + rename_when: If `'conflict'`, only conflicting names are renamed. + If `'always'`, every imported source name is passed through + `rename_theirs`. + """ + view = source if isinstance(source, ILibraryView) else LibraryView(source) + source_order = list(view.source_order()) + child_graph = view.child_graph(dangling='include') + + source_to_visible = _plan_source_names( + self, + source_order, + self._entries, + rename_theirs = rename_theirs, + rename_when = rename_when, + ) + visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()} + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return _source_rename_map(source_to_visible) + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> 'OverlayLibrary': + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> 'OverlayLibrary': + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + + def remap(target: str | None) -> str | None: + return None if target is None else self._effective_target(layer, target) + + if source_pat.refs: + source_pat.refs = map_targets(source_pat.refs, remap) + pat = source_pat + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = {child for child, refs in entry.refs.items() if child is not None and refs} + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) + + @dataclass class _BuildRecipe: """ Captured deferred call to a pattern factory. """ @@ -1903,8 +2393,6 @@ class _BuildSessionLibrary(ILibrary): """ def __init__(self, builder: BuildLibrary) -> None: - from .file.gdsii_lazy_core import OverlayLibrary # noqa: PLC0415 - self._builder = builder self._overlay = OverlayLibrary() self._built: set[str] = set() @@ -2107,13 +2595,13 @@ class _BuildSessionLibrary(ILibrary): self._ensure_named(dep) pattern = declaration.func(*declaration.args, **declaration.kwargs) if not isinstance(pattern, Pattern): - raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') + raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') # noqa: TRY301 else: pattern = declaration.deepcopy() if name in self._overlay: if self._overlay[name] is not pattern: - raise BuildError( + raise BuildError( # noqa: TRY301 f'Recipe for "{name}" wrote a different pattern into the session under its own name.' ) else: diff --git a/masque/test/test_gdsii_lazy.py b/masque/test/test_gdsii_lazy.py index 0134d6e..3abaab8 100644 --- a/masque/test/test_gdsii_lazy.py +++ b/masque/test/test_gdsii_lazy.py @@ -6,7 +6,7 @@ from numpy.testing import assert_allclose from ..file import gdsii, gdsii_lazy from ..pattern import Pattern -from ..library import Library +from ..library import Library, OverlayLibrary def _make_lazy_port_library() -> Library: @@ -73,7 +73,7 @@ def test_gdsii_lazy_overlay_add_source_stays_lazy_for_processed_view(tmp_path: P raw, _ = gdsii_lazy.readfile(gds_file) processed = raw.with_ports_from_data(layers=[(10, 0)], max_depth=2) - overlay = gdsii_lazy.OverlayLibrary() + overlay = OverlayLibrary() overlay.add_source(processed) assert not raw._cache @@ -85,7 +85,7 @@ def test_gdsii_lazy_overlay_add_source_stays_lazy_for_processed_view(tmp_path: P def test_gdsii_lazy_overlay_add_source_can_rename_every_source_cell() -> None: src = _make_lazy_port_library() - overlay = gdsii_lazy.OverlayLibrary() + overlay = OverlayLibrary() rename_map = overlay.add_source( src, @@ -106,10 +106,10 @@ def test_gdsii_lazy_overlay_add_source_rename_when_validation() -> None: src = _make_lazy_port_library() with pytest.raises(TypeError, match='rename_theirs'): - gdsii_lazy.OverlayLibrary().add_source(src, rename_when='always') + OverlayLibrary().add_source(src, rename_when='always') with pytest.raises(ValueError, match='rename mode'): - gdsii_lazy.OverlayLibrary().add_source(src, rename_when='sometimes') # type: ignore[arg-type] + OverlayLibrary().add_source(src, rename_when='sometimes') # type: ignore[arg-type] def test_gdsii_lazy_processed_write_roundtrips_without_explicit_units(tmp_path: Path) -> None: diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index eb62721..0984b20 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -10,7 +10,7 @@ import pytest pytest.importorskip('pyarrow') from .. import PatternError -from ..library import Library +from ..library import Library, OverlayLibrary from ..pattern import Pattern from ..repetition import Grid from ..file import gdsii, gdsii_lazy_arrow @@ -155,7 +155,7 @@ def test_gdsii_lazy_arrow_local_and_global_refs(tmp_path: Path) -> None: assert sum(arr.shape[0] for arr in local['mid']) == 5 global_refs = lib.find_refs_global('leaf') - assert {path for path in global_refs} == {('top', 'mid', 'leaf')} + assert set(global_refs) == {('top', 'mid', 'leaf')} assert global_refs[('top', 'mid', 'leaf')].shape[0] == 5 @@ -265,7 +265,7 @@ def test_gdsii_lazy_overlay_merge_and_write(tmp_path: Path) -> None: lib_a, _ = gdsii_lazy_arrow.readfile(gds_a) lib_b, _ = gdsii_lazy_arrow.readfile(gds_b) - overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay = OverlayLibrary() overlay.add_source(lib_a) rename_map = overlay.add_source(lib_b, rename_theirs=lambda lib, name: lib.get_name(name)) renamed_leaf = rename_map['leaf'] @@ -293,7 +293,7 @@ def test_gdsii_writer_accepts_overlay_library(tmp_path: Path) -> None: lib, info = gdsii_lazy_arrow.readfile(gds_file) - overlay = gdsii_lazy_arrow.OverlayLibrary() + overlay = OverlayLibrary() overlay.add_source(lib) overlay.rename('leaf', 'leaf_copy', move_references=True) From 02a3708b302b68ec8996e2c8c4eead5cc1255c59 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 13:17:36 -0700 Subject: [PATCH 08/10] [library] Split library.py into multiple files --- masque/file/gdsii_lazy_core.py | 3 +- masque/library.py | 2841 -------------------------------- masque/library/__init__.py | 29 + masque/library/base.py | 1259 ++++++++++++++ masque/library/build.py | 742 +++++++++ masque/library/lazy.py | 169 ++ masque/library/mapping.py | 112 ++ masque/library/overlay.py | 515 ++++++ masque/library/utils.py | 124 ++ 9 files changed, 2952 insertions(+), 2842 deletions(-) delete mode 100644 masque/library.py create mode 100644 masque/library/__init__.py create mode 100644 masque/library/base.py create mode 100644 masque/library/build.py create mode 100644 masque/library/lazy.py create mode 100644 masque/library/mapping.py create mode 100644 masque/library/overlay.py create mode 100644 masque/library/utils.py diff --git a/masque/file/gdsii_lazy_core.py b/masque/file/gdsii_lazy_core.py index d10a1a9..988f9a4 100644 --- a/masque/file/gdsii_lazy_core.py +++ b/masque/file/gdsii_lazy_core.py @@ -18,7 +18,8 @@ import klamath from . import gdsii from .utils import tmpfile from ..error import LibraryError -from ..library import ILibraryView, OverlayLibrary, _SourceEntry, _materialize_detached_pattern +from ..library import ILibraryView, OverlayLibrary +from ..library.overlay import _SourceEntry, _materialize_detached_pattern if TYPE_CHECKING: from collections.abc import Iterator, Mapping diff --git a/masque/library.py b/masque/library.py deleted file mode 100644 index 0a8ae73..0000000 --- a/masque/library.py +++ /dev/null @@ -1,2841 +0,0 @@ -""" -Library classes for managing unique name->pattern mappings and deferred loading or execution. - -Classes include: -- `ILibraryView`: Defines a general interface for read-only name->pattern mappings. -- `LibraryView`: An implementation of `ILibraryView` backed by an arbitrary `Mapping`. - Can be used to wrap any arbitrary `Mapping` to give it all the functionality in `ILibraryView` -- `ILibrary`: Defines a general interface for mutable name->pattern mappings. -- `Library`: An implementation of `ILibrary` backed by an arbitrary `MutableMapping`. - Can be used to wrap any arbitrary `MutableMapping` to give it all the functionality in `ILibrary`. - By default, uses a `dict` as the underylingmapping. -- `LazyLibrary`: An implementation of `ILibrary` which enables on-demand loading or generation - of patterns. -- `AbstractView`: Provides a way to use []-indexing to generate abstracts for patterns in the linked - library. Generated with `ILibraryView.abstract_view()`. -""" -from typing import Self, Any, cast, TypeAlias, Protocol, Literal -from collections.abc import Container, Iterator, Mapping, MutableMapping, Sequence, Callable -import logging -import re -import copy -from functools import wraps -from pprint import pformat -from collections import defaultdict -from abc import ABCMeta, abstractmethod -from contextvars import ContextVar -from dataclasses import dataclass, replace -from graphlib import TopologicalSorter, CycleError - -import numpy -from numpy.typing import ArrayLike, NDArray - -from .error import BuildError, LibraryError, PatternError -from .utils import layer_t, apply_transforms -from .shapes import Shape, Polygon -from .label import Label -from .abstract import Abstract -from .pattern import Pattern, map_layers, map_targets - -logger = logging.getLogger(__name__) - -_ACTIVE_BUILD_SESSIONS: ContextVar[dict[int, '_BuildSessionLibrary'] | None] = ContextVar( - 'masque_active_build_sessions', - default=None, -) - - -class visitor_function_t(Protocol): - """ Signature for `Library.dfs()` visitor functions. """ - def __call__( - self, - pattern: 'Pattern', - hierarchy: tuple[str | None, ...], - memo: dict, - transform: NDArray[numpy.float64] | Literal[False], - ) -> 'Pattern': - ... - - -TreeView: TypeAlias = Mapping[str, 'Pattern'] -""" A name-to-`Pattern` mapping which is expected to have only one top-level cell """ - -Tree: TypeAlias = MutableMapping[str, 'Pattern'] -""" A mutable name-to-`Pattern` mapping which is expected to have only one top-level cell """ - -dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include'] -""" How helpers should handle refs whose targets are not present in the library. """ - - -@dataclass(frozen=True) -class CellProvenance: - """ - Provenance record for one cell in a completed build output. - - Each output name in a `BuildReport` maps to one `CellProvenance`. The - record captures both where the cell came from and how its visible name was - chosen. - - Attributes: - requested_name: First name requested for this cell during the build. - kind: Whether the cell came from a declaration, helper emission, or an - imported source library. - owner_declared_name: Declared cell responsible for this output cell, if - any. Imported source cells leave this as `None`. - build_chain: Declared-cell dependency chain that was active when the - cell was emitted. - """ - requested_name: str - kind: Literal['declared', 'helper', 'source'] - owner_declared_name: str | None - build_chain: tuple[str, ...] - - -@dataclass(frozen=True) -class BuildReport: - """ - Immutable summary of one `BuildLibrary.validate()` or `.build()` run. - - The report is designed to answer two questions after a build completes: - which declared cells depended on which other declared cells, and where each - output cell came from. - - Attributes: - requested_roots: Roots explicitly requested for the run. A full - `build()` uses all declared cells. - provenance: Mapping from final output name to provenance metadata. - dependency_graph: Declared-cell dependency graph discovered through - library-mediated reads and explicit recipe hints. - """ - requested_roots: tuple[str, ...] - provenance: Mapping[str, CellProvenance] - dependency_graph: Mapping[str, frozenset[str]] - - -SINGLE_USE_PREFIX = '_' -""" -Names starting with this prefix are assumed to refer to single-use patterns, -which may be renamed automatically by `ILibrary.add()` (via -`rename_theirs=_rename_patterns()` ) -""" -# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere? - - -def _rename_patterns(lib: 'ILibraryView', name: str) -> str: - """ - The default `rename_theirs` function for `ILibrary.add`. - - Treats names starting with `SINGLE_USE_PREFIX` (default: one underscore) as - "one-offs" for which name conflicts should be automatically resolved. - Conflicts are resolved by calling `lib.get_name(SINGLE_USE_PREFIX + stem)` - where `stem = name.removeprefix(SINGLE_USE_PREFIX).split('$')[0]`. - Names lacking the prefix are directly returned (not renamed). - - Args: - lib: The library into which `name` is to be added (but is presumed to conflict) - name: The original name, to be modified - - Returns: - The new name, not guaranteed to be conflict-free! - """ - if not name.startswith(SINGLE_USE_PREFIX): - return name - - stem = name.removeprefix(SINGLE_USE_PREFIX).split('$')[0] - return lib.get_name(SINGLE_USE_PREFIX + stem) - - -def _plan_source_names( - target: 'ILibraryView', - source_order: Sequence[str], - existing_names: Container[str], - *, - rename_theirs: Callable[['ILibraryView', str], str] | None = None, - rename_when: Literal['conflict', 'always'] = 'conflict', - ) -> dict[str, str]: - if rename_when not in ('conflict', 'always'): - raise ValueError(f'Unknown source rename mode: {rename_when!r}') - if rename_when == 'always' and rename_theirs is None: - raise TypeError('rename_theirs is required when rename_when="always"') - - source_to_visible: dict[str, str] = {} - visible_names: set[str] = set() - - for name in source_order: - visible = name - if rename_when == 'always': - assert rename_theirs is not None - visible = rename_theirs(target, name) - elif visible in existing_names or visible in visible_names: - if rename_theirs is None: - raise LibraryError(f'Conflicting name while adding source: {name!r}') - visible = rename_theirs(target, name) - if visible in existing_names or visible in visible_names: - raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') - source_to_visible[name] = visible - visible_names.add(visible) - - return source_to_visible - - -def _source_rename_map(source_to_visible: Mapping[str, str]) -> dict[str, str]: - return { - source_name: visible_name - for source_name, visible_name in source_to_visible.items() - if source_name != visible_name - } - - -class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): - """ - Interface for a read-only library. - - A library is a mapping from unique names (str) to collections of geometry (`Pattern`). - """ - # inherited abstract functions - #def __getitem__(self, key: str) -> 'Pattern': - #def __iter__(self) -> Iterator[str]: - #def __len__(self) -> int: - - #__contains__, keys, items, values, get, __eq__, __ne__ supplied by Mapping - - def __repr__(self) -> str: - return '' - - def abstract_view(self) -> 'AbstractView': - """ - Returns: - An AbstractView into this library - """ - return AbstractView(self) - - def abstract(self, name: str) -> Abstract: - """ - Return an `Abstract` (name & ports) for the pattern in question. - - Args: - name: The pattern name - - Returns: - An `Abstract` object for the pattern - """ - return Abstract(name=name, ports=self[name].ports) - - def source_order(self) -> tuple[str, ...]: - """ - Return names in the library's preferred source order. - - Source-backed views may override this to preserve on-disk ordering - without materializing patterns. - """ - return tuple(self.keys()) - - def dangling_refs( - self, - tops: str | Sequence[str] | None = None, - ) -> set[str | None]: - """ - Get the set of all pattern names not present in the library but referenced - by `tops`, recursively traversing any refs. - - If `tops` are not given, all patterns in the library are checked. - - Args: - tops: Name(s) of the pattern(s) to check. - Default is all patterns in the library. - - Returns: - Set of all referenced pattern names - """ - if tops is None: - tops = tuple(self.keys()) - - referenced = self.referenced_patterns(tops) - return referenced - set(self.keys()) - - def referenced_patterns( - self, - tops: str | Sequence[str] | None = None, - skip: set[str | None] | None = None, - ) -> set[str | None]: - """ - Get the set of all pattern names referenced by `tops`. Recursively traverses into any refs. - - If `tops` are not given, all patterns in the library are checked. - - Args: - tops: Name(s) of the pattern(s) to check. - Default is all patterns in the library. - skip: Memo, set patterns which have already been traversed. - - Returns: - Set of all referenced pattern names - """ - if tops is None: - tops = tuple(self.keys()) - - if skip is None: - skip = {None} - - if isinstance(tops, str): - tops = (tops,) - tops = set(tops) - skip |= tops # don't re-visit tops - - # Get referenced patterns for all tops - targets = set() - for top in set(tops): - targets |= self[top].referenced_patterns() - - # Perform recursive lookups, but only once for each name - for target in targets - skip: - assert target is not None - skip.add(target) - if target in self: - targets |= self.referenced_patterns(target, skip=skip) - - return targets - - def subtree( - self, - tops: str | Sequence[str], - ) -> 'ILibraryView': - """ - Return a new `ILibraryView`, containing only the specified patterns and the patterns they - reference (recursively). - Dangling references do not cause an error. - - Args: - tops: Name(s) of patterns to keep - - Returns: - A `LibraryView` containing only `tops` and the patterns they reference. - """ - if isinstance(tops, str): - tops = (tops,) - - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - - filtered = {kk: vv for kk, vv in self.items() if kk in keep} - new = LibraryView(filtered) - return new - - def polygonize( - self, - num_vertices: int | None = None, - max_arclen: float | None = None, - ) -> Self: - """ - Calls `.polygonize(...)` on each pattern in this library. - Arguments are passed on to `shape.to_polygons(...)`. - - Args: - num_vertices: Number of points to use for each polygon. Can be overridden by - `max_arclen` if that results in more points. Optional, defaults to shapes' - internal defaults. - max_arclen: Maximum arclength which can be approximated by a single line - segment. Optional, defaults to shapes' internal defaults. - - Returns: - self - """ - for pat in self.values(): - pat.polygonize(num_vertices, max_arclen) - return self - - def manhattanize( - self, - grid_x: ArrayLike, - grid_y: ArrayLike, - ) -> Self: - """ - Calls `.manhattanize(grid_x, grid_y)` on each pattern in this library. - - Args: - grid_x: List of allowed x-coordinates for the Manhattanized polygon edges. - grid_y: List of allowed y-coordinates for the Manhattanized polygon edges. - - Returns: - self - """ - for pat in self.values(): - pat.manhattanize(grid_x, grid_y) - return self - - def flatten( - self, - tops: str | Sequence[str], - flatten_ports: bool = False, - dangling_ok: bool = False, - ) -> dict[str, 'Pattern']: - """ - Returns copies of all `tops` patterns with all refs - removed and replaced with equivalent shapes. - Also returns flattened copies of all referenced patterns. - The originals in the calling `Library` are not modified. - For an in-place variant, see `Pattern.flatten`. - - Args: - tops: The pattern(s) to flatten. - flatten_ports: If `True`, keep ports from any referenced - patterns; otherwise discard them. - dangling_ok: If `True`, no error will be thrown if any - ref points to a name which is not present in the library. - Default False. - - Returns: - {name: flat_pattern} mapping for all flattened patterns. - """ - if isinstance(tops, str): - tops = (tops,) - - flattened: dict[str, Pattern | None] = {} - - def flatten_single(name: str) -> None: - flattened[name] = None - pat = self[name].deepcopy() - refs_by_target = tuple((target, tuple(refs)) for target, refs in pat.refs.items()) - - for target, refs in refs_by_target: - if target is None: - continue - if dangling_ok and target not in self: - continue - if target not in flattened: - flatten_single(target) - - target_pat = flattened[target] - if target_pat is None: - raise PatternError(f'Circular reference in {name} to {target}') - ports_only = flatten_ports and bool(target_pat.ports) - if target_pat.is_empty() and not ports_only: # avoid some extra allocations - continue - - for ref in refs: - if flatten_ports and ref.repetition is not None and target_pat.ports: - raise PatternError( - f'Cannot flatten ports from repeated ref to {target!r}; ' - 'flatten with flatten_ports=False or expand/rename the ports manually first.' - ) - p = ref.as_pattern(pattern=target_pat) - if not flatten_ports: - p.ports.clear() - pat.append(p) - - for target in set(pat.refs.keys()) & set(self.keys()): - del pat.refs[target] - - flattened[name] = pat - - for top in tops: - flatten_single(top) - - assert None not in flattened.values() - return cast('dict[str, Pattern]', flattened) - - def get_name( - self, - name: str = SINGLE_USE_PREFIX * 2, - sanitize: bool = True, - max_length: int = 32, - quiet: bool | None = None, - ) -> str: - """ - Find a unique name for the pattern. - - This function may be overridden in a subclass or monkey-patched to fit the caller's requirements. - - Args: - name: Preferred name for the pattern. Default is `SINGLE_USE_PREFIX * 2`. - sanitize: Allows only alphanumeric charaters and _?$. Replaces invalid characters with underscores. - max_length: Names longer than this will be truncated. - quiet: If `True`, suppress log messages. Default `None` suppresses messages only if - the name starts with `SINGLE_USE_PREFIX`. - - Returns: - Name, unique within this library. - """ - if quiet is None: - quiet = name.startswith(SINGLE_USE_PREFIX) - - if sanitize: - # Remove invalid characters - sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name) - else: - sanitized_name = name - - suffixed_name = sanitized_name - if sanitized_name in self: - ii = sum(1 for nn in self.keys() if nn.startswith(sanitized_name)) - else: - ii = 0 - while suffixed_name in self or suffixed_name == '': - suffixed_name = sanitized_name + b64suffix(ii) - ii += 1 - - if len(suffixed_name) > max_length: - if name == '': - raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}') - - cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True) - else: - cropped_name = suffixed_name - - if not quiet: - logger.info(f'Requested name "{name}" changed to "{cropped_name}"') - - return cropped_name - - def tops(self) -> list[str]: - """ - Return the list of all patterns that are not referenced by any other pattern in the library. - - Returns: - A list of pattern names in which no pattern is referenced by any other pattern. - """ - names = set(self.keys()) - not_toplevel: set[str | None] = set() - for name in names: - not_toplevel |= set(self[name].refs.keys()) - - toplevel = list(names - not_toplevel) - return toplevel - - def top(self) -> str: - """ - Return the name of the topcell, or raise an exception if there isn't a single topcell - - Raises: - LibraryError if there is not exactly one topcell. - """ - tops = self.tops() - if len(tops) != 1: - raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}') - return tops[0] - - def top_pattern(self) -> 'Pattern': - """ - Shorthand for self[self.top()] - - Raises: - LibraryError if there is not exactly one topcell. - """ - return self[self.top()] - - @staticmethod - def _dangling_refs_error(dangling: set[str], context: str) -> LibraryError: - dangling_list = sorted(dangling) - return LibraryError(f'Dangling refs found while {context}: ' + pformat(dangling_list)) - - def _raw_child_graph(self) -> tuple[dict[str, set[str]], set[str]]: - existing = set(self.keys()) - graph: dict[str, set[str]] = {} - dangling: set[str] = set() - for name, pat in self.items(): - children = {child for child, refs in pat.refs.items() if child is not None and refs} - graph[name] = children - dangling |= children - existing - return graph, dangling - - def dfs( - self, - pattern: 'Pattern', - visit_before: visitor_function_t | None = None, - visit_after: visitor_function_t | None = None, - *, - hierarchy: tuple[str | None, ...] = (None,), - transform: ArrayLike | bool | None = False, - memo: dict | None = None, - ) -> Self: - """ - Convenience function. - Performs a depth-first traversal of a pattern and its referenced patterns. - At each pattern in the tree, the following sequence is called: - ``` - current_pattern = visit_before(current_pattern, **vist_args) - for target in current_pattern.refs: - for ref in pattern.refs[target]: - self.dfs(target, visit_before, visit_after, - hierarchy + (sp.target,), updated_transform, memo) - current_pattern = visit_after(current_pattern, **visit_args) - ``` - where `visit_args` are - `hierarchy`: (top_pattern_or_None, L1_pattern, L2_pattern, ..., parent_pattern, target_pattern) - tuple of all parent-and-higher pattern names. Top pattern name may be - `None` if not provided in first call to .dfs() - `transform`: numpy.ndarray containing cumulative - [x_offset, y_offset, rotation (rad), mirror_x (0 or 1)] - for the instance being visited - `memo`: Arbitrary dict (not altered except by `visit_before()` and `visit_after()`) - - Args: - pattern: Pattern object to start at ("top"/root node of the tree). - visit_before: Function to call before traversing refs. - Should accept a `Pattern` and `**visit_args`, and return the (possibly modified) - pattern. Default `None` (not called). - visit_after: Function to call after traversing refs. - Should accept a `Pattern` and `**visit_args`, and return the (possibly modified) - pattern. Default `None` (not called). - transform: Initial value for `visit_args['transform']`. - Can be `False`, in which case the transform is not calculated. - `True` or `None` is interpreted as `[0, 0, 0, 0]`. - memo: Arbitrary dict for use by `visit_*()` functions. Default `None` (empty dict). - hierarchy: Tuple of patterns specifying the hierarchy above the current pattern. - Default is (None,), which will be used as a placeholder for the top pattern's - name if not overridden. - - Returns: - self - """ - if memo is None: - memo = {} - - if transform is None or transform is True: - transform = numpy.array([0, 0, 0, 0, 1], dtype=float) - elif transform is not False: - transform = numpy.asarray(transform, dtype=float) - if transform.size == 4: - transform = numpy.append(transform, 1.0) - - original_pattern = pattern - - if visit_before is not None: - pattern = visit_before(pattern, hierarchy=hierarchy, memo=memo, transform=transform) - - for target in pattern.refs: - if target is None: - continue - if target in hierarchy: - raise LibraryError(f'.dfs() called on pattern with circular reference to "{target}"') - - for ref in pattern.refs[target]: - ref_transforms: list[bool] | NDArray[numpy.float64] - if transform is not False: - ref_transforms = apply_transforms(transform, ref.as_transforms()) - else: - ref_transforms = [False] - - for ref_transform in ref_transforms: - self.dfs( - pattern = self[target], - visit_before = visit_before, - visit_after = visit_after, - hierarchy = hierarchy + (target,), - transform = ref_transform, - memo = memo, - ) - - if visit_after is not None: - pattern = visit_after(pattern, hierarchy=hierarchy, memo=memo, transform=transform) - - if pattern is not original_pattern: - name = hierarchy[-1] - if not isinstance(self, ILibrary): - raise LibraryError('visit_* functions returned a new `Pattern` object' - ' but the library is immutable') - if name is None: - # The top pattern is not the original pattern, but we don't know what to call it! - raise LibraryError('visit_* functions returned a new `Pattern` object' - ' but no top-level name was provided in `hierarchy`') - - del cast('ILibrary', self)[name] - cast('ILibrary', self)[name] = pattern - - return self - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - """ - Return a mapping from pattern name to a set of all child patterns - (patterns it references). - - Only non-empty ref lists with non-`None` targets are treated as graph edges. - - Args: - dangling: How refs to missing targets are handled. `'error'` raises, - `'ignore'` drops those edges, and `'include'` exposes them as - synthetic leaf nodes. - - Returns: - Mapping from pattern name to a set of all pattern names it references. - """ - graph, dangling_refs = self._raw_child_graph() - if dangling == 'error': - if dangling_refs: - raise self._dangling_refs_error(dangling_refs, 'building child graph') - return graph - if dangling == 'ignore': - existing = set(graph) - return {name: {child for child in children if child in existing} for name, children in graph.items()} - - for target in dangling_refs: - graph.setdefault(target, set()) - return graph - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - """ - Return a mapping from pattern name to a set of all parent patterns - (patterns which reference it). - - Args: - dangling: How refs to missing targets are handled. `'error'` raises, - `'ignore'` drops those targets, and `'include'` adds them as - synthetic keys whose values are their existing parents. - - Returns: - Mapping from pattern name to a set of all patterns which reference it. - """ - child_graph, dangling_refs = self._raw_child_graph() - if dangling == 'error' and dangling_refs: - raise self._dangling_refs_error(dangling_refs, 'building parent graph') - - existing = set(child_graph) - igraph: dict[str, set[str]] = {name: set() for name in existing} - for parent, children in child_graph.items(): - for child in children: - if child in existing: - igraph[child].add(parent) - elif dangling == 'include': - igraph.setdefault(child, set()).add(parent) - return igraph - - def child_order( - self, - dangling: dangling_mode_t = 'error', - ) -> list[str]: - """ - Return a topologically sorted list of graph node names. - Child (referenced) patterns will appear before their parents. - - Args: - dangling: Passed to `child_graph()`. - - Return: - Topologically sorted list of pattern names. - """ - try: - return cast('list[str]', list(TopologicalSorter(self.child_graph(dangling=dangling)).static_order())) - except CycleError as exc: - cycle = exc.args[1] if len(exc.args) > 1 else None - if cycle is None: - raise LibraryError('Cycle found while building child order') from exc - raise LibraryError(f'Cycle found while building child order: {cycle}') from exc - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - """ - Find the location and orientation of all refs pointing to `name`. - Refs with a `repetition` are resolved into multiple instances (locations). - - Args: - name: Name of the referenced pattern. - parent_graph: Mapping from pattern name to the set of patterns which - reference it. Default (`None`) calls `self.parent_graph()`. - The provided graph may be for a superset of `self` (i.e. it may - contain additional patterns which are not present in self; they - will be ignored). - dangling: How refs to missing targets are handled if `parent_graph` - is not provided. `'include'` also allows querying missing names. - - Returns: - Mapping of {parent_name: transform_list}, where transform_list - is an Nx4 ndarray with rows - `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. - """ - instances = defaultdict(list) - if parent_graph is None: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return instances - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') - if dangling == 'ignore': - return instances - - for parent in parent_graph.get(name, set()): - if parent not in self: # parent_graph may be a for a superset of self - continue - for ref in self[parent].refs[name]: - instances[parent].append(ref.as_transforms()) - - return instances - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - """ - Find the absolute (top-level) location and orientation of all refs (including - repetitions) pointing to `name`. - - Args: - name: Name of the referenced pattern. - order: List of pattern names in which children are guaranteed - to appear before their parents (i.e. topologically sorted). - Default (`None`) calls `self.child_order()`. - parent_graph: Passed to `find_refs_local`. - Mapping from pattern name to the set of patterns which - reference it. Default (`None`) calls `self.parent_graph()`. - The provided graph may be for a superset of `self` (i.e. it may - contain additional patterns which are not present in self; they - will be ignored). - dangling: How refs to missing targets are handled if `order` or - `parent_graph` are not provided. `'include'` also allows - querying missing names. - - Returns: - Mapping of `{hierarchy: transform_list}`, where `hierarchy` is a tuple of the form - `(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx4 ndarray - with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. - """ - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - - transforms: dict[str, list[tuple[ - tuple[str, ...], - NDArray[numpy.float64] - ]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - for path, inner in inners: - combined = apply_transforms(numpy.concatenate(outer), inner) - transforms[parent].append(( - (next_name,) + path, - combined, - )) - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - full_path = (parent,) + path - assert full_path not in result - result[full_path] = instances - return result - - - -class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): - """ - Interface for a writeable library. - - A library is a mapping from unique names (str) to collections of geometry (`Pattern`). - """ - # inherited abstract functions - #def __getitem__(self, key: str) -> 'Pattern': - #def __iter__(self) -> Iterator[str]: - #def __len__(self) -> int: - #def __setitem__(self, key: str, value: 'Pattern | Callable[[], Pattern]') -> None: - #def __delitem__(self, key: str) -> None: - - @abstractmethod - def __setitem__( - self, - key: str, - value: 'Pattern | Callable[[], Pattern]', - ) -> None: - pass - - @abstractmethod - def __delitem__(self, key: str) -> None: - pass - - @abstractmethod - def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: - pass - - def resolve( - self, - other: 'Abstract | str | Pattern | TreeView', - append: bool = False, - ) -> 'Abstract | Pattern': - """ - Resolve another device (name, Abstract, Pattern, or TreeView) into an Abstract or Pattern. - If it is a TreeView, it is first added into this library. - - Args: - other: The device to resolve. - append: If True and `other` is an `Abstract`, returns the full `Pattern` from the library. - - Returns: - An `Abstract` or `Pattern` object. - """ - from .pattern import Pattern #noqa: PLC0415 - if not isinstance(other, str | Abstract | Pattern): - # We got a TreeView; add it into self and grab its topcell as an Abstract - other = self << other - - if isinstance(other, str): - other = self.abstract(other) - if append and isinstance(other, Abstract): - other = self[other.name] - return other - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> Self: - """ - Rename a pattern. - - Args: - old_name: Current name for the pattern - new_name: New name for the pattern - move_references: If `True`, any refs in this library pointing to `old_name` - will be updated to point to `new_name`. - - Returns: - self - """ - if old_name not in self: - raise LibraryError(f'"{old_name}" does not exist in the library.') - if old_name == new_name: - return self - - self[new_name] = self[old_name] - del self[old_name] - if move_references: - self.move_references(old_name, new_name) - return self - - def rename_top(self, name: str) -> Self: - """ - Rename the (single) top pattern - """ - self.rename(self.top(), name, move_references=True) - return self - - def move_references(self, old_target: str, new_target: str) -> Self: - """ - Change all references pointing at `old_target` into references pointing at `new_target`. - - Args: - old_target: Current reference target - new_target: New target for the reference - - Returns: - self - """ - if old_target == new_target: - return self - - for pattern in self.values(): - if old_target in pattern.refs: - pattern.refs[new_target].extend(pattern.refs[old_target]) - del pattern.refs[old_target] - return self - - def map_layers( - self, - map_layer: Callable[[layer_t], layer_t], - ) -> Self: - """ - Move all the elements in all patterns from one layer onto a different layer. - Can also handle multiple such mappings simultaneously. - - Args: - map_layer: Callable which may be called with each layer present in `elements`, - and should return the new layer to which it will be mapped. - A simple example which maps `old_layer` to `new_layer` and leaves all others - as-is would look like `lambda layer: {old_layer: new_layer}.get(layer, layer)` - - Returns: - self - """ - for pattern in self.values(): - pattern.shapes = map_layers(pattern.shapes, map_layer) - pattern.labels = map_layers(pattern.labels, map_layer) - return self - - def mkpat(self, name: str) -> tuple[str, 'Pattern']: - """ - Convenience method to create an empty pattern, add it to the library, - and return both the pattern and name. - - Args: - name: Name for the pattern - - Returns: - (name, pattern) tuple - """ - from .pattern import Pattern #noqa: PLC0415 - pat = Pattern() - self[name] = pat - return name, pat - - def add( - self, - other: Mapping[str, 'Pattern'], - rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns, - mutate_other: bool = False, - ) -> dict[str, str]: - """ - Add items from another library into this one. - - If any name in `other` is already present in `self`, `rename_theirs(self, name)` is called - to pick a new name for the newly-added pattern. If the new name still conflicts with a name - in `self` a `LibraryError` is raised. All references to the original name (within `other)` - are updated to the new name. - If `mutate_other=False` (default), all changes are made to a deepcopy of `other`. - - By default, `rename_theirs` makes no changes to the name (causing a `LibraryError`) unless the - name starts with `SINGLE_USE_PREFIX`. Prefixed names are truncated to before their first - non-prefix '$' and then passed to `self.get_name()` to create a new unique name. - - Args: - other: The library to insert keys from. - rename_theirs: Called as rename_theirs(self, name) for each duplicate name - encountered in `other`. Should return the new name for the pattern in - `other`. See above for default behavior. - mutate_other: If `True`, modify the original library and its contained patterns - (e.g. when renaming patterns and updating refs). Otherwise, operate on a deepcopy - (default). - - Returns: - A mapping of `{old_name: new_name}` for all names in `other` which were - renamed while being added. Unchanged names are omitted. - - Raises: - `LibraryError` if a duplicate name is encountered even after applying `rename_theirs()`. - """ - from .pattern import map_targets #noqa: PLC0415 - duplicates = set(self.keys()) & set(other.keys()) - - if not duplicates: - if mutate_other: - temp = other - else: - temp = Library(copy.deepcopy(dict(other))) - - for key in temp: - self._merge(key, temp, key) - return {} - - if mutate_other: - if isinstance(other, Library): - temp = other - else: - temp = Library(dict(other)) - else: - temp = Library(copy.deepcopy(dict(other))) - rename_map = {} - for old_name in temp: - if old_name in self: - new_name = rename_theirs(self, old_name) - if new_name in self: - raise LibraryError(f'Unresolved duplicate key encountered in library merge: {old_name} -> {new_name}') - rename_map[old_name] = new_name - else: - new_name = old_name - - self._merge(new_name, temp, old_name) - - # Update references in the newly-added cells - for old_name in temp: - new_name = rename_map.get(old_name, old_name) - pat = self[new_name] - pat.refs = map_targets(pat.refs, lambda tt: cast('dict[str | None, str | None]', rename_map).get(tt, tt)) - - return rename_map - - def __lshift__(self, other: TreeView) -> str: - """ - `add()` items from a tree (single-topcell name: pattern mapping) into this one, - and return the name of the tree's topcell (in this library; it may have changed - based on `add()`'s default `rename_theirs` argument). - - Raises: - LibraryError if there is more than one topcell in `other`. - """ - if len(other) == 1: - name = next(iter(other)) - else: - if not isinstance(other, ILibraryView): - other = LibraryView(other) - - tops = other.tops() - if len(tops) > 1: - raise LibraryError('Received a library containing multiple topcells!') - - name = tops[0] - - rename_map = self.add(other) - new_name = rename_map.get(name, name) - return new_name - - def __le__(self, other: Mapping[str, 'Pattern']) -> Abstract: - """ - Perform the same operation as `__lshift__` / `<<`, but return an `Abstract` instead - of just the pattern's name. - - Raises: - LibraryError if there is more than one topcell in `other`. - """ - new_name = self << other - return self.abstract(new_name) - - def dedup( - self, - norm_value: int = int(1e6), - exclude_types: tuple[type] = (Polygon,), - label2name: Callable[[tuple], str] | None = None, - threshold: int = 2, - ) -> Self: - """ - Iterates through all `Pattern`s. Within each `Pattern`, it iterates - over all shapes, calling `.normalized_form(norm_value)` on them to retrieve a scale-, - offset-, and rotation-independent form. Each shape whose normalized form appears - more than once is removed and re-added using `Ref` objects referencing a newly-created - `Pattern` containing only the normalized form of the shape. - - Note: - The default norm_value was chosen to give a reasonable precision when using - integer values for coordinates. - - Args: - norm_value: Passed to `shape.normalized_form(norm_value)`. Default `1e6` (see function - note) - exclude_types: Shape types passed in this argument are always left untouched, for - speed or convenience. Default: `(shapes.Polygon,)` - label2name: Given a label tuple as returned by `shape.normalized_form(...)`, pick - a name for the generated pattern. - Default `self.get_name(SINGLE_USE_PREIX + 'shape')`. - threshold: Only replace shapes with refs if there will be at least this many - instances. - - Returns: - self - """ - # This currently simplifies globally (same shape in different patterns is - # merged into the same ref target). - - from .pattern import Pattern #noqa: PLC0415 - - if exclude_types is None: - exclude_types = () - - if label2name is None: - def label2name(label: tuple) -> str: # noqa: ARG001 - return self.get_name(SINGLE_USE_PREFIX + 'shape') - - used_names = set(self.keys()) - - def reserve_target_name(label: tuple) -> str: - base_name = label2name(label) - name = base_name - ii = sum(1 for nn in used_names if nn.startswith(base_name)) if base_name in used_names else 0 - while name in used_names or name == '': - name = base_name + b64suffix(ii) - ii += 1 - used_names.add(name) - return name - - shape_counts: MutableMapping[tuple, int] = defaultdict(int) - shape_funcs = {} - - # ## First pass ## - # Using the label tuple from `.normalized_form()` as a key, check how many of each shape - # are present and store the shape function for each one - for pat in tuple(self.values()): - for layer, sseq in pat.shapes.items(): - for shape in sseq: - if not any(isinstance(shape, t) for t in exclude_types): - base_label, _values, func = shape.normalized_form(norm_value) - label = (*base_label, layer) - shape_funcs[label] = func - shape_counts[label] += 1 - - shape_pats = {} - target_names = {} - for label, count in shape_counts.items(): - if count < threshold: - continue - - shape_func = shape_funcs[label] - shape_pat = Pattern() - shape_pat.shapes[label[-1]] += [shape_func()] - shape_pats[label] = shape_pat - target_names[label] = reserve_target_name(label) - - # ## Second pass ## - for pat in tuple(self.values()): - # Store `[(index_in_shapes, values_from_normalized_form), ...]` for all shapes which - # are to be replaced. - # The `values` are `(offset, scale, rotation)`. - - shape_table: dict[tuple, list] = defaultdict(list) - for layer, sseq in pat.shapes.items(): - for ii, shape in enumerate(sseq): - if any(isinstance(shape, tt) for tt in exclude_types): - continue - - base_label, values, _func = shape.normalized_form(norm_value) - label = (*base_label, layer) - - if label not in shape_pats: - continue - - shape_table[label].append((ii, values)) - - # For repeated shapes, create a `Pattern` holding a normalized shape object, - # and add `pat.refs` entries for each occurrence in pat. Also, note down that - # we should delete the `pat.shapes` entries for which we made `Ref`s. - for label, shape_entries in shape_table.items(): - layer = label[-1] - target = target_names[label] - shapes_to_remove = [] - for ii, values in shape_entries: - offset, scale, rotation, mirror_x = values - pat.ref(target=target, offset=offset, scale=scale, - rotation=rotation, mirrored=mirror_x) - shapes_to_remove.append(ii) - - # Remove any shapes for which we have created refs. - for ii in sorted(shapes_to_remove, reverse=True): - del pat.shapes[layer][ii] - - for ll, pp in shape_pats.items(): - self[target_names[ll]] = pp - - return self - - def wrap_repeated_shapes( - self, - name_func: Callable[['Pattern', Shape | Label], str] | None = None, - ) -> Self: - """ - Wraps all shapes and labels with a non-`None` `repetition` attribute - into a `Ref`/`Pattern` combination, and applies the `repetition` - to each `Ref` instead of its contained shape. - - Args: - name_func: Function f(this_pattern, shape) which generates a name for the - wrapping pattern. - Default is `self.get_name(SINGLE_USE_PREFIX + 'rep')`. - - Returns: - self - """ - from .pattern import Pattern #noqa: PLC0415 - - if name_func is None: - def name_func(_pat: Pattern, _shape: Shape | Label) -> str: - return self.get_name(SINGLE_USE_PREFIX + 'rep') - - for pat in tuple(self.values()): - for layer in pat.shapes: - new_shapes = [] - for shape in pat.shapes[layer]: - if shape.repetition is None: - new_shapes.append(shape) - continue - - name = name_func(pat, shape) - self[name] = Pattern(shapes={layer: [shape]}) - pat.ref(name, repetition=shape.repetition) - shape.repetition = None - pat.shapes[layer] = new_shapes - - for layer in pat.labels: - new_labels = [] - for label in pat.labels[layer]: - if label.repetition is None: - new_labels.append(label) - continue - name = name_func(pat, label) - self[name] = Pattern(labels={layer: [label]}) - pat.ref(name, repetition=label.repetition) - label.repetition = None - pat.labels[layer] = new_labels - - return self - - def resolve_repeated_refs(self, name: str | None = None) -> Self: - """ - Expand all repeated references into multiple individual references. - Alters the library in-place. - - Args: - name: If specified, only resolve repeated refs in this pattern. - Otherwise, resolve in all patterns. - - Returns: - self - """ - if name is not None: - self[name].resolve_repeated_refs() - else: - for pat in self.values(): - pat.resolve_repeated_refs() - return self - - def subtree( - self, - tops: str | Sequence[str], - ) -> Self: - """ - Return a new `ILibraryView`, containing only the specified patterns and the patterns they - reference (recursively). - Dangling references do not cause an error. - - Args: - tops: Name(s) of patterns to keep - - Returns: - An object of the same type as `self` containing only `tops` and the patterns they reference. - """ - if isinstance(tops, str): - tops = (tops,) - - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - - new = type(self)() - for key in keep & set(self.keys()): - new._merge(key, self, key) - return new - - def prune_empty( - self, - repeat: bool = True, - dangling: dangling_mode_t = 'error', - ) -> set[str]: - """ - Delete any empty patterns (i.e. where `Pattern.is_empty` returns `True`). - - Args: - repeat: Also recursively delete any patterns which only contain(ed) empty patterns. - dangling: Passed to `parent_graph()`. - - Returns: - A set containing the names of all deleted patterns - """ - parent_graph = self.parent_graph(dangling=dangling) - empty = {name for name, pat in self.items() if pat.is_empty()} - trimmed = set() - while empty: - parents = set() - for name in empty: - del self[name] - for parent in parent_graph[name]: - del self[parent].refs[name] - parents |= parent_graph[name] - - trimmed |= empty - if not repeat: - break - - empty = {parent for parent in parents if self[parent].is_empty()} - return trimmed - - def delete( - self, - key: str, - delete_refs: bool = True, - ) -> Self: - """ - Delete a pattern and (optionally) all refs pointing to that pattern. - - Args: - key: Name of the pattern to be deleted. - delete_refs: If `True` (default), also delete all refs pointing to the pattern. - """ - del self[key] - if delete_refs: - for pat in self.values(): - if key in pat.refs: - del pat.refs[key] - return self - - -class LibraryView(ILibraryView): - """ - Default implementation for a read-only library. - - A library is a mapping from unique names (str) to collections of geometry (`Pattern`). - This library is backed by an arbitrary python object which implements the `Mapping` interface. - """ - mapping: Mapping[str, 'Pattern'] - - def __init__( - self, - mapping: Mapping[str, 'Pattern'], - ) -> None: - self.mapping = mapping - - def __getitem__(self, key: str) -> 'Pattern': - return self.mapping[key] - - def __iter__(self) -> Iterator[str]: - return iter(self.mapping) - - def __len__(self) -> int: - return len(self.mapping) - - def __contains__(self, key: object) -> bool: - return key in self.mapping - - def __repr__(self) -> str: - return f'' - - -class Library(ILibrary): - """ - Default implementation for a writeable library. - - A library is a mapping from unique names (str) to collections of geometry (`Pattern`). - This library is backed by an arbitrary python object which implements the `MutableMapping` interface. - """ - mapping: MutableMapping[str, 'Pattern'] - - def __init__( - self, - mapping: MutableMapping[str, 'Pattern'] | None = None, - ) -> None: - if mapping is None: - self.mapping = {} - else: - self.mapping = mapping - - def __getitem__(self, key: str) -> 'Pattern': - return self.mapping[key] - - def __iter__(self) -> Iterator[str]: - return iter(self.mapping) - - def __len__(self) -> int: - return len(self.mapping) - - def __contains__(self, key: object) -> bool: - return key in self.mapping - - def __setitem__( - self, - key: str, - value: 'Pattern | Callable[[], Pattern]', - ) -> None: - if key in self.mapping: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - - value = value() if callable(value) else value - self.mapping[key] = value - - def __delitem__(self, key: str) -> None: - del self.mapping[key] - - def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: - self[key_self] = other[key_other] - - def __repr__(self) -> str: - return f'' - - @classmethod - def mktree(cls: type[Self], name: str) -> tuple[Self, 'Pattern']: - """ - Create a new Library and immediately add a pattern - - Args: - name: The name for the new pattern (usually the name of the topcell). - - Returns: - The newly created `Library` and the newly created `Pattern` - """ - from .pattern import Pattern #noqa: PLC0415 - tree = cls() - pat = Pattern() - tree[name] = pat - return tree, pat - - -@dataclass -class _SourceLayer: - """ One imported source layer tracked by an `OverlayLibrary`. """ - library: ILibraryView - source_to_visible: dict[str, str] - visible_to_source: dict[str, str] - child_graph: dict[str, set[str]] - order: list[str] - - -@dataclass(frozen=True) -class _SourceEntry: - """ Reference to a single visible source-backed cell in an overlay. """ - layer_index: int - source_name: str - - -def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: - func = getattr(view, '_materialize_pattern', None) - if callable(func): - return cast('Pattern', func(name, persist=False)) - return view[name].deepcopy() - - -class PortsLibraryView(ILibraryView): - """ - Read-only view which imports ports into cells on first materialization. - - The wrapped source remains untouched; this view owns a separate processed - cache so direct-copy workflows can continue to use the raw source view. - - Graph queries, source ordering, and copy-through capabilities are delegated - to the wrapped source whenever possible, while `__getitem__` and - `materialize_many()` return port-imported patterns. - """ - - def __init__( - self, - source: ILibraryView, - *, - layers: Sequence[layer_t], - max_depth: int = 0, - skip_subcells: bool = True, - ) -> None: - self._source = source - self._layers = tuple(layers) - self._max_depth = max_depth - self._skip_subcells = skip_subcells - self._cache: dict[str, Pattern] = {} - self._lookups_in_progress: list[str] = [] - if hasattr(source, 'library_info'): - self.library_info = cast('dict[str, Any]', source.library_info) - - def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) - - def __iter__(self) -> Iterator[str]: - return iter(self._source) - - def __len__(self) -> int: - return len(self._source) - - def __contains__(self, key: object) -> bool: - return key in self._source - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: - from .utils.ports2data import data_to_ports # noqa: PLC0415 - - if name in self._cache: - return self._cache[name] - - if name in self._lookups_in_progress: - chain = ' -> '.join(self._lookups_in_progress + [name]) - raise LibraryError( - f'Detected circular reference or recursive lookup of "{name}".\n' - f'Lookup chain: {chain}\n' - 'This may be caused by an invalid (cyclical) reference, or buggy code.' - ) - - self._lookups_in_progress.append(name) - try: - pat = _materialize_detached_pattern(self._source, name) - pat = data_to_ports( - layers=self._layers, - library=self, - pattern=pat, - name=name, - max_depth=self._max_depth, - skip_subcells=self._skip_subcells, - ) - finally: - self._lookups_in_progress.pop() - - if persist: - self._cache[name] = pat - return pat - - def materialize_many( - self, - names: Sequence[str], - *, - persist: bool = True, - ) -> LibraryView: - mats = { - name: self._materialize_pattern(name, persist=persist) - for name in dict.fromkeys(names) - } - return LibraryView(mats) - - def source_order(self) -> tuple[str, ...]: - return self._source.source_order() - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._source.child_graph(dangling=dangling) - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._source.parent_graph(dangling=dangling) - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) - keep |= set(tops) - return self.materialize_many(tuple(keep), persist=True) - - def tops(self) -> list[str]: - return self._source.tops() - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - finder = getattr(self._source, 'find_refs_local', None) - if callable(finder): - return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) - return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - finder = getattr(self._source, 'find_refs_global', None) - if callable(finder): - return cast( - 'dict[tuple[str, ...], NDArray[numpy.float64]]', - finder(name, order=order, parent_graph=parent_graph, dangling=dangling), - ) - return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) - - def raw_struct_bytes(self, name: str) -> bytes: - reader = getattr(self._source, 'raw_struct_bytes', None) - if not callable(reader): - raise TypeError('raw_struct_bytes') - return cast('bytes', reader(name)) - - def can_copy_raw_struct(self, name: str) -> bool: - can_copy = getattr(self._source, 'can_copy_raw_struct', None) - if not callable(can_copy): - return False - return bool(can_copy(name)) - - def close(self) -> None: - closer = getattr(self._source, 'close', None) - if callable(closer): - closer() - - def __enter__(self) -> 'PortsLibraryView': - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - -class OverlayLibrary(ILibrary): - """ - Mutable overlay over one or more source libraries. - - Source-backed cells remain lazy until accessed through `__getitem__`, at - which point that visible cell is promoted into an overlay-owned materialized - `Pattern`. - """ - - def __init__(self) -> None: - self._layers: list[_SourceLayer] = [] - self._entries: dict[str, Pattern | _SourceEntry] = {} - self._order: list[str] = [] - self._target_remap: dict[str, str] = {} - - def __iter__(self) -> Iterator[str]: - return (name for name in self._order if name in self._entries) - - def __len__(self) -> int: - return len(self._entries) - - def __contains__(self, key: object) -> bool: - return key in self._entries - - def __getitem__(self, key: str) -> Pattern: - return self._materialize_pattern(key, persist=True) - - def __setitem__( - self, - key: str, - value: Pattern | Callable[[], Pattern], - ) -> None: - if key in self._entries: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - pattern = value() if callable(value) else value - self._entries[key] = pattern - if key not in self._order: - self._order.append(key) - - def __delitem__(self, key: str) -> None: - if key not in self._entries: - raise KeyError(key) - del self._entries[key] - - def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: - self[key_self] = copy.deepcopy(other[key_other]) - - def add_source( - self, - source: Mapping[str, Pattern] | ILibraryView, - *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, - rename_when: Literal['conflict', 'always'] = 'conflict', - ) -> dict[str, str]: - """ - Add a source-backed library layer. - - Args: - rename_theirs: Function used to choose visible names for imported - source cells. - rename_when: If `'conflict'`, only conflicting names are renamed. - If `'always'`, every imported source name is passed through - `rename_theirs`. - """ - view = source if isinstance(source, ILibraryView) else LibraryView(source) - source_order = list(view.source_order()) - child_graph = view.child_graph(dangling='include') - - source_to_visible = _plan_source_names( - self, - source_order, - self._entries, - rename_theirs = rename_theirs, - rename_when = rename_when, - ) - visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()} - - layer = _SourceLayer( - library=view, - source_to_visible=source_to_visible, - visible_to_source=visible_to_source, - child_graph=child_graph, - order=[source_to_visible[name] for name in source_order], - ) - layer_index = len(self._layers) - self._layers.append(layer) - - for source_name, visible_name in source_to_visible.items(): - self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) - if visible_name not in self._order: - self._order.append(visible_name) - - return _source_rename_map(source_to_visible) - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> 'OverlayLibrary': - if old_name not in self._entries: - raise LibraryError(f'"{old_name}" does not exist in the library.') - if old_name == new_name: - return self - if new_name in self._entries: - raise LibraryError(f'"{new_name}" already exists in the library.') - - entry = self._entries.pop(old_name) - self._entries[new_name] = entry - if isinstance(entry, _SourceEntry): - layer = self._layers[entry.layer_index] - layer.source_to_visible[entry.source_name] = new_name - del layer.visible_to_source[old_name] - layer.visible_to_source[new_name] = entry.source_name - - idx = self._order.index(old_name) - self._order[idx] = new_name - - if move_references: - self.move_references(old_name, new_name) - return self - - def _resolve_target(self, target: str) -> str: - seen: set[str] = set() - current = target - while current in self._target_remap: - if current in seen: - raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') - seen.add(current) - current = self._target_remap[current] - return current - - def _set_target_remap(self, old_target: str, new_target: str) -> None: - resolved_new = self._resolve_target(new_target) - if resolved_new == old_target: - raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') - self._target_remap[old_target] = resolved_new - for key in list(self._target_remap): - self._target_remap[key] = self._resolve_target(self._target_remap[key]) - - def move_references(self, old_target: str, new_target: str) -> 'OverlayLibrary': - if old_target == new_target: - return self - self._set_target_remap(old_target, new_target) - for entry in list(self._entries.values()): - if isinstance(entry, Pattern) and old_target in entry.refs: - entry.refs[new_target].extend(entry.refs[old_target]) - del entry.refs[old_target] - return self - - def _effective_target(self, layer: _SourceLayer, target: str) -> str: - visible = layer.source_to_visible.get(target, target) - return self._resolve_target(visible) - - def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: - if name not in self._entries: - raise KeyError(name) - entry = self._entries[name] - if isinstance(entry, Pattern): - return entry - - layer = self._layers[entry.layer_index] - source_pat = layer.library[entry.source_name].deepcopy() - - def remap(target: str | None) -> str | None: - return None if target is None else self._effective_target(layer, target) - - if source_pat.refs: - source_pat.refs = map_targets(source_pat.refs, remap) - pat = source_pat - if persist: - self._entries[name] = pat - return pat - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - graph: dict[str, set[str]] = {} - for name in self._order: - if name not in self._entries: - continue - entry = self._entries[name] - if isinstance(entry, Pattern): - graph[name] = {child for child, refs in entry.refs.items() if child is not None and refs} - continue - layer = self._layers[entry.layer_index] - children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} - graph[name] = children - - existing = set(graph) - dangling_refs = set().union(*(children - existing for children in graph.values())) - if dangling == 'error': - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') - return graph - if dangling == 'ignore': - return {name: {child for child in children if child in existing} for name, children in graph.items()} - - for child in dangling_refs: - graph.setdefault(cast('str', child), set()) - return graph - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') - existing = set(self.keys()) - igraph: dict[str, set[str]] = {name: set() for name in child_graph} - for parent, children in child_graph.items(): - for child in children: - if child in existing or dangling == 'include': - igraph.setdefault(child, set()).add(parent) - if dangling == 'error': - raw = self.child_graph(dangling='include') - dangling_refs = set().union(*(children - existing for children in raw.values())) - if dangling_refs: - raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') - return igraph - - def subtree( - self, - tops: str | Sequence[str], - ) -> ILibraryView: - if isinstance(tops, str): - tops = (tops,) - keep = cast('set[str]', self.referenced_patterns(tops) - {None}) - keep |= set(tops) - return LibraryView({name: self[name] for name in keep}) - - def find_refs_local( - self, - name: str, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[str, list[NDArray[numpy.float64]]]: - instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) - if parent_graph is None: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return instances - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') - if dangling == 'ignore': - return instances - - for parent in parent_graph.get(name, set()): - pat = self._materialize_pattern(parent, persist=False) - for ref in pat.refs.get(name, []): - instances[parent].append(ref.as_transforms()) - return instances - - def find_refs_global( - self, - name: str, - order: list[str] | None = None, - parent_graph: dict[str, set[str]] | None = None, - dangling: dangling_mode_t = 'error', - ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: - graph_mode = 'ignore' if dangling == 'ignore' else 'include' - if order is None: - order = self.child_order(dangling=graph_mode) - if parent_graph is None: - parent_graph = self.parent_graph(dangling=graph_mode) - - if name not in self: - if name not in parent_graph: - return {} - if dangling == 'error': - raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') - if dangling == 'ignore': - return {} - - self_keys = set(self.keys()) - transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] - transforms = defaultdict(list) - for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): - transforms[parent] = [((name,), numpy.concatenate(vals))] - - for next_name in order: - if next_name not in transforms: - continue - if not parent_graph.get(next_name, set()) & self_keys: - continue - - outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) - inners = transforms.pop(next_name) - for parent, outer in outers.items(): - outer_tf = numpy.concatenate(outer) - for path, inner in inners: - combined = apply_transforms(outer_tf, inner) - transforms[parent].append(((next_name,) + path, combined)) - - result = {} - for parent, targets in transforms.items(): - for path, instances in targets: - result[(parent,) + path] = instances - return result - - def source_order(self) -> tuple[str, ...]: - return tuple(name for name in self._order if name in self._entries) - - -@dataclass -class _BuildRecipe: - """ Captured deferred call to a pattern factory. """ - func: Callable[..., 'Pattern'] - args: tuple[Any, ...] - kwargs: dict[str, Any] - explicit_dependencies: tuple[str, ...] = () - - def depends_on(self, *names: str) -> '_BuildRecipe': - self.explicit_dependencies += tuple(names) - return self - - -def cell(func: Callable[..., 'Pattern']) -> Callable[..., _BuildRecipe]: - """ - Wrap a plain pattern factory so calls return deferred build recipes. - - Use as either `cell(fn)(...)` or `@cell`. - """ - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> _BuildRecipe: - return _BuildRecipe(func=func, args=args, kwargs=kwargs) - - return wrapper - - -class BuildCellsView: - """ - Attribute-based declaration namespace for `BuildLibrary`. - - This is the ergonomic authoring surface exposed as `builder.cells`. It is - intentionally write-focused: attribute assignment and deletion register - declarations, while attribute reads fail with guidance to build first and - use the returned library. - """ - - def __init__(self, library: 'BuildLibrary') -> None: - object.__setattr__(self, '_library', library) - - def __getattr__(self, name: str) -> 'Pattern': - raise BuildError( - f'BuildLibrary.cells.{name} is write-only during authoring. ' - 'Call build() and index the returned library instead.' - ) - - def __setattr__(self, name: str, value: 'Pattern | _BuildRecipe') -> None: - if name.startswith('_'): - object.__setattr__(self, name, value) - return - self._library[name] = value - - def __delattr__(self, name: str) -> None: - if name.startswith('_'): - raise AttributeError(name) - del self._library[name] - - -class BuildLibrary(ILibrary): - """ - Two-phase declaration surface for mixed imported/generated libraries. - - A `BuildLibrary` collects three kinds of inputs: - - direct declared `Pattern` objects - - deferred recipes created with `cell(...)` - - imported source-backed library views added with `add_source(...)` - - The builder itself is not a normal readable library during authoring. - Instead, `validate()` and `build()` create a temporary build-session library - that recipes can read from and write helper cells into while dependencies - are resolved. `build()` then freezes the builder on success and returns the - built library plus a `BuildReport`. - """ - - def __init__(self) -> None: - self.cells = BuildCellsView(self) - self._frozen = False - self._declarations: dict[str, Pattern | _BuildRecipe] = {} - self._sources: list[tuple[ILibraryView, dict[str, str]]] = [] - self._names: dict[str, None] = {} - - def _active_session(self) -> '_BuildSessionLibrary | None': - sessions = _ACTIVE_BUILD_SESSIONS.get() - if sessions is None: - return None - return sessions.get(id(self)) - - def _require_active_session(self, operation: str) -> '_BuildSessionLibrary': - session = self._active_session() - if session is None: - raise BuildError( - f'BuildLibrary.{operation}() is only available while validate() or build() is running. ' - 'Use the built output library for reads.' - ) - return session - - def _assert_editable(self) -> None: - if self._frozen: - raise BuildError('This BuildLibrary has already been built successfully and is now frozen.') - - def __iter__(self) -> Iterator[str]: - session = self._active_session() - if session is not None: - return iter(session) - return iter(self._names) - - def __len__(self) -> int: - session = self._active_session() - if session is not None: - return len(session) - return len(self._names) - - def __contains__(self, key: object) -> bool: - session = self._active_session() - if session is not None: - return key in session - return key in self._names - - def __getitem__(self, key: str) -> 'Pattern': - return self._require_active_session('__getitem__')[key] - - def __setitem__( - self, - key: str, - value: 'Pattern | _BuildRecipe', - ) -> None: - session = self._active_session() - if session is not None: - session[key] = value - return - - self._assert_editable() - if key in self._names: - raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!') - - if isinstance(value, _BuildRecipe): - declaration = value - else: - if callable(value): - raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.') - declaration = value - - self._declarations[key] = declaration - self._names[key] = None - - def __delitem__(self, key: str) -> None: - session = self._active_session() - if session is not None: - del session[key] - return - - self._assert_editable() - if key not in self._declarations: - raise KeyError(key) - del self._declarations[key] - del self._names[key] - - def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: - session = self._active_session() - if session is not None: - session._merge(key_self, other, key_other) - return - self[key_self] = copy.deepcopy(other[key_other]) - - def add( - self, - other: Mapping[str, 'Pattern'], - rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns, - mutate_other: bool = False, - ) -> dict[str, str]: - from .pattern import map_targets # noqa: PLC0415 - - session = self._active_session() - if session is not None: - return session.add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) - - self._assert_editable() - - source_backed = isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView) - if source_backed: - if mutate_other: - raise BuildError('BuildLibrary.add(..., mutate_other=True) is not supported for source-backed inputs.') - return self.add_source( - other, - rename_theirs = rename_theirs, - rename_when = 'conflict', - ) - - source_order = tuple(other.keys()) - source_to_visible = _plan_source_names( - self, - source_order, - self._names, - rename_theirs = rename_theirs, - rename_when = 'conflict', - ) - rename_map = _source_rename_map(source_to_visible) - - if mutate_other: - temp = other - else: - temp = Library(copy.deepcopy(dict(other))) - - for source_name in source_order: - visible_name = source_to_visible[source_name] - pattern = temp[source_name] - if rename_map: - pattern.refs = map_targets( - pattern.refs, - lambda target: cast('dict[str | None, str | None]', rename_map).get(target, target), - ) - self[visible_name] = pattern - - return rename_map - - def __lshift__(self, other: TreeView) -> str: - session = self._active_session() - if session is not None: - return session << other - - self._assert_editable() - if len(other) == 1: - name = next(iter(other)) - elif isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView): - source_order = other.source_order() - child_graph = other.child_graph(dangling='include') - referenced = set().union(*child_graph.values()) if child_graph else set() - tops = [candidate for candidate in source_order if candidate not in referenced] - if len(tops) != 1: - raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}') - name = tops[0] - else: - return super().__lshift__(other) - - rename_map = self.add(other) - return rename_map.get(name, name) - - def __le__(self, other: Mapping[str, 'Pattern']) -> Abstract: - if self._active_session() is not None: - return super().__le__(other) - raise BuildError('BuildLibrary.__le__() is only available while validate() or build() is running.') - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> Self: - """ - Rename a helper cell during an active build session. - - During authoring, declared cells must be registered under their - intended final names and imported source cells must be renamed through - `add_source(...)`. - """ - session = self._active_session() - if session is not None: - session.rename(old_name, new_name, move_references=move_references) - return self - - self._assert_editable() - if old_name == new_name: - return self - if old_name in self._declarations: - raise BuildError( - f'Cannot rename declared build cell "{old_name}" during authoring. ' - 'Register it under the intended final name instead.' - ) - if old_name not in self._names: - raise LibraryError(f'"{old_name}" does not exist in the builder.') - raise BuildError( - f'Cannot rename imported source cell "{old_name}" during authoring. ' - 'Choose visible source names with add_source(..., rename_theirs=..., rename_when=...).' - ) - - def abstract(self, name: str) -> Abstract: - return self._require_active_session('abstract').abstract(name) - - def resolve( - self, - other: 'Abstract | str | Pattern | TreeView', - append: bool = False, - ) -> 'Abstract | Pattern': - return self._require_active_session('resolve').resolve(other, append=append) - - def add_source( - self, - source: Mapping[str, 'Pattern'] | ILibraryView, - *, - rename_theirs: Callable[[ILibraryView, str], str] | None = None, - rename_when: Literal['conflict', 'always'] = 'conflict', - ) -> dict[str, str]: - """ - Register an imported source-backed library with the builder. - - The source is not materialized immediately. Its names are scanned once - to reserve visible builder names, then the source is read again when a - build session starts. The source's cell membership must not be - structurally mutated between `add_source()` and `build()`/`validate()`. - Source cells may be renamed on entry to avoid collisions with existing - declarations or other imported sources. - - Args: - rename_theirs: Function used to choose visible names for imported - source cells. - rename_when: If `'conflict'`, only conflicting names are renamed. - If `'always'`, every imported source name is passed through - `rename_theirs`. - - Returns: - Mapping of `{source_name: visible_name}` for imported names that - were renamed while being added. - """ - if self._active_session() is not None: - raise BuildError('BuildLibrary.add_source() is only available while authoring, not during validate() or build().') - self._assert_editable() - - view = source if isinstance(source, ILibraryView) else LibraryView(source) - source_order = tuple(view.source_order()) - source_to_visible = _plan_source_names( - self, - source_order, - self._names, - rename_theirs = rename_theirs, - rename_when = rename_when, - ) - - self._sources.append((view, dict(source_to_visible))) - for source_name in source_order: - visible = source_to_visible[source_name] - self._names[visible] = None - return _source_rename_map(source_to_visible) - - def validate( - self, - names: Sequence[str] | None = None, - *, - allow_dangling: bool = False, - ) -> BuildReport: - """ - Run the full build logic and return a `BuildReport` without producing output. - - This is a dry run over the same dependency resolution and recipe - execution path used by `build()`. Any generated library is discarded - after validation completes. - """ - _session, report = self._run_build(names=names, allow_dangling=allow_dangling) - return report - - def build( - self, - *, - output: Literal['overlay', 'library'] = 'overlay', - allow_dangling: bool = False, - ) -> tuple[ILibrary, BuildReport]: - """ - Materialize declarations and return a usable output library plus report. - - Args: - output: `'overlay'` preserves imported source-backed cells where - possible, while `'library'` eagerly materializes the full - result. - allow_dangling: If `False`, fail the build when the completed - library still contains dangling references. - """ - if output not in ('overlay', 'library'): - raise ValueError(f'Unknown build output mode: {output!r}') - self._assert_editable() - session, report = self._run_build(names=None, allow_dangling=allow_dangling) - if output == 'library': - built_output = session.to_library() - else: - built_output = session.to_overlay() - self._frozen = True - return built_output, report - - def _run_build( - self, - *, - names: Sequence[str] | None, - allow_dangling: bool, - ) -> tuple['_BuildSessionLibrary', BuildReport]: - roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys())) - unknown = [name for name in roots if name not in self._names] - if unknown: - raise BuildError(f'Unknown build roots requested: {unknown}') - - session = _BuildSessionLibrary(self) - sessions = dict(_ACTIVE_BUILD_SESSIONS.get() or {}) - sessions[id(self)] = session - token = _ACTIVE_BUILD_SESSIONS.set(sessions) - try: - session.materialize_many(roots) - if not allow_dangling: - session.child_graph(dangling='error') - finally: - _ACTIVE_BUILD_SESSIONS.reset(token) - - report = session.build_report(roots) - return session, report - - -class _BuildSessionLibrary(ILibrary): - """ - Internal overlay-backed library used while a `BuildLibrary` is executing. - - This object provides the mutable-library surface that recipes expect while - also tracking declared-cell dependencies, helper-cell provenance, and - imported source cells. It exists only for the duration of a validation or - build run. - """ - - def __init__(self, builder: BuildLibrary) -> None: - self._builder = builder - self._overlay = OverlayLibrary() - self._built: set[str] = set() - self._declared_stack: list[str] = [] - self._names = dict(builder._names) - self._provenance: dict[str, CellProvenance] = {} - self._dependency_graph: defaultdict[str, set[str]] = defaultdict(set) - self._install_sources() - - def _install_sources(self) -> None: - for source_library, source_to_visible in self._builder._sources: - source_order = source_library.source_order() - expected_names = set(source_to_visible) - actual_names = set(source_order) - if actual_names != expected_names: - added_names = sorted(actual_names - expected_names) - removed_names = sorted(expected_names - actual_names) - detail = [] - if added_names: - detail.append(f'added={added_names}') - if removed_names: - detail.append(f'removed={removed_names}') - raise BuildError( - 'Imported source library changed after add_source() was called ' - f'({", ".join(detail)}). ' - 'Do not structurally mutate source libraries between add_source() and build()/validate().' - ) - - def rename_source(_lib: ILibraryView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str: - return mapping[name] - - self._overlay.add_source( - source_library, - rename_theirs = rename_source, - rename_when = 'always', - ) - - for source_name in source_order: - visible_name = source_to_visible[source_name] - self._provenance[visible_name] = CellProvenance( - requested_name = source_name, - kind = 'source', - owner_declared_name = None, - build_chain = (), - ) - - def __iter__(self) -> Iterator[str]: - return iter(self._names) - - def __len__(self) -> int: - return len(self._names) - - def __contains__(self, key: object) -> bool: - return key in self._names - - def _current_declared(self) -> str | None: - if not self._declared_stack: - return None - return self._declared_stack[-1] - - def _record_dependency(self, target: str) -> None: - current = self._current_declared() - if current is None or current == target or target not in self._builder._declarations: - return - self._dependency_graph[current].add(target) - - def _guard_mutable_output_name(self, key: str, *, operation: str) -> None: - if key in self._builder._declarations: - raise BuildError(f'Cannot {operation} declared build cell "{key}" during an active build session.') - - provenance = self._provenance.get(key) - if provenance is not None and provenance.kind == 'source': - raise BuildError(f'Cannot {operation} imported source cell "{key}" during an active build session.') - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> Self: - if old_name == new_name: - return self - if old_name not in self._overlay: - if old_name in self._builder._declarations: - self._guard_mutable_output_name(old_name, operation='rename') - raise LibraryError(f'"{old_name}" does not exist in the library.') - - self._guard_mutable_output_name(old_name, operation='rename') - if new_name in self._names: - raise LibraryError(f'"{new_name}" already exists in the library.') - - self._overlay.rename(old_name, new_name, move_references=move_references) - self._names = { - new_name if name == old_name else name: None - for name in self._names - } - - provenance = self._provenance.pop(old_name) - self._provenance[new_name] = provenance - return self - - def __getitem__(self, key: str) -> 'Pattern': - if key in self._builder._declarations: - self._record_dependency(key) - self._ensure_declared(key) - return self._overlay[key] - - def __setitem__( - self, - key: str, - value: 'Pattern | Callable[[], Pattern]', - ) -> None: - if key in self._overlay: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - current = self._current_declared() - if key in self._builder._declarations and key != current: - raise LibraryError(f'"{key}" is reserved for a declared cell and cannot be used as a helper name.') - - pattern = value() if callable(value) else value - self._overlay[key] = pattern - self._names.setdefault(key, None) - - kind: Literal['declared', 'helper'] - if current is not None and key == current: - kind = 'declared' - else: - kind = 'helper' - - self._provenance[key] = CellProvenance( - requested_name = key, - kind = kind, - owner_declared_name = current if kind == 'helper' else key, - build_chain = tuple(self._declared_stack), - ) - - def __delitem__(self, key: str) -> None: - if key not in self._overlay: - if key in self._builder._declarations: - self._guard_mutable_output_name(key, operation='delete') - raise KeyError(key) - - self._guard_mutable_output_name(key, operation='delete') - if key in self._overlay: - del self._overlay[key] - self._names.pop(key, None) - self._provenance.pop(key, None) - - def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: - self[key_self] = copy.deepcopy(other[key_other]) - - def add( - self, - other: Mapping[str, 'Pattern'], - rename_theirs: Callable[['ILibraryView', str], str] = _rename_patterns, - mutate_other: bool = False, - ) -> dict[str, str]: - rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) - - current = self._current_declared() - for old_name, new_name in rename_map.items(): - if new_name in self._provenance: - self._provenance[new_name] = replace( - self._provenance[new_name], - requested_name = old_name, - owner_declared_name = current if current is not None else self._provenance[new_name].owner_declared_name, - ) - return rename_map - - def _wrap_error(self, name: str, exc: Exception) -> BuildError: - chain = tuple(self._declared_stack) - msg = [f'Failed while building declared cell "{name}"'] - if chain: - msg.append(f'Dependency chain: {" -> ".join(chain)}') - msg.append(f'Cause: {exc}') - return BuildError('\n'.join(msg)) - - def _ensure_named(self, name: str) -> None: - if name in self._builder._declarations: - self._record_dependency(name) - self._ensure_declared(name) - return - if name in self._overlay: - return - raise BuildError(f'Missing dependency "{name}"') - - def _ensure_declared(self, name: str) -> None: - from .pattern import Pattern # noqa: PLC0415 - - if name in self._built: - return - if name in self._declared_stack: - chain = ' -> '.join(self._declared_stack + [name]) - raise BuildError(f'Cycle detected while building declared cells: {chain}') - - declaration = self._builder._declarations[name] - self._declared_stack.append(name) - try: - if isinstance(declaration, _BuildRecipe): - for dep in declaration.explicit_dependencies: - self._ensure_named(dep) - pattern = declaration.func(*declaration.args, **declaration.kwargs) - if not isinstance(pattern, Pattern): - raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') # noqa: TRY301 - else: - pattern = declaration.deepcopy() - - if name in self._overlay: - if self._overlay[name] is not pattern: - raise BuildError( # noqa: TRY301 - f'Recipe for "{name}" wrote a different pattern into the session under its own name.' - ) - else: - self[name] = pattern - self._built.add(name) - except Exception as exc: - raise self._wrap_error(name, exc) from exc - finally: - self._declared_stack.pop() - - def materialize_many(self, names: Sequence[str]) -> None: - for name in dict.fromkeys(names): - self._ensure_named(name) - - def source_order(self) -> tuple[str, ...]: - return self._overlay.source_order() - - def child_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._overlay.child_graph(dangling=dangling) - - def parent_graph( - self, - dangling: dangling_mode_t = 'error', - ) -> dict[str, set[str]]: - return self._overlay.parent_graph(dangling=dangling) - - def build_report(self, requested_roots: Sequence[str]) -> BuildReport: - dependency_graph = { - name: frozenset(self._dependency_graph.get(name, set())) - for name in self._builder._declarations - if name in self._dependency_graph or name in requested_roots - } - return BuildReport( - requested_roots = tuple(dict.fromkeys(requested_roots)), - provenance = dict(self._provenance), - dependency_graph = dependency_graph, - ) - - def to_overlay(self) -> ILibrary: - return self._overlay - - def to_library(self) -> Library: - mapping = {name: self._overlay[name] for name in self._overlay.source_order()} - return Library(mapping) - - -class LazyLibrary(ILibrary): - """ - This class is usually used to create a library of Patterns by mapping names to - functions which generate or load the relevant `Pattern` object as-needed. - - TODO: lots of stuff causes recursive loads (e.g. data_to_ports?). What should you avoid? - """ - mapping: dict[str, Callable[[], 'Pattern']] - cache: dict[str, 'Pattern'] - _lookups_in_progress: list[str] - - def __init__(self) -> None: - self.mapping = {} - self.cache = {} - self._lookups_in_progress = [] - - def __setitem__( - self, - key: str, - value: 'Pattern | Callable[[], Pattern]', - ) -> None: - if key in self.mapping: - raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') - - if callable(value): - value_func = value - else: - value_func = lambda: cast('Pattern', value) # noqa: E731 - - self.mapping[key] = value_func - if key in self.cache: - del self.cache[key] - - def __delitem__(self, key: str) -> None: - del self.mapping[key] - if key in self.cache: - del self.cache[key] - - def __getitem__(self, key: str) -> 'Pattern': - logger.debug(f'loading {key}') - if key in self.cache: - logger.debug(f'found {key} in cache') - return self.cache[key] - - if key in self._lookups_in_progress: - chain = ' -> '.join(self._lookups_in_progress + [key]) - raise LibraryError( - f'Detected circular reference or recursive lookup of "{key}".\n' - f'Lookup chain: {chain}\n' - 'This may be caused by an invalid (cyclical) reference, or buggy code.\n' - 'If you are lazy-loading a file, try a non-lazy load and check for reference cycles.' - ) - - self._lookups_in_progress.append(key) - try: - func = self.mapping[key] - pat = func() - finally: - self._lookups_in_progress.pop() - self.cache[key] = pat - return pat - - def __iter__(self) -> Iterator[str]: - return iter(self.mapping) - - def __len__(self) -> int: - return len(self.mapping) - - def __contains__(self, key: object) -> bool: - return key in self.mapping - - def _merge(self, key_self: str, other: Mapping[str, 'Pattern'], key_other: str) -> None: - if isinstance(other, LazyLibrary): - self.mapping[key_self] = other.mapping[key_other] - if key_other in other.cache: - self.cache[key_self] = other.cache[key_other] - else: - self[key_self] = other[key_other] - - def __repr__(self) -> str: - return '' - - def rename( - self, - old_name: str, - new_name: str, - move_references: bool = False, - ) -> Self: - """ - Rename a pattern. - - Args: - old_name: Current name for the pattern - new_name: New name for the pattern - move_references: Whether to scan all refs in the pattern and - move them to point to `new_name` as necessary. - Default `False`. - - Returns: - self - """ - if old_name not in self.mapping: - raise LibraryError(f'"{old_name}" does not exist in the library.') - if old_name == new_name: - return self - - self[new_name] = self.mapping[old_name] # copy over function - if old_name in self.cache: - self.cache[new_name] = self.cache[old_name] - del self[old_name] - - if move_references: - self.move_references(old_name, new_name) - - return self - - def move_references(self, old_target: str, new_target: str) -> Self: - """ - Change all references pointing at `old_target` into references pointing at `new_target`. - - Args: - old_target: Current reference target - new_target: New target for the reference - - Returns: - self - """ - if old_target == new_target: - return self - - self.precache() - for pattern in self.cache.values(): - if old_target in pattern.refs: - pattern.refs[new_target].extend(pattern.refs[old_target]) - del pattern.refs[old_target] - return self - - def precache(self) -> Self: - """ - Force all patterns into the cache - - Returns: - self - """ - for key in self.mapping: - _ = self[key] # want to trigger our own __getitem__ - return self - - def __deepcopy__(self, memo: dict | None = None) -> 'LazyLibrary': - raise LibraryError('LazyLibrary cannot be deepcopied (deepcopy doesn\'t descend into closures)') - - -class AbstractView(Mapping[str, Abstract]): - """ - A read-only mapping from names to `Abstract` objects. - - This is usually just used as a shorthand for repeated calls to `library.abstract()`. - """ - library: ILibraryView - - def __init__(self, library: ILibraryView) -> None: - self.library = library - - def __getitem__(self, key: str) -> Abstract: - return self.library.abstract(key) - - def __iter__(self) -> Iterator[str]: - return self.library.__iter__() - - def __len__(self) -> int: - return self.library.__len__() - - -def b64suffix(ii: int) -> str: - """ - Turn an integer into a base64-equivalent suffix. - - This could be done with base64.b64encode, but this way is faster for many small `ii`. - """ - def i2a(nn: int) -> str: - return 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$?'[nn] - - parts = ['$', i2a(ii % 64)] - ii >>= 6 - while ii: - parts.append(i2a(ii % 64)) - ii >>= 6 - return ''.join(parts) diff --git a/masque/library/__init__.py b/masque/library/__init__.py new file mode 100644 index 0000000..b05b020 --- /dev/null +++ b/masque/library/__init__.py @@ -0,0 +1,29 @@ +"""Library classes for managing name-to-pattern mappings.""" +from .utils import ( + SINGLE_USE_PREFIX as SINGLE_USE_PREFIX, + Tree as Tree, + TreeView as TreeView, + b64suffix as b64suffix, + dangling_mode_t as dangling_mode_t, + visitor_function_t as visitor_function_t, +) +from .base import ( + AbstractView as AbstractView, + ILibrary as ILibrary, + ILibraryView as ILibraryView, +) +from .mapping import ( + Library as Library, + LibraryView as LibraryView, +) +from .overlay import ( + OverlayLibrary as OverlayLibrary, + PortsLibraryView as PortsLibraryView, +) +from .build import ( + BuildLibrary as BuildLibrary, + BuildReport as BuildReport, + CellProvenance as CellProvenance, + cell as cell, +) +from .lazy import LazyLibrary as LazyLibrary diff --git a/masque/library/base.py b/masque/library/base.py new file mode 100644 index 0000000..283ca6d --- /dev/null +++ b/masque/library/base.py @@ -0,0 +1,1259 @@ +"""Core library interfaces.""" +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from collections import defaultdict +from collections.abc import Callable, Mapping, MutableMapping, Sequence +from graphlib import CycleError, TopologicalSorter +from pprint import pformat +from typing import TYPE_CHECKING, Self, cast +import copy +import logging +import re + +import numpy + +from ..abstract import Abstract +from ..error import LibraryError, PatternError +from ..pattern import Pattern, map_layers +from ..shapes import Polygon, Shape +from ..utils import apply_transforms, layer_t +from .utils import SINGLE_USE_PREFIX, TreeView, b64suffix, dangling_mode_t, _rename_patterns, visitor_function_t + +if TYPE_CHECKING: + from collections.abc import Iterator + + from numpy.typing import ArrayLike, NDArray + + from ..label import Label + + +logger = logging.getLogger(__name__) + + +class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): + """ + Interface for a read-only library. + + A library is a mapping from unique names (str) to collections of geometry (`Pattern`). + """ + # inherited abstract functions + #def __getitem__(self, key: str) -> 'Pattern': + #def __iter__(self) -> Iterator[str]: + #def __len__(self) -> int: + + #__contains__, keys, items, values, get, __eq__, __ne__ supplied by Mapping + + def __repr__(self) -> str: + return '' + + def abstract_view(self) -> AbstractView: + """ + Returns: + An AbstractView into this library + """ + return AbstractView(self) + + def abstract(self, name: str) -> Abstract: + """ + Return an `Abstract` (name & ports) for the pattern in question. + + Args: + name: The pattern name + + Returns: + An `Abstract` object for the pattern + """ + return Abstract(name=name, ports=self[name].ports) + + def source_order(self) -> tuple[str, ...]: + """ + Return names in the library's preferred source order. + + Source-backed views may override this to preserve on-disk ordering + without materializing patterns. + """ + return tuple(self.keys()) + + def dangling_refs( + self, + tops: str | Sequence[str] | None = None, + ) -> set[str | None]: + """ + Get the set of all pattern names not present in the library but referenced + by `tops`, recursively traversing any refs. + + If `tops` are not given, all patterns in the library are checked. + + Args: + tops: Name(s) of the pattern(s) to check. + Default is all patterns in the library. + + Returns: + Set of all referenced pattern names + """ + if tops is None: + tops = tuple(self.keys()) + + referenced = self.referenced_patterns(tops) + return referenced - set(self.keys()) + + def referenced_patterns( + self, + tops: str | Sequence[str] | None = None, + skip: set[str | None] | None = None, + ) -> set[str | None]: + """ + Get the set of all pattern names referenced by `tops`. Recursively traverses into any refs. + + If `tops` are not given, all patterns in the library are checked. + + Args: + tops: Name(s) of the pattern(s) to check. + Default is all patterns in the library. + skip: Memo, set patterns which have already been traversed. + + Returns: + Set of all referenced pattern names + """ + if tops is None: + tops = tuple(self.keys()) + + if skip is None: + skip = {None} + + if isinstance(tops, str): + tops = (tops,) + tops = set(tops) + skip |= tops # don't re-visit tops + + # Get referenced patterns for all tops + targets = set() + for top in set(tops): + targets |= self[top].referenced_patterns() + + # Perform recursive lookups, but only once for each name + for target in targets - skip: + assert target is not None + skip.add(target) + if target in self: + targets |= self.referenced_patterns(target, skip=skip) + + return targets + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + """ + Return a new `ILibraryView`, containing only the specified patterns and the patterns they + reference (recursively). + Dangling references do not cause an error. + + Args: + tops: Name(s) of patterns to keep + + Returns: + A `LibraryView` containing only `tops` and the patterns they reference. + """ + if isinstance(tops, str): + tops = (tops,) + + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + + from .mapping import LibraryView # noqa: PLC0415 + + filtered = {kk: vv for kk, vv in self.items() if kk in keep} + new = LibraryView(filtered) + return new + + def polygonize( + self, + num_vertices: int | None = None, + max_arclen: float | None = None, + ) -> Self: + """ + Calls `.polygonize(...)` on each pattern in this library. + Arguments are passed on to `shape.to_polygons(...)`. + + Args: + num_vertices: Number of points to use for each polygon. Can be overridden by + `max_arclen` if that results in more points. Optional, defaults to shapes' + internal defaults. + max_arclen: Maximum arclength which can be approximated by a single line + segment. Optional, defaults to shapes' internal defaults. + + Returns: + self + """ + for pat in self.values(): + pat.polygonize(num_vertices, max_arclen) + return self + + def manhattanize( + self, + grid_x: ArrayLike, + grid_y: ArrayLike, + ) -> Self: + """ + Calls `.manhattanize(grid_x, grid_y)` on each pattern in this library. + + Args: + grid_x: List of allowed x-coordinates for the Manhattanized polygon edges. + grid_y: List of allowed y-coordinates for the Manhattanized polygon edges. + + Returns: + self + """ + for pat in self.values(): + pat.manhattanize(grid_x, grid_y) + return self + + def flatten( + self, + tops: str | Sequence[str], + flatten_ports: bool = False, + dangling_ok: bool = False, + ) -> dict[str, Pattern]: + """ + Returns copies of all `tops` patterns with all refs + removed and replaced with equivalent shapes. + Also returns flattened copies of all referenced patterns. + The originals in the calling `Library` are not modified. + For an in-place variant, see `Pattern.flatten`. + + Args: + tops: The pattern(s) to flatten. + flatten_ports: If `True`, keep ports from any referenced + patterns; otherwise discard them. + dangling_ok: If `True`, no error will be thrown if any + ref points to a name which is not present in the library. + Default False. + + Returns: + {name: flat_pattern} mapping for all flattened patterns. + """ + if isinstance(tops, str): + tops = (tops,) + + flattened: dict[str, Pattern | None] = {} + + def flatten_single(name: str) -> None: + flattened[name] = None + pat = self[name].deepcopy() + refs_by_target = tuple((target, tuple(refs)) for target, refs in pat.refs.items()) + + for target, refs in refs_by_target: + if target is None: + continue + if dangling_ok and target not in self: + continue + if target not in flattened: + flatten_single(target) + + target_pat = flattened[target] + if target_pat is None: + raise PatternError(f'Circular reference in {name} to {target}') + ports_only = flatten_ports and bool(target_pat.ports) + if target_pat.is_empty() and not ports_only: # avoid some extra allocations + continue + + for ref in refs: + if flatten_ports and ref.repetition is not None and target_pat.ports: + raise PatternError( + f'Cannot flatten ports from repeated ref to {target!r}; ' + 'flatten with flatten_ports=False or expand/rename the ports manually first.' + ) + p = ref.as_pattern(pattern=target_pat) + if not flatten_ports: + p.ports.clear() + pat.append(p) + + for target in set(pat.refs.keys()) & set(self.keys()): + del pat.refs[target] + + flattened[name] = pat + + for top in tops: + flatten_single(top) + + assert None not in flattened.values() + return cast('dict[str, Pattern]', flattened) + + def get_name( + self, + name: str = SINGLE_USE_PREFIX * 2, + sanitize: bool = True, + max_length: int = 32, + quiet: bool | None = None, + ) -> str: + """ + Find a unique name for the pattern. + + This function may be overridden in a subclass or monkey-patched to fit the caller's requirements. + + Args: + name: Preferred name for the pattern. Default is `SINGLE_USE_PREFIX * 2`. + sanitize: Allows only alphanumeric charaters and _?$. Replaces invalid characters with underscores. + max_length: Names longer than this will be truncated. + quiet: If `True`, suppress log messages. Default `None` suppresses messages only if + the name starts with `SINGLE_USE_PREFIX`. + + Returns: + Name, unique within this library. + """ + if quiet is None: + quiet = name.startswith(SINGLE_USE_PREFIX) + + if sanitize: + # Remove invalid characters + sanitized_name = re.compile(r'[^A-Za-z0-9_\?\$]').sub('_', name) + else: + sanitized_name = name + + suffixed_name = sanitized_name + if sanitized_name in self: + ii = sum(1 for nn in self.keys() if nn.startswith(sanitized_name)) + else: + ii = 0 + while suffixed_name in self or suffixed_name == '': + suffixed_name = sanitized_name + b64suffix(ii) + ii += 1 + + if len(suffixed_name) > max_length: + if name == '': + raise LibraryError(f'No valid pattern names remaining within the specified {max_length=}') + + cropped_name = self.get_name(sanitized_name[:-1], sanitize=sanitize, max_length=max_length, quiet=True) + else: + cropped_name = suffixed_name + + if not quiet: + logger.info(f'Requested name "{name}" changed to "{cropped_name}"') + + return cropped_name + + def tops(self) -> list[str]: + """ + Return the list of all patterns that are not referenced by any other pattern in the library. + + Returns: + A list of pattern names in which no pattern is referenced by any other pattern. + """ + names = set(self.keys()) + not_toplevel: set[str | None] = set() + for name in names: + not_toplevel |= set(self[name].refs.keys()) + + toplevel = list(names - not_toplevel) + return toplevel + + def top(self) -> str: + """ + Return the name of the topcell, or raise an exception if there isn't a single topcell + + Raises: + LibraryError if there is not exactly one topcell. + """ + tops = self.tops() + if len(tops) != 1: + raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}') + return tops[0] + + def top_pattern(self) -> Pattern: + """ + Shorthand for self[self.top()] + + Raises: + LibraryError if there is not exactly one topcell. + """ + return self[self.top()] + + @staticmethod + def _dangling_refs_error(dangling: set[str], context: str) -> LibraryError: + dangling_list = sorted(dangling) + return LibraryError(f'Dangling refs found while {context}: ' + pformat(dangling_list)) + + def _raw_child_graph(self) -> tuple[dict[str, set[str]], set[str]]: + existing = set(self.keys()) + graph: dict[str, set[str]] = {} + dangling: set[str] = set() + for name, pat in self.items(): + children = {child for child, refs in pat.refs.items() if child is not None and refs} + graph[name] = children + dangling |= children - existing + return graph, dangling + + def dfs( + self, + pattern: Pattern, + visit_before: visitor_function_t | None = None, + visit_after: visitor_function_t | None = None, + *, + hierarchy: tuple[str | None, ...] = (None,), + transform: ArrayLike | bool | None = False, + memo: dict | None = None, + ) -> Self: + """ + Convenience function. + Performs a depth-first traversal of a pattern and its referenced patterns. + At each pattern in the tree, the following sequence is called: + ``` + current_pattern = visit_before(current_pattern, **vist_args) + for target in current_pattern.refs: + for ref in pattern.refs[target]: + self.dfs(target, visit_before, visit_after, + hierarchy + (sp.target,), updated_transform, memo) + current_pattern = visit_after(current_pattern, **visit_args) + ``` + where `visit_args` are + `hierarchy`: (top_pattern_or_None, L1_pattern, L2_pattern, ..., parent_pattern, target_pattern) + tuple of all parent-and-higher pattern names. Top pattern name may be + `None` if not provided in first call to .dfs() + `transform`: numpy.ndarray containing cumulative + [x_offset, y_offset, rotation (rad), mirror_x (0 or 1)] + for the instance being visited + `memo`: Arbitrary dict (not altered except by `visit_before()` and `visit_after()`) + + Args: + pattern: Pattern object to start at ("top"/root node of the tree). + visit_before: Function to call before traversing refs. + Should accept a `Pattern` and `**visit_args`, and return the (possibly modified) + pattern. Default `None` (not called). + visit_after: Function to call after traversing refs. + Should accept a `Pattern` and `**visit_args`, and return the (possibly modified) + pattern. Default `None` (not called). + transform: Initial value for `visit_args['transform']`. + Can be `False`, in which case the transform is not calculated. + `True` or `None` is interpreted as `[0, 0, 0, 0]`. + memo: Arbitrary dict for use by `visit_*()` functions. Default `None` (empty dict). + hierarchy: Tuple of patterns specifying the hierarchy above the current pattern. + Default is (None,), which will be used as a placeholder for the top pattern's + name if not overridden. + + Returns: + self + """ + if memo is None: + memo = {} + + if transform is None or transform is True: + transform = numpy.array([0, 0, 0, 0, 1], dtype=float) + elif transform is not False: + transform = numpy.asarray(transform, dtype=float) + if transform.size == 4: + transform = numpy.append(transform, 1.0) + + original_pattern = pattern + + if visit_before is not None: + pattern = visit_before(pattern, hierarchy=hierarchy, memo=memo, transform=transform) + + for target in pattern.refs: + if target is None: + continue + if target in hierarchy: + raise LibraryError(f'.dfs() called on pattern with circular reference to "{target}"') + + for ref in pattern.refs[target]: + ref_transforms: list[bool] | NDArray[numpy.float64] + if transform is not False: + ref_transforms = apply_transforms(transform, ref.as_transforms()) + else: + ref_transforms = [False] + + for ref_transform in ref_transforms: + self.dfs( + pattern = self[target], + visit_before = visit_before, + visit_after = visit_after, + hierarchy = hierarchy + (target,), + transform = ref_transform, + memo = memo, + ) + + if visit_after is not None: + pattern = visit_after(pattern, hierarchy=hierarchy, memo=memo, transform=transform) + + if pattern is not original_pattern: + name = hierarchy[-1] + if not isinstance(self, ILibrary): + raise LibraryError('visit_* functions returned a new `Pattern` object' + ' but the library is immutable') + if name is None: + # The top pattern is not the original pattern, but we don't know what to call it! + raise LibraryError('visit_* functions returned a new `Pattern` object' + ' but no top-level name was provided in `hierarchy`') + + del cast('ILibrary', self)[name] + cast('ILibrary', self)[name] = pattern + + return self + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + """ + Return a mapping from pattern name to a set of all child patterns + (patterns it references). + + Only non-empty ref lists with non-`None` targets are treated as graph edges. + + Args: + dangling: How refs to missing targets are handled. `'error'` raises, + `'ignore'` drops those edges, and `'include'` exposes them as + synthetic leaf nodes. + + Returns: + Mapping from pattern name to a set of all pattern names it references. + """ + graph, dangling_refs = self._raw_child_graph() + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(dangling_refs, 'building child graph') + return graph + if dangling == 'ignore': + existing = set(graph) + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for target in dangling_refs: + graph.setdefault(target, set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + """ + Return a mapping from pattern name to a set of all parent patterns + (patterns which reference it). + + Args: + dangling: How refs to missing targets are handled. `'error'` raises, + `'ignore'` drops those targets, and `'include'` adds them as + synthetic keys whose values are their existing parents. + + Returns: + Mapping from pattern name to a set of all patterns which reference it. + """ + child_graph, dangling_refs = self._raw_child_graph() + if dangling == 'error' and dangling_refs: + raise self._dangling_refs_error(dangling_refs, 'building parent graph') + + existing = set(child_graph) + igraph: dict[str, set[str]] = {name: set() for name in existing} + for parent, children in child_graph.items(): + for child in children: + if child in existing: + igraph[child].add(parent) + elif dangling == 'include': + igraph.setdefault(child, set()).add(parent) + return igraph + + def child_order( + self, + dangling: dangling_mode_t = 'error', + ) -> list[str]: + """ + Return a topologically sorted list of graph node names. + Child (referenced) patterns will appear before their parents. + + Args: + dangling: Passed to `child_graph()`. + + Return: + Topologically sorted list of pattern names. + """ + try: + return cast('list[str]', list(TopologicalSorter(self.child_graph(dangling=dangling)).static_order())) + except CycleError as exc: + cycle = exc.args[1] if len(exc.args) > 1 else None + if cycle is None: + raise LibraryError('Cycle found while building child order') from exc + raise LibraryError(f'Cycle found while building child order: {cycle}') from exc + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + """ + Find the location and orientation of all refs pointing to `name`. + Refs with a `repetition` are resolved into multiple instances (locations). + + Args: + name: Name of the referenced pattern. + parent_graph: Mapping from pattern name to the set of patterns which + reference it. Default (`None`) calls `self.parent_graph()`. + The provided graph may be for a superset of `self` (i.e. it may + contain additional patterns which are not present in self; they + will be ignored). + dangling: How refs to missing targets are handled if `parent_graph` + is not provided. `'include'` also allows querying missing names. + + Returns: + Mapping of {parent_name: transform_list}, where transform_list + is an Nx4 ndarray with rows + `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. + """ + instances = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + if parent not in self: # parent_graph may be a for a superset of self + continue + for ref in self[parent].refs[name]: + instances[parent].append(ref.as_transforms()) + + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + """ + Find the absolute (top-level) location and orientation of all refs (including + repetitions) pointing to `name`. + + Args: + name: Name of the referenced pattern. + order: List of pattern names in which children are guaranteed + to appear before their parents (i.e. topologically sorted). + Default (`None`) calls `self.child_order()`. + parent_graph: Passed to `find_refs_local`. + Mapping from pattern name to the set of patterns which + reference it. Default (`None`) calls `self.parent_graph()`. + The provided graph may be for a superset of `self` (i.e. it may + contain additional patterns which are not present in self; they + will be ignored). + dangling: How refs to missing targets are handled if `order` or + `parent_graph` are not provided. `'include'` also allows + querying missing names. + + Returns: + Mapping of `{hierarchy: transform_list}`, where `hierarchy` is a tuple of the form + `(toplevel_pattern, lvl1_pattern, ..., name)` and `transform_list` is an Nx4 ndarray + with rows `(x_offset, y_offset, rotation_ccw_rad, mirror_across_x)`. + """ + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + + transforms: dict[str, list[tuple[ + tuple[str, ...], + NDArray[numpy.float64] + ]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + for path, inner in inners: + combined = apply_transforms(numpy.concatenate(outer), inner) + transforms[parent].append(( + (next_name,) + path, + combined, + )) + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + full_path = (parent,) + path + assert full_path not in result + result[full_path] = instances + return result + + + +class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): + """ + Interface for a writeable library. + + A library is a mapping from unique names (str) to collections of geometry (`Pattern`). + """ + # inherited abstract functions + #def __getitem__(self, key: str) -> 'Pattern': + #def __iter__(self) -> Iterator[str]: + #def __len__(self) -> int: + #def __setitem__(self, key: str, value: 'Pattern | Callable[[], Pattern]') -> None: + #def __delitem__(self, key: str) -> None: + + @abstractmethod + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + pass + + @abstractmethod + def __delitem__(self, key: str) -> None: + pass + + @abstractmethod + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + pass + + def resolve( + self, + other: Abstract | str | Pattern | TreeView, + append: bool = False, + ) -> Abstract | Pattern: + """ + Resolve another device (name, Abstract, Pattern, or TreeView) into an Abstract or Pattern. + If it is a TreeView, it is first added into this library. + + Args: + other: The device to resolve. + append: If True and `other` is an `Abstract`, returns the full `Pattern` from the library. + + Returns: + An `Abstract` or `Pattern` object. + """ + from ..pattern import Pattern # noqa: PLC0415 + if not isinstance(other, str | Abstract | Pattern): + # We got a TreeView; add it into self and grab its topcell as an Abstract + other = self << other + + if isinstance(other, str): + other = self.abstract(other) + if append and isinstance(other, Abstract): + other = self[other.name] + return other + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + """ + Rename a pattern. + + Args: + old_name: Current name for the pattern + new_name: New name for the pattern + move_references: If `True`, any refs in this library pointing to `old_name` + will be updated to point to `new_name`. + + Returns: + self + """ + if old_name not in self: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + + self[new_name] = self[old_name] + del self[old_name] + if move_references: + self.move_references(old_name, new_name) + return self + + def rename_top(self, name: str) -> Self: + """ + Rename the (single) top pattern + """ + self.rename(self.top(), name, move_references=True) + return self + + def move_references(self, old_target: str, new_target: str) -> Self: + """ + Change all references pointing at `old_target` into references pointing at `new_target`. + + Args: + old_target: Current reference target + new_target: New target for the reference + + Returns: + self + """ + if old_target == new_target: + return self + + for pattern in self.values(): + if old_target in pattern.refs: + pattern.refs[new_target].extend(pattern.refs[old_target]) + del pattern.refs[old_target] + return self + + def map_layers( + self, + map_layer: Callable[[layer_t], layer_t], + ) -> Self: + """ + Move all the elements in all patterns from one layer onto a different layer. + Can also handle multiple such mappings simultaneously. + + Args: + map_layer: Callable which may be called with each layer present in `elements`, + and should return the new layer to which it will be mapped. + A simple example which maps `old_layer` to `new_layer` and leaves all others + as-is would look like `lambda layer: {old_layer: new_layer}.get(layer, layer)` + + Returns: + self + """ + for pattern in self.values(): + pattern.shapes = map_layers(pattern.shapes, map_layer) + pattern.labels = map_layers(pattern.labels, map_layer) + return self + + def mkpat(self, name: str) -> tuple[str, Pattern]: + """ + Convenience method to create an empty pattern, add it to the library, + and return both the pattern and name. + + Args: + name: Name for the pattern + + Returns: + (name, pattern) tuple + """ + from ..pattern import Pattern # noqa: PLC0415 + pat = Pattern() + self[name] = pat + return name, pat + + def add( + self, + other: Mapping[str, Pattern], + rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + mutate_other: bool = False, + ) -> dict[str, str]: + """ + Add items from another library into this one. + + If any name in `other` is already present in `self`, `rename_theirs(self, name)` is called + to pick a new name for the newly-added pattern. If the new name still conflicts with a name + in `self` a `LibraryError` is raised. All references to the original name (within `other)` + are updated to the new name. + If `mutate_other=False` (default), all changes are made to a deepcopy of `other`. + + By default, `rename_theirs` makes no changes to the name (causing a `LibraryError`) unless the + name starts with `SINGLE_USE_PREFIX`. Prefixed names are truncated to before their first + non-prefix '$' and then passed to `self.get_name()` to create a new unique name. + + Args: + other: The library to insert keys from. + rename_theirs: Called as rename_theirs(self, name) for each duplicate name + encountered in `other`. Should return the new name for the pattern in + `other`. See above for default behavior. + mutate_other: If `True`, modify the original library and its contained patterns + (e.g. when renaming patterns and updating refs). Otherwise, operate on a deepcopy + (default). + + Returns: + A mapping of `{old_name: new_name}` for all names in `other` which were + renamed while being added. Unchanged names are omitted. + + Raises: + `LibraryError` if a duplicate name is encountered even after applying `rename_theirs()`. + """ + from ..pattern import map_targets # noqa: PLC0415 + from .mapping import Library # noqa: PLC0415 + + duplicates = set(self.keys()) & set(other.keys()) + + if not duplicates: + if mutate_other: + temp = other + else: + temp = Library(copy.deepcopy(dict(other))) + + for key in temp: + self._merge(key, temp, key) + return {} + + if mutate_other: + if isinstance(other, Library): + temp = other + else: + temp = Library(dict(other)) + else: + temp = Library(copy.deepcopy(dict(other))) + rename_map = {} + for old_name in temp: + if old_name in self: + new_name = rename_theirs(self, old_name) + if new_name in self: + raise LibraryError(f'Unresolved duplicate key encountered in library merge: {old_name} -> {new_name}') + rename_map[old_name] = new_name + else: + new_name = old_name + + self._merge(new_name, temp, old_name) + + # Update references in the newly-added cells + for old_name in temp: + new_name = rename_map.get(old_name, old_name) + pat = self[new_name] + pat.refs = map_targets(pat.refs, lambda tt: cast('dict[str | None, str | None]', rename_map).get(tt, tt)) + + return rename_map + + def __lshift__(self, other: TreeView) -> str: + """ + `add()` items from a tree (single-topcell name: pattern mapping) into this one, + and return the name of the tree's topcell (in this library; it may have changed + based on `add()`'s default `rename_theirs` argument). + + Raises: + LibraryError if there is more than one topcell in `other`. + """ + from .mapping import LibraryView # noqa: PLC0415 + + if len(other) == 1: + name = next(iter(other)) + else: + if not isinstance(other, ILibraryView): + other = LibraryView(other) + + tops = other.tops() + if len(tops) > 1: + raise LibraryError('Received a library containing multiple topcells!') + + name = tops[0] + + rename_map = self.add(other) + new_name = rename_map.get(name, name) + return new_name + + def __le__(self, other: Mapping[str, Pattern]) -> Abstract: + """ + Perform the same operation as `__lshift__` / `<<`, but return an `Abstract` instead + of just the pattern's name. + + Raises: + LibraryError if there is more than one topcell in `other`. + """ + new_name = self << other + return self.abstract(new_name) + + def dedup( + self, + norm_value: int = int(1e6), + exclude_types: tuple[type] = (Polygon,), + label2name: Callable[[tuple], str] | None = None, + threshold: int = 2, + ) -> Self: + """ + Iterates through all `Pattern`s. Within each `Pattern`, it iterates + over all shapes, calling `.normalized_form(norm_value)` on them to retrieve a scale-, + offset-, and rotation-independent form. Each shape whose normalized form appears + more than once is removed and re-added using `Ref` objects referencing a newly-created + `Pattern` containing only the normalized form of the shape. + + Note: + The default norm_value was chosen to give a reasonable precision when using + integer values for coordinates. + + Args: + norm_value: Passed to `shape.normalized_form(norm_value)`. Default `1e6` (see function + note) + exclude_types: Shape types passed in this argument are always left untouched, for + speed or convenience. Default: `(shapes.Polygon,)` + label2name: Given a label tuple as returned by `shape.normalized_form(...)`, pick + a name for the generated pattern. + Default `self.get_name(SINGLE_USE_PREIX + 'shape')`. + threshold: Only replace shapes with refs if there will be at least this many + instances. + + Returns: + self + """ + # This currently simplifies globally (same shape in different patterns is + # merged into the same ref target). + + from ..pattern import Pattern # noqa: PLC0415 + + if exclude_types is None: + exclude_types = () + + if label2name is None: + def label2name(label: tuple) -> str: # noqa: ARG001 + return self.get_name(SINGLE_USE_PREFIX + 'shape') + + used_names = set(self.keys()) + + def reserve_target_name(label: tuple) -> str: + base_name = label2name(label) + name = base_name + ii = sum(1 for nn in used_names if nn.startswith(base_name)) if base_name in used_names else 0 + while name in used_names or name == '': + name = base_name + b64suffix(ii) + ii += 1 + used_names.add(name) + return name + + shape_counts: MutableMapping[tuple, int] = defaultdict(int) + shape_funcs = {} + + # ## First pass ## + # Using the label tuple from `.normalized_form()` as a key, check how many of each shape + # are present and store the shape function for each one + for pat in tuple(self.values()): + for layer, sseq in pat.shapes.items(): + for shape in sseq: + if not any(isinstance(shape, t) for t in exclude_types): + base_label, _values, func = shape.normalized_form(norm_value) + label = (*base_label, layer) + shape_funcs[label] = func + shape_counts[label] += 1 + + shape_pats = {} + target_names = {} + for label, count in shape_counts.items(): + if count < threshold: + continue + + shape_func = shape_funcs[label] + shape_pat = Pattern() + shape_pat.shapes[label[-1]] += [shape_func()] + shape_pats[label] = shape_pat + target_names[label] = reserve_target_name(label) + + # ## Second pass ## + for pat in tuple(self.values()): + # Store `[(index_in_shapes, values_from_normalized_form), ...]` for all shapes which + # are to be replaced. + # The `values` are `(offset, scale, rotation)`. + + shape_table: dict[tuple, list] = defaultdict(list) + for layer, sseq in pat.shapes.items(): + for ii, shape in enumerate(sseq): + if any(isinstance(shape, tt) for tt in exclude_types): + continue + + base_label, values, _func = shape.normalized_form(norm_value) + label = (*base_label, layer) + + if label not in shape_pats: + continue + + shape_table[label].append((ii, values)) + + # For repeated shapes, create a `Pattern` holding a normalized shape object, + # and add `pat.refs` entries for each occurrence in pat. Also, note down that + # we should delete the `pat.shapes` entries for which we made `Ref`s. + for label, shape_entries in shape_table.items(): + layer = label[-1] + target = target_names[label] + shapes_to_remove = [] + for ii, values in shape_entries: + offset, scale, rotation, mirror_x = values + pat.ref(target=target, offset=offset, scale=scale, + rotation=rotation, mirrored=mirror_x) + shapes_to_remove.append(ii) + + # Remove any shapes for which we have created refs. + for ii in sorted(shapes_to_remove, reverse=True): + del pat.shapes[layer][ii] + + for ll, pp in shape_pats.items(): + self[target_names[ll]] = pp + + return self + + def wrap_repeated_shapes( + self, + name_func: Callable[[Pattern, Shape | Label], str] | None = None, + ) -> Self: + """ + Wraps all shapes and labels with a non-`None` `repetition` attribute + into a `Ref`/`Pattern` combination, and applies the `repetition` + to each `Ref` instead of its contained shape. + + Args: + name_func: Function f(this_pattern, shape) which generates a name for the + wrapping pattern. + Default is `self.get_name(SINGLE_USE_PREFIX + 'rep')`. + + Returns: + self + """ + from ..pattern import Pattern # noqa: PLC0415 + + if name_func is None: + def name_func(_pat: Pattern, _shape: Shape | Label) -> str: + return self.get_name(SINGLE_USE_PREFIX + 'rep') + + for pat in tuple(self.values()): + for layer in pat.shapes: + new_shapes = [] + for shape in pat.shapes[layer]: + if shape.repetition is None: + new_shapes.append(shape) + continue + + name = name_func(pat, shape) + self[name] = Pattern(shapes={layer: [shape]}) + pat.ref(name, repetition=shape.repetition) + shape.repetition = None + pat.shapes[layer] = new_shapes + + for layer in pat.labels: + new_labels = [] + for label in pat.labels[layer]: + if label.repetition is None: + new_labels.append(label) + continue + name = name_func(pat, label) + self[name] = Pattern(labels={layer: [label]}) + pat.ref(name, repetition=label.repetition) + label.repetition = None + pat.labels[layer] = new_labels + + return self + + def resolve_repeated_refs(self, name: str | None = None) -> Self: + """ + Expand all repeated references into multiple individual references. + Alters the library in-place. + + Args: + name: If specified, only resolve repeated refs in this pattern. + Otherwise, resolve in all patterns. + + Returns: + self + """ + if name is not None: + self[name].resolve_repeated_refs() + else: + for pat in self.values(): + pat.resolve_repeated_refs() + return self + + def subtree( + self, + tops: str | Sequence[str], + ) -> Self: + """ + Return a new `ILibraryView`, containing only the specified patterns and the patterns they + reference (recursively). + Dangling references do not cause an error. + + Args: + tops: Name(s) of patterns to keep + + Returns: + An object of the same type as `self` containing only `tops` and the patterns they reference. + """ + if isinstance(tops, str): + tops = (tops,) + + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + + new = type(self)() + for key in keep & set(self.keys()): + new._merge(key, self, key) + return new + + def prune_empty( + self, + repeat: bool = True, + dangling: dangling_mode_t = 'error', + ) -> set[str]: + """ + Delete any empty patterns (i.e. where `Pattern.is_empty` returns `True`). + + Args: + repeat: Also recursively delete any patterns which only contain(ed) empty patterns. + dangling: Passed to `parent_graph()`. + + Returns: + A set containing the names of all deleted patterns + """ + parent_graph = self.parent_graph(dangling=dangling) + empty = {name for name, pat in self.items() if pat.is_empty()} + trimmed = set() + while empty: + parents = set() + for name in empty: + del self[name] + for parent in parent_graph[name]: + del self[parent].refs[name] + parents |= parent_graph[name] + + trimmed |= empty + if not repeat: + break + + empty = {parent for parent in parents if self[parent].is_empty()} + return trimmed + + def delete( + self, + key: str, + delete_refs: bool = True, + ) -> Self: + """ + Delete a pattern and (optionally) all refs pointing to that pattern. + + Args: + key: Name of the pattern to be deleted. + delete_refs: If `True` (default), also delete all refs pointing to the pattern. + """ + del self[key] + if delete_refs: + for pat in self.values(): + if key in pat.refs: + del pat.refs[key] + return self + +class AbstractView(Mapping[str, Abstract]): + """ + A read-only mapping from names to `Abstract` objects. + + This is usually just used as a shorthand for repeated calls to `library.abstract()`. + """ + library: ILibraryView + + def __init__(self, library: ILibraryView) -> None: + self.library = library + + def __getitem__(self, key: str) -> Abstract: + return self.library.abstract(key) + + def __iter__(self) -> Iterator[str]: + return self.library.__iter__() + + def __len__(self) -> int: + return self.library.__len__() diff --git a/masque/library/build.py b/masque/library/build.py new file mode 100644 index 0000000..8582a52 --- /dev/null +++ b/masque/library/build.py @@ -0,0 +1,742 @@ +"""Two-phase build library implementation.""" +from __future__ import annotations + +from collections import defaultdict +from contextvars import ContextVar +from dataclasses import dataclass, replace +from functools import wraps +from pprint import pformat +from typing import TYPE_CHECKING, Any, Literal, Self, cast +import copy + +from ..error import BuildError, LibraryError +from .base import ILibrary, ILibraryView +from .utils import TreeView, dangling_mode_t, _plan_source_names, _rename_patterns, _source_rename_map +from .mapping import Library, LibraryView +from .overlay import OverlayLibrary + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping, Sequence + + from ..abstract import Abstract + from ..pattern import Pattern + + +_ACTIVE_BUILD_SESSIONS: ContextVar[dict[int, _BuildSessionLibrary] | None] = ContextVar( + 'masque_active_build_sessions', + default=None, +) + + +@dataclass(frozen=True) +class CellProvenance: + """ + Provenance record for one cell in a completed build output. + + Each output name in a `BuildReport` maps to one `CellProvenance`. The + record captures both where the cell came from and how its visible name was + chosen. + + Attributes: + requested_name: First name requested for this cell during the build. + kind: Whether the cell came from a declaration, helper emission, or an + imported source library. + owner_declared_name: Declared cell responsible for this output cell, if + any. Imported source cells leave this as `None`. + build_chain: Declared-cell dependency chain that was active when the + cell was emitted. + """ + requested_name: str + kind: Literal['declared', 'helper', 'source'] + owner_declared_name: str | None + build_chain: tuple[str, ...] + + +@dataclass(frozen=True) +class BuildReport: + """ + Immutable summary of one `BuildLibrary.validate()` or `.build()` run. + + The report is designed to answer two questions after a build completes: + which declared cells depended on which other declared cells, and where each + output cell came from. + + Attributes: + requested_roots: Roots explicitly requested for the run. A full + `build()` uses all declared cells. + provenance: Mapping from final output name to provenance metadata. + dependency_graph: Declared-cell dependency graph discovered through + library-mediated reads and explicit recipe hints. + """ + requested_roots: tuple[str, ...] + provenance: Mapping[str, CellProvenance] + dependency_graph: Mapping[str, frozenset[str]] + +@dataclass +class _BuildRecipe: + """ Captured deferred call to a pattern factory. """ + func: Callable[..., Pattern] + args: tuple[Any, ...] + kwargs: dict[str, Any] + explicit_dependencies: tuple[str, ...] = () + + def depends_on(self, *names: str) -> _BuildRecipe: + self.explicit_dependencies += tuple(names) + return self + + +def cell(func: Callable[..., Pattern]) -> Callable[..., _BuildRecipe]: + """ + Wrap a plain pattern factory so calls return deferred build recipes. + + Use as either `cell(fn)(...)` or `@cell`. + """ + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> _BuildRecipe: + return _BuildRecipe(func=func, args=args, kwargs=kwargs) + + return wrapper + + +class BuildCellsView: + """ + Attribute-based declaration namespace for `BuildLibrary`. + + This is the ergonomic authoring surface exposed as `builder.cells`. It is + intentionally write-focused: attribute assignment and deletion register + declarations, while attribute reads fail with guidance to build first and + use the returned library. + """ + + def __init__(self, library: BuildLibrary) -> None: + object.__setattr__(self, '_library', library) + + def __getattr__(self, name: str) -> Pattern: + raise BuildError( + f'BuildLibrary.cells.{name} is write-only during authoring. ' + 'Call build() and index the returned library instead.' + ) + + def __setattr__(self, name: str, value: Pattern | _BuildRecipe) -> None: + if name.startswith('_'): + object.__setattr__(self, name, value) + return + self._library[name] = value + + def __delattr__(self, name: str) -> None: + if name.startswith('_'): + raise AttributeError(name) + del self._library[name] + + +class BuildLibrary(ILibrary): + """ + Two-phase declaration surface for mixed imported/generated libraries. + + A `BuildLibrary` collects three kinds of inputs: + - direct declared `Pattern` objects + - deferred recipes created with `cell(...)` + - imported source-backed library views added with `add_source(...)` + + The builder itself is not a normal readable library during authoring. + Instead, `validate()` and `build()` create a temporary build-session library + that recipes can read from and write helper cells into while dependencies + are resolved. `build()` then freezes the builder on success and returns the + built library plus a `BuildReport`. + """ + + def __init__(self) -> None: + self.cells = BuildCellsView(self) + self._frozen = False + self._declarations: dict[str, Pattern | _BuildRecipe] = {} + self._sources: list[tuple[ILibraryView, dict[str, str]]] = [] + self._names: dict[str, None] = {} + + def _active_session(self) -> _BuildSessionLibrary | None: + sessions = _ACTIVE_BUILD_SESSIONS.get() + if sessions is None: + return None + return sessions.get(id(self)) + + def _require_active_session(self, operation: str) -> _BuildSessionLibrary: + session = self._active_session() + if session is None: + raise BuildError( + f'BuildLibrary.{operation}() is only available while validate() or build() is running. ' + 'Use the built output library for reads.' + ) + return session + + def _assert_editable(self) -> None: + if self._frozen: + raise BuildError('This BuildLibrary has already been built successfully and is now frozen.') + + def __iter__(self) -> Iterator[str]: + session = self._active_session() + if session is not None: + return iter(session) + return iter(self._names) + + def __len__(self) -> int: + session = self._active_session() + if session is not None: + return len(session) + return len(self._names) + + def __contains__(self, key: object) -> bool: + session = self._active_session() + if session is not None: + return key in session + return key in self._names + + def __getitem__(self, key: str) -> Pattern: + return self._require_active_session('__getitem__')[key] + + def __setitem__( + self, + key: str, + value: Pattern | _BuildRecipe, + ) -> None: + session = self._active_session() + if session is not None: + session[key] = value + return + + self._assert_editable() + if key in self._names: + raise LibraryError(f'"{key}" already exists in the builder. Overwriting is not allowed!') + + if isinstance(value, _BuildRecipe): + declaration = value + else: + if callable(value): + raise TypeError('BuildLibrary recipes must be wrapped with cell(fn)(...) or @cell.') + declaration = value + + self._declarations[key] = declaration + self._names[key] = None + + def __delitem__(self, key: str) -> None: + session = self._active_session() + if session is not None: + del session[key] + return + + self._assert_editable() + if key not in self._declarations: + raise KeyError(key) + del self._declarations[key] + del self._names[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + session = self._active_session() + if session is not None: + session._merge(key_self, other, key_other) + return + self[key_self] = copy.deepcopy(other[key_other]) + + def add( + self, + other: Mapping[str, Pattern], + rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + mutate_other: bool = False, + ) -> dict[str, str]: + from ..pattern import map_targets # noqa: PLC0415 + + session = self._active_session() + if session is not None: + return session.add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) + + self._assert_editable() + + source_backed = isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView) + if source_backed: + if mutate_other: + raise BuildError('BuildLibrary.add(..., mutate_other=True) is not supported for source-backed inputs.') + return self.add_source( + other, + rename_theirs = rename_theirs, + rename_when = 'conflict', + ) + + source_order = tuple(other.keys()) + source_to_visible = _plan_source_names( + self, + source_order, + self._names, + rename_theirs = rename_theirs, + rename_when = 'conflict', + ) + rename_map = _source_rename_map(source_to_visible) + + if mutate_other: + temp = other + else: + temp = Library(copy.deepcopy(dict(other))) + + for source_name in source_order: + visible_name = source_to_visible[source_name] + pattern = temp[source_name] + if rename_map: + pattern.refs = map_targets( + pattern.refs, + lambda target: cast('dict[str | None, str | None]', rename_map).get(target, target), + ) + self[visible_name] = pattern + + return rename_map + + def __lshift__(self, other: TreeView) -> str: + session = self._active_session() + if session is not None: + return session << other + + self._assert_editable() + if len(other) == 1: + name = next(iter(other)) + elif isinstance(other, ILibraryView) and not isinstance(other, Library | LibraryView): + source_order = other.source_order() + child_graph = other.child_graph(dangling='include') + referenced = set().union(*child_graph.values()) if child_graph else set() + tops = [candidate for candidate in source_order if candidate not in referenced] + if len(tops) != 1: + raise LibraryError(f'Asked for the single topcell, but found the following: {pformat(tops)}') + name = tops[0] + else: + return super().__lshift__(other) + + rename_map = self.add(other) + return rename_map.get(name, name) + + def __le__(self, other: Mapping[str, Pattern]) -> Abstract: + if self._active_session() is not None: + return super().__le__(other) + raise BuildError('BuildLibrary.__le__() is only available while validate() or build() is running.') + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + """ + Rename a helper cell during an active build session. + + During authoring, declared cells must be registered under their + intended final names and imported source cells must be renamed through + `add_source(...)`. + """ + session = self._active_session() + if session is not None: + session.rename(old_name, new_name, move_references=move_references) + return self + + self._assert_editable() + if old_name == new_name: + return self + if old_name in self._declarations: + raise BuildError( + f'Cannot rename declared build cell "{old_name}" during authoring. ' + 'Register it under the intended final name instead.' + ) + if old_name not in self._names: + raise LibraryError(f'"{old_name}" does not exist in the builder.') + raise BuildError( + f'Cannot rename imported source cell "{old_name}" during authoring. ' + 'Choose visible source names with add_source(..., rename_theirs=..., rename_when=...).' + ) + + def abstract(self, name: str) -> Abstract: + return self._require_active_session('abstract').abstract(name) + + def resolve( + self, + other: Abstract | str | Pattern | TreeView, + append: bool = False, + ) -> Abstract | Pattern: + return self._require_active_session('resolve').resolve(other, append=append) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_when: Literal['conflict', 'always'] = 'conflict', + ) -> dict[str, str]: + """ + Register an imported source-backed library with the builder. + + The source is not materialized immediately. Its names are scanned once + to reserve visible builder names, then the source is read again when a + build session starts. The source's cell membership must not be + structurally mutated between `add_source()` and `build()`/`validate()`. + Source cells may be renamed on entry to avoid collisions with existing + declarations or other imported sources. + + Args: + rename_theirs: Function used to choose visible names for imported + source cells. + rename_when: If `'conflict'`, only conflicting names are renamed. + If `'always'`, every imported source name is passed through + `rename_theirs`. + + Returns: + Mapping of `{source_name: visible_name}` for imported names that + were renamed while being added. + """ + if self._active_session() is not None: + raise BuildError('BuildLibrary.add_source() is only available while authoring, not during validate() or build().') + self._assert_editable() + + view = source if isinstance(source, ILibraryView) else LibraryView(source) + source_order = tuple(view.source_order()) + source_to_visible = _plan_source_names( + self, + source_order, + self._names, + rename_theirs = rename_theirs, + rename_when = rename_when, + ) + + self._sources.append((view, dict(source_to_visible))) + for source_name in source_order: + visible = source_to_visible[source_name] + self._names[visible] = None + return _source_rename_map(source_to_visible) + + def validate( + self, + names: Sequence[str] | None = None, + *, + allow_dangling: bool = False, + ) -> BuildReport: + """ + Run the full build logic and return a `BuildReport` without producing output. + + This is a dry run over the same dependency resolution and recipe + execution path used by `build()`. Any generated library is discarded + after validation completes. + """ + _session, report = self._run_build(names=names, allow_dangling=allow_dangling) + return report + + def build( + self, + *, + output: Literal['overlay', 'library'] = 'overlay', + allow_dangling: bool = False, + ) -> tuple[ILibrary, BuildReport]: + """ + Materialize declarations and return a usable output library plus report. + + Args: + output: `'overlay'` preserves imported source-backed cells where + possible, while `'library'` eagerly materializes the full + result. + allow_dangling: If `False`, fail the build when the completed + library still contains dangling references. + """ + if output not in ('overlay', 'library'): + raise ValueError(f'Unknown build output mode: {output!r}') + self._assert_editable() + session, report = self._run_build(names=None, allow_dangling=allow_dangling) + if output == 'library': + built_output = session.to_library() + else: + built_output = session.to_overlay() + self._frozen = True + return built_output, report + + def _run_build( + self, + *, + names: Sequence[str] | None, + allow_dangling: bool, + ) -> tuple[_BuildSessionLibrary, BuildReport]: + roots = tuple(dict.fromkeys(names if names is not None else self._declarations.keys())) + unknown = [name for name in roots if name not in self._names] + if unknown: + raise BuildError(f'Unknown build roots requested: {unknown}') + + session = _BuildSessionLibrary(self) + sessions = dict(_ACTIVE_BUILD_SESSIONS.get() or {}) + sessions[id(self)] = session + token = _ACTIVE_BUILD_SESSIONS.set(sessions) + try: + session.materialize_many(roots) + if not allow_dangling: + session.child_graph(dangling='error') + finally: + _ACTIVE_BUILD_SESSIONS.reset(token) + + report = session.build_report(roots) + return session, report + + +class _BuildSessionLibrary(ILibrary): + """ + Internal overlay-backed library used while a `BuildLibrary` is executing. + + This object provides the mutable-library surface that recipes expect while + also tracking declared-cell dependencies, helper-cell provenance, and + imported source cells. It exists only for the duration of a validation or + build run. + """ + + def __init__(self, builder: BuildLibrary) -> None: + self._builder = builder + self._overlay = OverlayLibrary() + self._built: set[str] = set() + self._declared_stack: list[str] = [] + self._names = dict(builder._names) + self._provenance: dict[str, CellProvenance] = {} + self._dependency_graph: defaultdict[str, set[str]] = defaultdict(set) + self._install_sources() + + def _install_sources(self) -> None: + for source_library, source_to_visible in self._builder._sources: + source_order = source_library.source_order() + expected_names = set(source_to_visible) + actual_names = set(source_order) + if actual_names != expected_names: + added_names = sorted(actual_names - expected_names) + removed_names = sorted(expected_names - actual_names) + detail = [] + if added_names: + detail.append(f'added={added_names}') + if removed_names: + detail.append(f'removed={removed_names}') + raise BuildError( + 'Imported source library changed after add_source() was called ' + f'({", ".join(detail)}). ' + 'Do not structurally mutate source libraries between add_source() and build()/validate().' + ) + + def rename_source(_lib: ILibraryView, name: str, *, mapping: Mapping[str, str] = source_to_visible) -> str: + return mapping[name] + + self._overlay.add_source( + source_library, + rename_theirs = rename_source, + rename_when = 'always', + ) + + for source_name in source_order: + visible_name = source_to_visible[source_name] + self._provenance[visible_name] = CellProvenance( + requested_name = source_name, + kind = 'source', + owner_declared_name = None, + build_chain = (), + ) + + def __iter__(self) -> Iterator[str]: + return iter(self._names) + + def __len__(self) -> int: + return len(self._names) + + def __contains__(self, key: object) -> bool: + return key in self._names + + def _current_declared(self) -> str | None: + if not self._declared_stack: + return None + return self._declared_stack[-1] + + def _record_dependency(self, target: str) -> None: + current = self._current_declared() + if current is None or current == target or target not in self._builder._declarations: + return + self._dependency_graph[current].add(target) + + def _guard_mutable_output_name(self, key: str, *, operation: str) -> None: + if key in self._builder._declarations: + raise BuildError(f'Cannot {operation} declared build cell "{key}" during an active build session.') + + provenance = self._provenance.get(key) + if provenance is not None and provenance.kind == 'source': + raise BuildError(f'Cannot {operation} imported source cell "{key}" during an active build session.') + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + if old_name == new_name: + return self + if old_name not in self._overlay: + if old_name in self._builder._declarations: + self._guard_mutable_output_name(old_name, operation='rename') + raise LibraryError(f'"{old_name}" does not exist in the library.') + + self._guard_mutable_output_name(old_name, operation='rename') + if new_name in self._names: + raise LibraryError(f'"{new_name}" already exists in the library.') + + self._overlay.rename(old_name, new_name, move_references=move_references) + self._names = { + new_name if name == old_name else name: None + for name in self._names + } + + provenance = self._provenance.pop(old_name) + self._provenance[new_name] = provenance + return self + + def __getitem__(self, key: str) -> Pattern: + if key in self._builder._declarations: + self._record_dependency(key) + self._ensure_declared(key) + return self._overlay[key] + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._overlay: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + current = self._current_declared() + if key in self._builder._declarations and key != current: + raise LibraryError(f'"{key}" is reserved for a declared cell and cannot be used as a helper name.') + + pattern = value() if callable(value) else value + self._overlay[key] = pattern + self._names.setdefault(key, None) + + kind: Literal['declared', 'helper'] + if current is not None and key == current: + kind = 'declared' + else: + kind = 'helper' + + self._provenance[key] = CellProvenance( + requested_name = key, + kind = kind, + owner_declared_name = current if kind == 'helper' else key, + build_chain = tuple(self._declared_stack), + ) + + def __delitem__(self, key: str) -> None: + if key not in self._overlay: + if key in self._builder._declarations: + self._guard_mutable_output_name(key, operation='delete') + raise KeyError(key) + + self._guard_mutable_output_name(key, operation='delete') + if key in self._overlay: + del self._overlay[key] + self._names.pop(key, None) + self._provenance.pop(key, None) + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add( + self, + other: Mapping[str, Pattern], + rename_theirs: Callable[[ILibraryView, str], str] = _rename_patterns, + mutate_other: bool = False, + ) -> dict[str, str]: + rename_map = super().add(other, rename_theirs=rename_theirs, mutate_other=mutate_other) + + current = self._current_declared() + for old_name, new_name in rename_map.items(): + if new_name in self._provenance: + self._provenance[new_name] = replace( + self._provenance[new_name], + requested_name = old_name, + owner_declared_name = current if current is not None else self._provenance[new_name].owner_declared_name, + ) + return rename_map + + def _wrap_error(self, name: str, exc: Exception) -> BuildError: + chain = tuple(self._declared_stack) + msg = [f'Failed while building declared cell "{name}"'] + if chain: + msg.append(f'Dependency chain: {" -> ".join(chain)}') + msg.append(f'Cause: {exc}') + return BuildError('\n'.join(msg)) + + def _ensure_named(self, name: str) -> None: + if name in self._builder._declarations: + self._record_dependency(name) + self._ensure_declared(name) + return + if name in self._overlay: + return + raise BuildError(f'Missing dependency "{name}"') + + def _ensure_declared(self, name: str) -> None: + from ..pattern import Pattern # noqa: PLC0415 + + if name in self._built: + return + if name in self._declared_stack: + chain = ' -> '.join(self._declared_stack + [name]) + raise BuildError(f'Cycle detected while building declared cells: {chain}') + + declaration = self._builder._declarations[name] + self._declared_stack.append(name) + try: + if isinstance(declaration, _BuildRecipe): + for dep in declaration.explicit_dependencies: + self._ensure_named(dep) + pattern = declaration.func(*declaration.args, **declaration.kwargs) + if not isinstance(pattern, Pattern): + raise BuildError(f'Recipe for "{name}" returned {type(pattern).__name__}, expected Pattern') # noqa: TRY301 + else: + pattern = declaration.deepcopy() + + if name in self._overlay: + if self._overlay[name] is not pattern: + raise BuildError( # noqa: TRY301 + f'Recipe for "{name}" wrote a different pattern into the session under its own name.' + ) + else: + self[name] = pattern + self._built.add(name) + except Exception as exc: + raise self._wrap_error(name, exc) from exc + finally: + self._declared_stack.pop() + + def materialize_many(self, names: Sequence[str]) -> None: + for name in dict.fromkeys(names): + self._ensure_named(name) + + def source_order(self) -> tuple[str, ...]: + return self._overlay.source_order() + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._overlay.child_graph(dangling=dangling) + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._overlay.parent_graph(dangling=dangling) + + def build_report(self, requested_roots: Sequence[str]) -> BuildReport: + dependency_graph = { + name: frozenset(self._dependency_graph.get(name, set())) + for name in self._builder._declarations + if name in self._dependency_graph or name in requested_roots + } + return BuildReport( + requested_roots = tuple(dict.fromkeys(requested_roots)), + provenance = dict(self._provenance), + dependency_graph = dependency_graph, + ) + + def to_overlay(self) -> ILibrary: + return self._overlay + + def to_library(self) -> Library: + mapping = {name: self._overlay[name] for name in self._overlay.source_order()} + return Library(mapping) diff --git a/masque/library/lazy.py b/masque/library/lazy.py new file mode 100644 index 0000000..973a431 --- /dev/null +++ b/masque/library/lazy.py @@ -0,0 +1,169 @@ +"""Closure-backed lazy library implementation.""" +from __future__ import annotations + +from pprint import pformat +from typing import TYPE_CHECKING, Self, cast +import logging + +from ..error import LibraryError +from .base import ILibrary + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + + from ..pattern import Pattern + + +logger = logging.getLogger(__name__) + + +class LazyLibrary(ILibrary): + """ + This class is usually used to create a library of Patterns by mapping names to + functions which generate or load the relevant `Pattern` object as-needed. + + TODO: lots of stuff causes recursive loads (e.g. data_to_ports?). What should you avoid? + """ + mapping: dict[str, Callable[[], Pattern]] + cache: dict[str, Pattern] + _lookups_in_progress: list[str] + + def __init__(self) -> None: + self.mapping = {} + self.cache = {} + self._lookups_in_progress = [] + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self.mapping: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + + if callable(value): + value_func = value + else: + value_func = lambda: cast('Pattern', value) # noqa: E731 + + self.mapping[key] = value_func + if key in self.cache: + del self.cache[key] + + def __delitem__(self, key: str) -> None: + del self.mapping[key] + if key in self.cache: + del self.cache[key] + + def __getitem__(self, key: str) -> Pattern: + logger.debug(f'loading {key}') + if key in self.cache: + logger.debug(f'found {key} in cache') + return self.cache[key] + + if key in self._lookups_in_progress: + chain = ' -> '.join(self._lookups_in_progress + [key]) + raise LibraryError( + f'Detected circular reference or recursive lookup of "{key}".\n' + f'Lookup chain: {chain}\n' + 'This may be caused by an invalid (cyclical) reference, or buggy code.\n' + 'If you are lazy-loading a file, try a non-lazy load and check for reference cycles.' + ) + + self._lookups_in_progress.append(key) + try: + func = self.mapping[key] + pat = func() + finally: + self._lookups_in_progress.pop() + self.cache[key] = pat + return pat + + def __iter__(self) -> Iterator[str]: + return iter(self.mapping) + + def __len__(self) -> int: + return len(self.mapping) + + def __contains__(self, key: object) -> bool: + return key in self.mapping + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + if isinstance(other, LazyLibrary): + self.mapping[key_self] = other.mapping[key_other] + if key_other in other.cache: + self.cache[key_self] = other.cache[key_other] + else: + self[key_self] = other[key_other] + + def __repr__(self) -> str: + return '' + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> Self: + """ + Rename a pattern. + + Args: + old_name: Current name for the pattern + new_name: New name for the pattern + move_references: Whether to scan all refs in the pattern and + move them to point to `new_name` as necessary. + Default `False`. + + Returns: + self + """ + if old_name not in self.mapping: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + + self[new_name] = self.mapping[old_name] # copy over function + if old_name in self.cache: + self.cache[new_name] = self.cache[old_name] + del self[old_name] + + if move_references: + self.move_references(old_name, new_name) + + return self + + def move_references(self, old_target: str, new_target: str) -> Self: + """ + Change all references pointing at `old_target` into references pointing at `new_target`. + + Args: + old_target: Current reference target + new_target: New target for the reference + + Returns: + self + """ + if old_target == new_target: + return self + + self.precache() + for pattern in self.cache.values(): + if old_target in pattern.refs: + pattern.refs[new_target].extend(pattern.refs[old_target]) + del pattern.refs[old_target] + return self + + def precache(self) -> Self: + """ + Force all patterns into the cache + + Returns: + self + """ + for key in self.mapping: + _ = self[key] # want to trigger our own __getitem__ + return self + + def __deepcopy__(self, memo: dict | None = None) -> LazyLibrary: + raise LibraryError('LazyLibrary cannot be deepcopied (deepcopy doesn\'t descend into closures)') diff --git a/masque/library/mapping.py b/masque/library/mapping.py new file mode 100644 index 0000000..95c7e73 --- /dev/null +++ b/masque/library/mapping.py @@ -0,0 +1,112 @@ +"""Concrete mapping-backed library implementations.""" +from __future__ import annotations + +from pprint import pformat +from typing import TYPE_CHECKING, Self + +from ..error import LibraryError +from .base import ILibrary, ILibraryView + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping, MutableMapping + + from ..pattern import Pattern + + +class LibraryView(ILibraryView): + """ + Default implementation for a read-only library. + + A library is a mapping from unique names (str) to collections of geometry (`Pattern`). + This library is backed by an arbitrary python object which implements the `Mapping` interface. + """ + mapping: Mapping[str, Pattern] + + def __init__( + self, + mapping: Mapping[str, Pattern], + ) -> None: + self.mapping = mapping + + def __getitem__(self, key: str) -> Pattern: + return self.mapping[key] + + def __iter__(self) -> Iterator[str]: + return iter(self.mapping) + + def __len__(self) -> int: + return len(self.mapping) + + def __contains__(self, key: object) -> bool: + return key in self.mapping + + def __repr__(self) -> str: + return f'' + + +class Library(ILibrary): + """ + Default implementation for a writeable library. + + A library is a mapping from unique names (str) to collections of geometry (`Pattern`). + This library is backed by an arbitrary python object which implements the `MutableMapping` interface. + """ + mapping: MutableMapping[str, Pattern] + + def __init__( + self, + mapping: MutableMapping[str, Pattern] | None = None, + ) -> None: + if mapping is None: + self.mapping = {} + else: + self.mapping = mapping + + def __getitem__(self, key: str) -> Pattern: + return self.mapping[key] + + def __iter__(self) -> Iterator[str]: + return iter(self.mapping) + + def __len__(self) -> int: + return len(self.mapping) + + def __contains__(self, key: object) -> bool: + return key in self.mapping + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self.mapping: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + + value = value() if callable(value) else value + self.mapping[key] = value + + def __delitem__(self, key: str) -> None: + del self.mapping[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = other[key_other] + + def __repr__(self) -> str: + return f'' + + @classmethod + def mktree(cls: type[Self], name: str) -> tuple[Self, Pattern]: + """ + Create a new Library and immediately add a pattern + + Args: + name: The name for the new pattern (usually the name of the topcell). + + Returns: + The newly created `Library` and the newly created `Pattern` + """ + from ..pattern import Pattern # noqa: PLC0415 + tree = cls() + pat = Pattern() + tree[name] = pat + return tree, pat diff --git a/masque/library/overlay.py b/masque/library/overlay.py new file mode 100644 index 0000000..9637236 --- /dev/null +++ b/masque/library/overlay.py @@ -0,0 +1,515 @@ +"""Overlay and ports-importing library views.""" +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, cast +import copy + +import numpy + +from ..error import LibraryError +from ..pattern import Pattern, map_targets +from ..utils import apply_transforms, layer_t +from .base import ILibrary, ILibraryView +from .utils import dangling_mode_t, _plan_source_names, _source_rename_map +from .mapping import LibraryView + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping, Sequence + + from numpy.typing import NDArray + + +@dataclass +class _SourceLayer: + """ One imported source layer tracked by an `OverlayLibrary`. """ + library: ILibraryView + source_to_visible: dict[str, str] + visible_to_source: dict[str, str] + child_graph: dict[str, set[str]] + order: list[str] + + +@dataclass(frozen=True) +class _SourceEntry: + """ Reference to a single visible source-backed cell in an overlay. """ + layer_index: int + source_name: str + + +def _materialize_detached_pattern(view: ILibraryView, name: str) -> Pattern: + func = getattr(view, '_materialize_pattern', None) + if callable(func): + return cast('Pattern', func(name, persist=False)) + return view[name].deepcopy() + + +class PortsLibraryView(ILibraryView): + """ + Read-only view which imports ports into cells on first materialization. + + The wrapped source remains untouched; this view owns a separate processed + cache so direct-copy workflows can continue to use the raw source view. + + Graph queries, source ordering, and copy-through capabilities are delegated + to the wrapped source whenever possible, while `__getitem__` and + `materialize_many()` return port-imported patterns. + """ + + def __init__( + self, + source: ILibraryView, + *, + layers: Sequence[layer_t], + max_depth: int = 0, + skip_subcells: bool = True, + ) -> None: + self._source = source + self._layers = tuple(layers) + self._max_depth = max_depth + self._skip_subcells = skip_subcells + self._cache: dict[str, Pattern] = {} + self._lookups_in_progress: list[str] = [] + if hasattr(source, 'library_info'): + self.library_info = cast('dict[str, Any]', source.library_info) + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __iter__(self) -> Iterator[str]: + return iter(self._source) + + def __len__(self) -> int: + return len(self._source) + + def __contains__(self, key: object) -> bool: + return key in self._source + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + from ..utils.ports2data import data_to_ports # noqa: PLC0415 + + if name in self._cache: + return self._cache[name] + + if name in self._lookups_in_progress: + chain = ' -> '.join(self._lookups_in_progress + [name]) + raise LibraryError( + f'Detected circular reference or recursive lookup of "{name}".\n' + f'Lookup chain: {chain}\n' + 'This may be caused by an invalid (cyclical) reference, or buggy code.' + ) + + self._lookups_in_progress.append(name) + try: + pat = _materialize_detached_pattern(self._source, name) + pat = data_to_ports( + layers=self._layers, + library=self, + pattern=pat, + name=name, + max_depth=self._max_depth, + skip_subcells=self._skip_subcells, + ) + finally: + self._lookups_in_progress.pop() + + if persist: + self._cache[name] = pat + return pat + + def materialize_many( + self, + names: Sequence[str], + *, + persist: bool = True, + ) -> LibraryView: + mats = { + name: self._materialize_pattern(name, persist=persist) + for name in dict.fromkeys(names) + } + return LibraryView(mats) + + def source_order(self) -> tuple[str, ...]: + return self._source.source_order() + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.child_graph(dangling=dangling) + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + return self._source.parent_graph(dangling=dangling) + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self._source.referenced_patterns(tops) - {None}) + keep |= set(tops) + return self.materialize_many(tuple(keep), persist=True) + + def tops(self) -> list[str]: + return self._source.tops() + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + finder = getattr(self._source, 'find_refs_local', None) + if callable(finder): + return cast('dict[str, list[NDArray[numpy.float64]]]', finder(name, parent_graph=parent_graph, dangling=dangling)) + return super().find_refs_local(name, parent_graph=parent_graph, dangling=dangling) + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + finder = getattr(self._source, 'find_refs_global', None) + if callable(finder): + return cast( + 'dict[tuple[str, ...], NDArray[numpy.float64]]', + finder(name, order=order, parent_graph=parent_graph, dangling=dangling), + ) + return super().find_refs_global(name, order=order, parent_graph=parent_graph, dangling=dangling) + + def raw_struct_bytes(self, name: str) -> bytes: + reader = getattr(self._source, 'raw_struct_bytes', None) + if not callable(reader): + raise TypeError('raw_struct_bytes') + return cast('bytes', reader(name)) + + def can_copy_raw_struct(self, name: str) -> bool: + can_copy = getattr(self._source, 'can_copy_raw_struct', None) + if not callable(can_copy): + return False + return bool(can_copy(name)) + + def close(self) -> None: + closer = getattr(self._source, 'close', None) + if callable(closer): + closer() + + def __enter__(self) -> PortsLibraryView: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + +class OverlayLibrary(ILibrary): + """ + Mutable overlay over one or more source libraries. + + Source-backed cells remain lazy until accessed through `__getitem__`, at + which point that visible cell is promoted into an overlay-owned materialized + `Pattern`. + """ + + def __init__(self) -> None: + self._layers: list[_SourceLayer] = [] + self._entries: dict[str, Pattern | _SourceEntry] = {} + self._order: list[str] = [] + self._target_remap: dict[str, str] = {} + + def __iter__(self) -> Iterator[str]: + return (name for name in self._order if name in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, key: object) -> bool: + return key in self._entries + + def __getitem__(self, key: str) -> Pattern: + return self._materialize_pattern(key, persist=True) + + def __setitem__( + self, + key: str, + value: Pattern | Callable[[], Pattern], + ) -> None: + if key in self._entries: + raise LibraryError(f'"{key}" already exists in the library. Overwriting is not allowed!') + pattern = value() if callable(value) else value + self._entries[key] = pattern + if key not in self._order: + self._order.append(key) + + def __delitem__(self, key: str) -> None: + if key not in self._entries: + raise KeyError(key) + del self._entries[key] + + def _merge(self, key_self: str, other: Mapping[str, Pattern], key_other: str) -> None: + self[key_self] = copy.deepcopy(other[key_other]) + + def add_source( + self, + source: Mapping[str, Pattern] | ILibraryView, + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_when: Literal['conflict', 'always'] = 'conflict', + ) -> dict[str, str]: + """ + Add a source-backed library layer. + + Args: + rename_theirs: Function used to choose visible names for imported + source cells. + rename_when: If `'conflict'`, only conflicting names are renamed. + If `'always'`, every imported source name is passed through + `rename_theirs`. + """ + view = source if isinstance(source, ILibraryView) else LibraryView(source) + source_order = list(view.source_order()) + child_graph = view.child_graph(dangling='include') + + source_to_visible = _plan_source_names( + self, + source_order, + self._entries, + rename_theirs = rename_theirs, + rename_when = rename_when, + ) + visible_to_source = {visible: source_name for source_name, visible in source_to_visible.items()} + + layer = _SourceLayer( + library=view, + source_to_visible=source_to_visible, + visible_to_source=visible_to_source, + child_graph=child_graph, + order=[source_to_visible[name] for name in source_order], + ) + layer_index = len(self._layers) + self._layers.append(layer) + + for source_name, visible_name in source_to_visible.items(): + self._entries[visible_name] = _SourceEntry(layer_index=layer_index, source_name=source_name) + if visible_name not in self._order: + self._order.append(visible_name) + + return _source_rename_map(source_to_visible) + + def rename( + self, + old_name: str, + new_name: str, + move_references: bool = False, + ) -> OverlayLibrary: + if old_name not in self._entries: + raise LibraryError(f'"{old_name}" does not exist in the library.') + if old_name == new_name: + return self + if new_name in self._entries: + raise LibraryError(f'"{new_name}" already exists in the library.') + + entry = self._entries.pop(old_name) + self._entries[new_name] = entry + if isinstance(entry, _SourceEntry): + layer = self._layers[entry.layer_index] + layer.source_to_visible[entry.source_name] = new_name + del layer.visible_to_source[old_name] + layer.visible_to_source[new_name] = entry.source_name + + idx = self._order.index(old_name) + self._order[idx] = new_name + + if move_references: + self.move_references(old_name, new_name) + return self + + def _resolve_target(self, target: str) -> str: + seen: set[str] = set() + current = target + while current in self._target_remap: + if current in seen: + raise LibraryError(f'Cycle encountered while resolving target remap for {target!r}') + seen.add(current) + current = self._target_remap[current] + return current + + def _set_target_remap(self, old_target: str, new_target: str) -> None: + resolved_new = self._resolve_target(new_target) + if resolved_new == old_target: + raise LibraryError(f'Ref target remap would create a cycle: {old_target!r} -> {new_target!r}') + self._target_remap[old_target] = resolved_new + for key in list(self._target_remap): + self._target_remap[key] = self._resolve_target(self._target_remap[key]) + + def move_references(self, old_target: str, new_target: str) -> OverlayLibrary: + if old_target == new_target: + return self + self._set_target_remap(old_target, new_target) + for entry in list(self._entries.values()): + if isinstance(entry, Pattern) and old_target in entry.refs: + entry.refs[new_target].extend(entry.refs[old_target]) + del entry.refs[old_target] + return self + + def _effective_target(self, layer: _SourceLayer, target: str) -> str: + visible = layer.source_to_visible.get(target, target) + return self._resolve_target(visible) + + def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern: + if name not in self._entries: + raise KeyError(name) + entry = self._entries[name] + if isinstance(entry, Pattern): + return entry + + layer = self._layers[entry.layer_index] + source_pat = layer.library[entry.source_name].deepcopy() + + def remap(target: str | None) -> str | None: + return None if target is None else self._effective_target(layer, target) + + if source_pat.refs: + source_pat.refs = map_targets(source_pat.refs, remap) + pat = source_pat + if persist: + self._entries[name] = pat + return pat + + def child_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for name in self._order: + if name not in self._entries: + continue + entry = self._entries[name] + if isinstance(entry, Pattern): + graph[name] = {child for child, refs in entry.refs.items() if child is not None and refs} + continue + layer = self._layers[entry.layer_index] + children = {self._effective_target(layer, child) for child in layer.child_graph.get(entry.source_name, set())} + graph[name] = children + + existing = set(graph) + dangling_refs = set().union(*(children - existing for children in graph.values())) + if dangling == 'error': + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building child graph') + return graph + if dangling == 'ignore': + return {name: {child for child in children if child in existing} for name, children in graph.items()} + + for child in dangling_refs: + graph.setdefault(cast('str', child), set()) + return graph + + def parent_graph( + self, + dangling: dangling_mode_t = 'error', + ) -> dict[str, set[str]]: + child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore') + existing = set(self.keys()) + igraph: dict[str, set[str]] = {name: set() for name in child_graph} + for parent, children in child_graph.items(): + for child in children: + if child in existing or dangling == 'include': + igraph.setdefault(child, set()).add(parent) + if dangling == 'error': + raw = self.child_graph(dangling='include') + dangling_refs = set().union(*(children - existing for children in raw.values())) + if dangling_refs: + raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph') + return igraph + + def subtree( + self, + tops: str | Sequence[str], + ) -> ILibraryView: + if isinstance(tops, str): + tops = (tops,) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) + keep |= set(tops) + return LibraryView({name: self[name] for name in keep}) + + def find_refs_local( + self, + name: str, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[str, list[NDArray[numpy.float64]]]: + instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list) + if parent_graph is None: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return instances + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding local refs for {name!r}') + if dangling == 'ignore': + return instances + + for parent in parent_graph.get(name, set()): + pat = self._materialize_pattern(parent, persist=False) + for ref in pat.refs.get(name, []): + instances[parent].append(ref.as_transforms()) + return instances + + def find_refs_global( + self, + name: str, + order: list[str] | None = None, + parent_graph: dict[str, set[str]] | None = None, + dangling: dangling_mode_t = 'error', + ) -> dict[tuple[str, ...], NDArray[numpy.float64]]: + graph_mode = 'ignore' if dangling == 'ignore' else 'include' + if order is None: + order = self.child_order(dangling=graph_mode) + if parent_graph is None: + parent_graph = self.parent_graph(dangling=graph_mode) + + if name not in self: + if name not in parent_graph: + return {} + if dangling == 'error': + raise self._dangling_refs_error({name}, f'finding global refs for {name!r}') + if dangling == 'ignore': + return {} + + self_keys = set(self.keys()) + transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]] + transforms = defaultdict(list) + for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items(): + transforms[parent] = [((name,), numpy.concatenate(vals))] + + for next_name in order: + if next_name not in transforms: + continue + if not parent_graph.get(next_name, set()) & self_keys: + continue + + outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling) + inners = transforms.pop(next_name) + for parent, outer in outers.items(): + outer_tf = numpy.concatenate(outer) + for path, inner in inners: + combined = apply_transforms(outer_tf, inner) + transforms[parent].append(((next_name,) + path, combined)) + + result = {} + for parent, targets in transforms.items(): + for path, instances in targets: + result[(parent,) + path] = instances + return result + + def source_order(self) -> tuple[str, ...]: + return tuple(name for name in self._order if name in self._entries) diff --git a/masque/library/utils.py b/masque/library/utils.py new file mode 100644 index 0000000..138c91a --- /dev/null +++ b/masque/library/utils.py @@ -0,0 +1,124 @@ +"""Shared types and helpers for library implementations.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal, Protocol, TypeAlias +from collections.abc import Callable, Container, Mapping, MutableMapping, Sequence + +from ..error import LibraryError + +if TYPE_CHECKING: + import numpy + from numpy.typing import NDArray + + from ..pattern import Pattern + from .base import ILibraryView + + +class visitor_function_t(Protocol): + """ Signature for `Library.dfs()` visitor functions. """ + def __call__( + self, + pattern: Pattern, + hierarchy: tuple[str | None, ...], + memo: dict, + transform: NDArray[numpy.float64] | Literal[False], + ) -> Pattern: + ... + + +TreeView: TypeAlias = Mapping[str, 'Pattern'] +""" A name-to-`Pattern` mapping which is expected to have only one top-level cell """ + +Tree: TypeAlias = MutableMapping[str, 'Pattern'] +""" A mutable name-to-`Pattern` mapping which is expected to have only one top-level cell """ + +dangling_mode_t: TypeAlias = Literal['error', 'ignore', 'include'] +""" How helpers should handle refs whose targets are not present in the library. """ +SINGLE_USE_PREFIX = '_' +""" +Names starting with this prefix are assumed to refer to single-use patterns, +which may be renamed automatically by `ILibrary.add()` (via +`rename_theirs=_rename_patterns()` ) +""" +# TODO what are the consequences of making '_' special? maybe we can make this decision everywhere? + + +def _rename_patterns(lib: ILibraryView, name: str) -> str: + """ + The default `rename_theirs` function for `ILibrary.add`. + + Treats names starting with `SINGLE_USE_PREFIX` (default: one underscore) as + "one-offs" for which name conflicts should be automatically resolved. + Conflicts are resolved by calling `lib.get_name(SINGLE_USE_PREFIX + stem)` + where `stem = name.removeprefix(SINGLE_USE_PREFIX).split('$')[0]`. + Names lacking the prefix are directly returned (not renamed). + + Args: + lib: The library into which `name` is to be added (but is presumed to conflict) + name: The original name, to be modified + + Returns: + The new name, not guaranteed to be conflict-free! + """ + if not name.startswith(SINGLE_USE_PREFIX): + return name + + stem = name.removeprefix(SINGLE_USE_PREFIX).split('$')[0] + return lib.get_name(SINGLE_USE_PREFIX + stem) + + +def _plan_source_names( + target: ILibraryView, + source_order: Sequence[str], + existing_names: Container[str], + *, + rename_theirs: Callable[[ILibraryView, str], str] | None = None, + rename_when: Literal['conflict', 'always'] = 'conflict', + ) -> dict[str, str]: + if rename_when not in ('conflict', 'always'): + raise ValueError(f'Unknown source rename mode: {rename_when!r}') + if rename_when == 'always' and rename_theirs is None: + raise TypeError('rename_theirs is required when rename_when="always"') + + source_to_visible: dict[str, str] = {} + visible_names: set[str] = set() + + for name in source_order: + visible = name + if rename_when == 'always': + assert rename_theirs is not None + visible = rename_theirs(target, name) + elif visible in existing_names or visible in visible_names: + if rename_theirs is None: + raise LibraryError(f'Conflicting name while adding source: {name!r}') + visible = rename_theirs(target, name) + if visible in existing_names or visible in visible_names: + raise LibraryError(f'Unresolved duplicate key encountered while adding source: {name!r} -> {visible!r}') + source_to_visible[name] = visible + visible_names.add(visible) + + return source_to_visible + + +def _source_rename_map(source_to_visible: Mapping[str, str]) -> dict[str, str]: + return { + source_name: visible_name + for source_name, visible_name in source_to_visible.items() + if source_name != visible_name + } + +def b64suffix(ii: int) -> str: + """ + Turn an integer into a base64-equivalent suffix. + + This could be done with base64.b64encode, but this way is faster for many small `ii`. + """ + def i2a(nn: int) -> str: + return 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$?'[nn] + + parts = ['$', i2a(ii % 64)] + ii >>= 6 + while ii: + parts.append(i2a(ii % 64)) + ii >>= 6 + return ''.join(parts) From 088c37e9d26d7c044912f2c80fb45a795ac6e5b8 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 13:40:58 -0700 Subject: [PATCH 09/10] [file.gdsii] rework gdsii module hierarchy --- examples/generate_gds_perf.py | 2 +- examples/profile_gdsii_readers.py | 2 +- examples/tutorial/library.py | 2 +- masque/file/gdsii/__init__.py | 10 ++++++ .../file/{gdsii_arrow.py => gdsii/arrow.py} | 16 +++++----- masque/file/{gdsii.py => gdsii/klamath.py} | 12 +++---- masque/file/{gdsii_lazy.py => gdsii/lazy.py} | 16 +++++----- .../lazy_arrow.py} | 32 +++++++++---------- .../lazy_write.py} | 12 +++---- masque/file/{gdsii_perf.py => gdsii/perf.py} | 8 ++--- masque/test/test_gdsii_arrow.py | 7 ++-- masque/test/test_gdsii_lazy.py | 3 +- masque/test/test_gdsii_lazy_arrow.py | 7 ++-- masque/test/test_gdsii_perf.py | 2 +- 14 files changed, 72 insertions(+), 59 deletions(-) create mode 100644 masque/file/gdsii/__init__.py rename masque/file/{gdsii_arrow.py => gdsii/arrow.py} (98%) rename masque/file/{gdsii.py => gdsii/klamath.py} (98%) rename masque/file/{gdsii_lazy.py => gdsii/lazy.py} (97%) rename masque/file/{gdsii_lazy_arrow.py => gdsii/lazy_arrow.py} (94%) rename masque/file/{gdsii_lazy_core.py => gdsii/lazy_write.py} (96%) rename masque/file/{gdsii_perf.py => gdsii/perf.py} (98%) diff --git a/examples/generate_gds_perf.py b/examples/generate_gds_perf.py index 6bdd999..7eaebbb 100644 --- a/examples/generate_gds_perf.py +++ b/examples/generate_gds_perf.py @@ -1,4 +1,4 @@ -from masque.file.gdsii_perf import main +from masque.file.gdsii.perf import main if __name__ == '__main__': diff --git a/examples/profile_gdsii_readers.py b/examples/profile_gdsii_readers.py index fb7c99e..0eb05a5 100644 --- a/examples/profile_gdsii_readers.py +++ b/examples/profile_gdsii_readers.py @@ -12,7 +12,7 @@ from masque import LibraryError READERS: dict[str, tuple[str, tuple[str, ...]]] = { 'gdsii': ('masque.file.gdsii', ('readfile',)), - 'gdsii_arrow': ('masque.file.gdsii_arrow', ('readfile', 'arrow_import', 'arrow_convert')), + 'gdsii_arrow': ('masque.file.gdsii.arrow', ('readfile', 'arrow_import', 'arrow_convert')), } diff --git a/examples/tutorial/library.py b/examples/tutorial/library.py index d0ca60e..d7a2615 100644 --- a/examples/tutorial/library.py +++ b/examples/tutorial/library.py @@ -13,7 +13,7 @@ from pprint import pformat from masque import BuildLibrary, Pather, Pattern, cell from masque.file.gdsii import writefile -from masque.file.gdsii_lazy import readfile +from masque.file.gdsii.lazy import readfile import basic_shapes import devices diff --git a/masque/file/gdsii/__init__.py b/masque/file/gdsii/__init__.py new file mode 100644 index 0000000..09a0b03 --- /dev/null +++ b/masque/file/gdsii/__init__.py @@ -0,0 +1,10 @@ +""" +GDSII file format readers and writers. +""" +from .klamath import check_valid_names as check_valid_names +from .klamath import read as read +from .klamath import read_elements as read_elements +from .klamath import readfile as readfile +from .klamath import rint_cast as rint_cast +from .klamath import write as write +from .klamath import writefile as writefile diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii/arrow.py similarity index 98% rename from masque/file/gdsii_arrow.py rename to masque/file/gdsii/arrow.py index 95229ea..2fcc7c9 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii/arrow.py @@ -43,12 +43,12 @@ from numpy.typing import ArrayLike, NDArray import pyarrow from pyarrow.cffi import ffi -from .utils import is_gzipped, tmpfile -from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, PolyCollection, RectCollection -from ..repetition import Grid -from ..utils import layer_t, annotations_t -from ..library import LazyLibrary, Library, ILibrary, ILibraryView +from ..utils import is_gzipped, tmpfile +from ... import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ...shapes import Polygon, Path, PolyCollection, RectCollection +from ...repetition import Grid +from ...utils import layer_t, annotations_t +from ...library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) @@ -136,7 +136,7 @@ def _installed_library_candidates() -> list[pathlib.Path]: def _repo_library_candidates() -> list[pathlib.Path]: - repo_root = pathlib.Path(__file__).resolve().parents[2] + repo_root = pathlib.Path(__file__).resolve().parents[3] library_name = _local_library_filename() return [ repo_root / 'klamath-rs' / 'target' / 'release' / library_name, @@ -167,7 +167,7 @@ def is_available() -> bool: def _get_clib() -> Any: - global clib + global clib # noqa: PLW0603 if clib is None: lib_path = find_klamath_rs_library() if lib_path is None: diff --git a/masque/file/gdsii.py b/masque/file/gdsii/klamath.py similarity index 98% rename from masque/file/gdsii.py rename to masque/file/gdsii/klamath.py index 21c6f94..13d9e81 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii/klamath.py @@ -33,12 +33,12 @@ from numpy.typing import ArrayLike, NDArray import klamath from klamath import records -from .utils import is_gzipped, tmpfile -from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape -from ..shapes import Polygon, Path, RectCollection -from ..repetition import Grid -from ..utils import layer_t, annotations_t -from ..library import Library, ILibrary +from ..utils import is_gzipped, tmpfile +from ... import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ...shapes import Polygon, Path, RectCollection +from ...repetition import Grid +from ...utils import layer_t, annotations_t +from ...library import Library, ILibrary logger = logging.getLogger(__name__) diff --git a/masque/file/gdsii_lazy.py b/masque/file/gdsii/lazy.py similarity index 97% rename from masque/file/gdsii_lazy.py rename to masque/file/gdsii/lazy.py index a0b6e6f..7ddc76d 100644 --- a/masque/file/gdsii_lazy.py +++ b/masque/file/gdsii/lazy.py @@ -10,7 +10,7 @@ This module provides the non-Arrow half of Masque's lazy GDS architecture: - the source can be wrapped in `PortsLibraryView` or merged through `OverlayLibrary` -The public surface intentionally parallels `gdsii_lazy_arrow` closely so that +The public surface intentionally parallels `gdsii.lazy_arrow` closely so that callers can swap between the classic and Arrow-backed implementations with minimal changes. """ @@ -29,24 +29,24 @@ import klamath import numpy from klamath import records -from . import gdsii -from .utils import is_gzipped -from .gdsii_lazy_core import write as write, writefile as writefile -from ..error import LibraryError -from ..library import ( +from . import klamath as gdsii +from .lazy_write import write as write, writefile as writefile +from ..utils import is_gzipped +from ...error import LibraryError +from ...library import ( ILibraryView, LibraryView, PortsLibraryView, dangling_mode_t, ) -from ..utils import apply_transforms +from ...utils import apply_transforms if TYPE_CHECKING: from collections.abc import Iterator, Sequence from numpy.typing import NDArray - from ..pattern import Pattern + from ...pattern import Pattern logger = logging.getLogger(__name__) diff --git a/masque/file/gdsii_lazy_arrow.py b/masque/file/gdsii/lazy_arrow.py similarity index 94% rename from masque/file/gdsii_lazy_arrow.py rename to masque/file/gdsii/lazy_arrow.py index a53c1ff..6d14f0a 100644 --- a/masque/file/gdsii_lazy_arrow.py +++ b/masque/file/gdsii/lazy_arrow.py @@ -1,7 +1,7 @@ """ Lazy GDSII readers and writers backed by native Arrow scan/materialize paths. -This module is intentionally separate from `gdsii_arrow` so the eager read path +This module is intentionally separate from `gdsii.arrow` so the eager read path keeps its current behavior and performance profile. """ from __future__ import annotations @@ -16,16 +16,16 @@ import pathlib import numpy -from . import gdsii_arrow -from .utils import is_gzipped -from .gdsii_lazy_core import write as write, writefile as writefile -from ..library import ( +from . import arrow +from .lazy_write import write as write, writefile as writefile +from ..utils import is_gzipped +from ...library import ( ILibraryView, LibraryView, PortsLibraryView, dangling_mode_t, ) -from ..utils import apply_transforms +from ...utils import apply_transforms if TYPE_CHECKING: from collections.abc import Iterator, Sequence @@ -33,7 +33,7 @@ if TYPE_CHECKING: from numpy.typing import NDArray import pyarrow - from ..pattern import Pattern + from ...pattern import Pattern logger = logging.getLogger(__name__) @@ -87,11 +87,11 @@ class _ScanPayload: refs: _ScanRefs def is_available() -> bool: - return gdsii_arrow.is_available() + return arrow.is_available() def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: - return gdsii_arrow._read_header(libarr) + return arrow._read_header(libarr) def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: @@ -119,10 +119,10 @@ def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: ref_values = refs.values ref_offsets = refs.offsets.to_numpy() targets = ref_values.field('target').to_numpy() - xy = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) - xy0 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) - xy1 = gdsii_arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) - counts = gdsii_arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) + xy = arrow._packed_xy_u64_to_pairs(ref_values.field('xy').to_numpy()) + xy0 = arrow._packed_xy_u64_to_pairs(ref_values.field('xy0').to_numpy()) + xy1 = arrow._packed_xy_u64_to_pairs(ref_values.field('xy1').to_numpy()) + counts = arrow._packed_counts_u32_to_pairs(ref_values.field('counts').to_numpy()) invert_y = ref_values.field('invert_y').to_numpy(zero_copy_only=False) angle_rad = ref_values.field('angle_rad').to_numpy() scale = ref_values.field('scale').to_numpy() @@ -228,7 +228,7 @@ class ArrowLibrary(ILibraryView): def from_file(cls, filename: str | pathlib.Path) -> ArrowLibrary: path = pathlib.Path(filename).expanduser().resolve() source = _open_source_buffer(path) - scan_arr = gdsii_arrow._scan_buffer_to_arrow(source.data) + scan_arr = arrow._scan_buffer_to_arrow(source.data) assert len(scan_arr) == 1 payload = _extract_scan_payload(scan_arr[0]) return cls(path=path, payload=payload, source=source) @@ -288,9 +288,9 @@ class ArrowLibrary(ILibraryView): ], dtype=numpy.uint64, ) - arrow_arr = gdsii_arrow._read_selected_cells_to_arrow(self._source.data, ranges) + arrow_arr = arrow._read_selected_cells_to_arrow(self._source.data, ranges) assert len(arrow_arr) == 1 - selected_lib, _info = gdsii_arrow.read_arrow(arrow_arr[0]) + selected_lib, _info = arrow.read_arrow(arrow_arr[0]) for name in uncached: pat = selected_lib[name] materialized[name] = pat diff --git a/masque/file/gdsii_lazy_core.py b/masque/file/gdsii/lazy_write.py similarity index 96% rename from masque/file/gdsii_lazy_core.py rename to masque/file/gdsii/lazy_write.py index 988f9a4..8cb167a 100644 --- a/masque/file/gdsii_lazy_core.py +++ b/masque/file/gdsii/lazy_write.py @@ -15,16 +15,16 @@ import pathlib import klamath -from . import gdsii -from .utils import tmpfile -from ..error import LibraryError -from ..library import ILibraryView, OverlayLibrary -from ..library.overlay import _SourceEntry, _materialize_detached_pattern +from . import klamath as gdsii +from ..utils import tmpfile +from ...error import LibraryError +from ...library import ILibraryView, OverlayLibrary +from ...library.overlay import _SourceEntry, _materialize_detached_pattern if TYPE_CHECKING: from collections.abc import Iterator, Mapping - from ..pattern import Pattern + from ...pattern import Pattern logger = logging.getLogger(__name__) diff --git a/masque/file/gdsii_perf.py b/masque/file/gdsii/perf.py similarity index 98% rename from masque/file/gdsii_perf.py rename to masque/file/gdsii/perf.py index 38d1a7d..339d1e0 100644 --- a/masque/file/gdsii_perf.py +++ b/masque/file/gdsii/perf.py @@ -244,7 +244,7 @@ def _annotation(index: int) -> dict[int, bytes]: return {1: f'perf-{index}'.encode('ASCII')} -def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: +def _make_box_cell(index: int, cfg: FixturePreset) -> list[elements.Element]: cell_elements: list[elements.Element] = [] xbase = (index % 17) * 600 ybase = (index // 17) * 180 @@ -277,7 +277,7 @@ def _make_box_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.E return cell_elements -def _make_poly_cell(name: str, index: int, cfg: FixturePreset) -> list[elements.Element]: +def _make_poly_cell(index: int, cfg: FixturePreset) -> list[elements.Element]: cell_elements: list[elements.Element] = [] xbase = (index % 19) * 900 ybase = (index // 19) * 260 @@ -368,12 +368,12 @@ def _poly_cluster_name(index: int) -> str: def _write_box_cells(stream: Any, cfg: FixturePreset) -> None: for idx in range(cfg.box_cells): - _write_struct(stream, _box_name(idx), _make_box_cell(_box_name(idx), idx, cfg)) + _write_struct(stream, _box_name(idx), _make_box_cell(idx, cfg)) def _write_poly_cells(stream: Any, cfg: FixturePreset) -> None: for idx in range(cfg.poly_cells): - _write_struct(stream, _poly_name(idx), _make_poly_cell(_poly_name(idx), idx, cfg)) + _write_struct(stream, _poly_name(idx), _make_poly_cell(idx, cfg)) def _write_wrappers(stream: Any, cfg: FixturePreset) -> None: diff --git a/masque/test/test_gdsii_arrow.py b/masque/test/test_gdsii_arrow.py index b2962f8..47f9ac8 100644 --- a/masque/test/test_gdsii_arrow.py +++ b/masque/test/test_gdsii_arrow.py @@ -14,8 +14,9 @@ from ..library import Library from ..pattern import Pattern from ..repetition import Grid from ..shapes import Path as MPath, Polygon, PolyCollection, RectCollection -from ..file import gdsii, gdsii_arrow -from ..file.gdsii_perf import write_fixture +from ..file import gdsii +from ..file.gdsii import arrow as gdsii_arrow +from ..file.gdsii.perf import write_fixture if not gdsii_arrow.is_available(): @@ -267,7 +268,7 @@ def test_gdsii_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> None: gds_file.write_bytes(b'not-a-gds') script = textwrap.dedent(f""" - from masque.file import gdsii_arrow + from masque.file.gdsii import arrow as gdsii_arrow try: gdsii_arrow.readfile({str(gds_file)!r}) except Exception as exc: diff --git a/masque/test/test_gdsii_lazy.py b/masque/test/test_gdsii_lazy.py index 3abaab8..026580f 100644 --- a/masque/test/test_gdsii_lazy.py +++ b/masque/test/test_gdsii_lazy.py @@ -4,7 +4,8 @@ import numpy import pytest from numpy.testing import assert_allclose -from ..file import gdsii, gdsii_lazy +from ..file import gdsii +from ..file.gdsii import lazy as gdsii_lazy from ..pattern import Pattern from ..library import Library, OverlayLibrary diff --git a/masque/test/test_gdsii_lazy_arrow.py b/masque/test/test_gdsii_lazy_arrow.py index 0984b20..0447921 100644 --- a/masque/test/test_gdsii_lazy_arrow.py +++ b/masque/test/test_gdsii_lazy_arrow.py @@ -13,8 +13,9 @@ from .. import PatternError from ..library import Library, OverlayLibrary from ..pattern import Pattern from ..repetition import Grid -from ..file import gdsii, gdsii_lazy_arrow -from ..file.gdsii_perf import write_fixture +from ..file import gdsii +from ..file.gdsii import lazy_arrow as gdsii_lazy_arrow +from ..file.gdsii.perf import write_fixture if not gdsii_lazy_arrow.is_available(): @@ -177,7 +178,7 @@ def test_gdsii_lazy_arrow_invalid_input_raises_klamath_error(tmp_path: Path) -> gds_file.write_bytes(b'not-a-gds') script = textwrap.dedent(f""" - from masque.file import gdsii_lazy_arrow + from masque.file.gdsii import lazy_arrow as gdsii_lazy_arrow try: gdsii_lazy_arrow.readfile({str(gds_file)!r}) except Exception as exc: diff --git a/masque/test/test_gdsii_perf.py b/masque/test/test_gdsii_perf.py index a595fe8..09b3a5a 100644 --- a/masque/test/test_gdsii_perf.py +++ b/masque/test/test_gdsii_perf.py @@ -3,7 +3,7 @@ import json from pathlib import Path from ..file import gdsii -from ..file.gdsii_perf import fixture_manifest, write_fixture +from ..file.gdsii.perf import fixture_manifest, write_fixture def test_gdsii_perf_fixture_smoke(tmp_path: Path) -> None: From 520f37aa292bee9087c6d63baba80526fda5017a Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Thu, 9 Jul 2026 14:21:05 -0700 Subject: [PATCH 10/10] [gdsii] misc cleanup --- masque/file/gdsii/arrow.py | 6 +---- masque/file/gdsii/lazy_arrow.py | 6 +---- masque/file/gdsii/lazy_write.py | 41 +++++++++++++++------------------ masque/file/gdsii/perf.py | 41 ++++++++++++++------------------- 4 files changed, 38 insertions(+), 56 deletions(-) diff --git a/masque/file/gdsii/arrow.py b/masque/file/gdsii/arrow.py index 2fcc7c9..c0d9c61 100644 --- a/masque/file/gdsii/arrow.py +++ b/masque/file/gdsii/arrow.py @@ -39,7 +39,7 @@ from pprint import pformat from klamath.basic import KlamathError import numpy -from numpy.typing import ArrayLike, NDArray +from numpy.typing import NDArray import pyarrow from pyarrow.cffi import ffi @@ -80,10 +80,6 @@ path_cap_map = { } -def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: - return numpy.rint(val).astype(numpy.int32) - - def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]: layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16) dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16) diff --git a/masque/file/gdsii/lazy_arrow.py b/masque/file/gdsii/lazy_arrow.py index 6d14f0a..5a0da0f 100644 --- a/masque/file/gdsii/lazy_arrow.py +++ b/masque/file/gdsii/lazy_arrow.py @@ -90,10 +90,6 @@ def is_available() -> bool: return arrow.is_available() -def _read_header(libarr: pyarrow.StructScalar) -> dict[str, Any]: - return arrow._read_header(libarr) - - def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: if is_gzipped(path): with gzip.open(path, mode='rb') as stream: @@ -106,7 +102,7 @@ def _open_source_buffer(path: pathlib.Path) -> _SourceBuffer: def _extract_scan_payload(libarr: pyarrow.StructScalar) -> _ScanPayload: - library_info = _read_header(libarr) + library_info = arrow._read_header(libarr) cell_names = libarr['cell_names'].as_py() cells = libarr['cells'] diff --git a/masque/file/gdsii/lazy_write.py b/masque/file/gdsii/lazy_write.py index 8cb167a..225787a 100644 --- a/masque/file/gdsii/lazy_write.py +++ b/masque/file/gdsii/lazy_write.py @@ -22,7 +22,7 @@ from ...library import ILibraryView, OverlayLibrary from ...library.overlay import _SourceEntry, _materialize_detached_pattern if TYPE_CHECKING: - from collections.abc import Iterator, Mapping + from collections.abc import Mapping from ...pattern import Pattern @@ -30,15 +30,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _iter_library_infos(library: Mapping[str, Pattern] | ILibraryView) -> Iterator[dict[str, Any]]: - info = getattr(library, 'library_info', None) - if isinstance(info, dict): - yield info - if isinstance(library, OverlayLibrary): - for layer in library._layers: - yield from _iter_library_infos(layer.library) - - def _get_write_info( library: Mapping[str, Pattern] | ILibraryView, *, @@ -49,7 +40,16 @@ def _get_write_info( if meters_per_unit is not None and logical_units_per_unit is not None and library_name is not None: return meters_per_unit, logical_units_per_unit, library_name - infos = list(_iter_library_infos(library)) + infos: list[dict[str, Any]] = [] + stack: list[Mapping[str, Pattern] | ILibraryView] = [library] + while stack: + current = stack.pop() + info = getattr(current, 'library_info', None) + if isinstance(info, dict): + infos.append(info) + if isinstance(current, OverlayLibrary): + stack.extend(reversed([layer.library for layer in current._layers])) + if infos: unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos} if len(unit_pairs) > 1: @@ -79,16 +79,6 @@ def _raw_struct_bytes(library: Mapping[str, Pattern] | ILibraryView, name: str) return cast('bytes', reader(name)) -def _can_copy_overlay_cell(library: OverlayLibrary, name: str, entry: _SourceEntry) -> bool: - layer = library._layers[entry.layer_index] - if name != entry.source_name: - return False - if not _can_copy_raw_cell(layer.library, entry.source_name): - return False - children = layer.child_graph.get(entry.source_name, set()) - return all(library._effective_target(layer, child) == child for child in children) - - def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None: elements: list[klamath.elements.Element] = [] elements += gdsii._shapes_to_elements(pat.shapes) @@ -122,8 +112,15 @@ def write( if isinstance(library, OverlayLibrary): for name in library.source_order(): entry = library._entries[name] - if isinstance(entry, _SourceEntry) and _can_copy_overlay_cell(library, name, entry): + can_copy_overlay = False + if isinstance(entry, _SourceEntry) and name == entry.source_name: layer = library._layers[entry.layer_index] + children = layer.child_graph.get(entry.source_name, set()) + can_copy_overlay = ( + _can_copy_raw_cell(layer.library, entry.source_name) + and all(library._effective_target(layer, child) == child for child in children) + ) + if can_copy_overlay: stream.write(_raw_struct_bytes(layer.library, entry.source_name)) else: _write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False)) diff --git a/masque/file/gdsii/perf.py b/masque/file/gdsii/perf.py index 339d1e0..64bd7ab 100644 --- a/masque/file/gdsii/perf.py +++ b/masque/file/gdsii/perf.py @@ -487,38 +487,31 @@ def _write_top(stream: Any, cfg: FixturePreset) -> None: _write_struct(stream, 'TOP', cell_elements) -def _poly_paths_total(cfg: FixturePreset) -> int: - return (cfg.poly_cells - 1) // cfg.path_stride + 1 - - -def _poly_texts_total(cfg: FixturePreset) -> int: - return (cfg.poly_cells - 1) // cfg.text_stride + 1 - - -def _ref_instances_per_box_cluster(cfg: FixturePreset) -> int: - array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) - array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] - return array_refs * array_mult + (cfg.box_cluster_refs - array_refs) - - -def _ref_instances_per_poly_cluster(cfg: FixturePreset) -> int: - array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) - array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] - return array_refs * array_mult + (cfg.poly_cluster_refs - array_refs) - - def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> FixtureManifest: base = PRESETS[preset] cfg = _scaled_preset(base, scale) + box_cluster_array_refs = min(cfg.box_cluster_refs, max(1, (3 * cfg.box_cluster_refs) // 4)) + box_cluster_array_mult = cfg.box_cluster_array[0] * cfg.box_cluster_array[1] + box_cluster_ref_instances = ( + box_cluster_array_refs * box_cluster_array_mult + + (cfg.box_cluster_refs - box_cluster_array_refs) + ) + poly_cluster_array_refs = min(cfg.poly_cluster_refs, cfg.poly_cluster_refs // 2) + poly_cluster_array_mult = cfg.poly_cluster_array[0] * cfg.poly_cluster_array[1] + poly_cluster_ref_instances = ( + poly_cluster_array_refs * poly_cluster_array_mult + + (cfg.poly_cluster_refs - poly_cluster_array_refs) + ) + flattened_box_placements = ( cfg.box_wrappers - + cfg.box_clusters * _ref_instances_per_box_cluster(cfg) + + cfg.box_clusters * box_cluster_ref_instances + cfg.top_direct_box_refs * cfg.top_box_array[0] * cfg.top_box_array[1] ) flattened_poly_placements = ( cfg.poly_wrappers - + cfg.poly_clusters * _ref_instances_per_poly_cluster(cfg) + + cfg.poly_clusters * poly_cluster_ref_instances + cfg.top_direct_poly_refs * cfg.top_poly_array[0] * cfg.top_poly_array[1] ) polygon_layers = max(1, cfg.polygon_layers) @@ -545,8 +538,8 @@ def fixture_manifest(path: str | Path, preset: str, scale: float = 1.0) -> Fixtu hierarchical_boxes_per_heavy_layer=cfg.box_cells * cfg.heavy_boxes_per_cell, hierarchical_boxes_per_regular_layer=cfg.box_cells * cfg.regular_boxes_per_cell, hierarchical_polygons_total=cfg.poly_cells * cfg.polygons_per_cell, - hierarchical_paths_total=_poly_paths_total(cfg), - hierarchical_texts_total=_poly_texts_total(cfg), + hierarchical_paths_total=(cfg.poly_cells - 1) // cfg.path_stride + 1, + hierarchical_texts_total=(cfg.poly_cells - 1) // cfg.text_stride + 1, flattened_box_placements=flattened_box_placements, flattened_poly_placements=flattened_poly_placements, estimated_flat_boxes_per_heavy_layer=flattened_box_placements * cfg.heavy_boxes_per_cell,