[library] More library rework
This commit is contained in:
parent
67afee7704
commit
0edd735a11
25 changed files with 2357 additions and 870 deletions
|
|
@ -26,7 +26,6 @@ import mmap
|
|||
import pathlib
|
||||
|
||||
import klamath
|
||||
import numpy
|
||||
from klamath import records
|
||||
|
||||
from . import klamath as gdsii
|
||||
|
|
@ -35,15 +34,16 @@ from ..utils import is_gzipped
|
|||
from ...error import LibraryError
|
||||
from ...library import (
|
||||
ILibraryView,
|
||||
LibraryView,
|
||||
IMaterializable,
|
||||
PortsLibraryView,
|
||||
dangling_mode_t,
|
||||
)
|
||||
from ...utils import apply_transforms
|
||||
from ...library.utils import _validate_dangling_mode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
|
||||
import numpy
|
||||
from numpy.typing import NDArray
|
||||
|
||||
from ...pattern import Pattern
|
||||
|
|
@ -122,7 +122,7 @@ def _scan_library(
|
|||
return library_info, order, cells
|
||||
|
||||
|
||||
class GdsLibrarySource(ILibraryView):
|
||||
class GdsLibrarySource(ILibraryView, IMaterializable):
|
||||
"""
|
||||
Read-only library backed by a seekable GDS stream.
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ class GdsLibrarySource(ILibraryView):
|
|||
return cls(source=source, library_info=library_info, cell_order=cell_order, cells=cells)
|
||||
|
||||
def __getitem__(self, key: str) -> Pattern:
|
||||
return self._materialize_pattern(key, persist=True)
|
||||
return self.materialize(key, persist=True)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._cell_order)
|
||||
|
|
@ -177,19 +177,7 @@ class GdsLibrarySource(ILibraryView):
|
|||
def source_order(self) -> tuple[str, ...]:
|
||||
return self._cell_order
|
||||
|
||||
def materialize_many(
|
||||
self,
|
||||
names: Sequence[str],
|
||||
*,
|
||||
persist: bool = True,
|
||||
) -> LibraryView:
|
||||
mats = {
|
||||
name: self._materialize_pattern(name, persist=persist)
|
||||
for name in dict.fromkeys(names)
|
||||
}
|
||||
return LibraryView(mats)
|
||||
|
||||
def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern:
|
||||
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
|
||||
if name in self._cache:
|
||||
return self._cache[name]
|
||||
|
||||
|
|
@ -223,6 +211,7 @@ class GdsLibrarySource(ILibraryView):
|
|||
self,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, set[str]]:
|
||||
_validate_dangling_mode(dangling)
|
||||
graph: dict[str, set[str]] = {}
|
||||
for name in self._cell_order:
|
||||
if name in self._cache:
|
||||
|
|
@ -243,41 +232,11 @@ class GdsLibrarySource(ILibraryView):
|
|||
graph.setdefault(cast('str', child), set())
|
||||
return graph
|
||||
|
||||
def parent_graph(
|
||||
self,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, set[str]]:
|
||||
child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore')
|
||||
existing = set(self.keys())
|
||||
igraph: dict[str, set[str]] = {name: set() for name in child_graph}
|
||||
for parent, children in child_graph.items():
|
||||
for child in children:
|
||||
if child in existing or dangling == 'include':
|
||||
igraph.setdefault(child, set()).add(parent)
|
||||
if dangling == 'error':
|
||||
raw = self.child_graph(dangling='include')
|
||||
dangling_refs = set().union(*(children - existing for children in raw.values()))
|
||||
if dangling_refs:
|
||||
raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph')
|
||||
return igraph
|
||||
|
||||
def subtree(
|
||||
self,
|
||||
tops: str | Sequence[str],
|
||||
) -> ILibraryView:
|
||||
if isinstance(tops, str):
|
||||
tops = (tops,)
|
||||
keep = cast('set[str]', self.referenced_patterns(tops) - {None})
|
||||
keep |= set(tops)
|
||||
return self.materialize_many(tuple(keep), persist=True)
|
||||
|
||||
def tops(self) -> list[str]:
|
||||
graph = self.child_graph(dangling='ignore')
|
||||
names = set(graph)
|
||||
not_toplevel: set[str] = set()
|
||||
for children in graph.values():
|
||||
not_toplevel |= children
|
||||
return list(names - not_toplevel)
|
||||
return super().subtree(tops)
|
||||
|
||||
def with_ports_from_data(
|
||||
self,
|
||||
|
|
@ -315,6 +274,7 @@ class GdsLibrarySource(ILibraryView):
|
|||
parent_graph: dict[str, set[str]] | None = None,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, list[NDArray[numpy.float64]]]:
|
||||
_validate_dangling_mode(dangling)
|
||||
instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list)
|
||||
if parent_graph is None:
|
||||
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
|
||||
|
|
@ -333,58 +293,11 @@ class GdsLibrarySource(ILibraryView):
|
|||
for ref in self._cache[parent].refs.get(name, []):
|
||||
instances[parent].append(ref.as_transforms())
|
||||
continue
|
||||
pat = self._materialize_pattern(parent, persist=False)
|
||||
pat = self.materialize(parent, persist=False)
|
||||
for ref in pat.refs.get(name, []):
|
||||
instances[parent].append(ref.as_transforms())
|
||||
return instances
|
||||
|
||||
def find_refs_global(
|
||||
self,
|
||||
name: str,
|
||||
order: list[str] | None = None,
|
||||
parent_graph: dict[str, set[str]] | None = None,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[tuple[str, ...], NDArray[numpy.float64]]:
|
||||
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
|
||||
if order is None:
|
||||
order = self.child_order(dangling=graph_mode)
|
||||
if parent_graph is None:
|
||||
parent_graph = self.parent_graph(dangling=graph_mode)
|
||||
|
||||
if name not in self:
|
||||
if name not in parent_graph:
|
||||
return {}
|
||||
if dangling == 'error':
|
||||
raise self._dangling_refs_error({name}, f'finding global refs for {name!r}')
|
||||
if dangling == 'ignore':
|
||||
return {}
|
||||
|
||||
self_keys = set(self.keys())
|
||||
transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]]
|
||||
transforms = defaultdict(list)
|
||||
for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items():
|
||||
transforms[parent] = [((name,), numpy.concatenate(vals))]
|
||||
|
||||
for next_name in order:
|
||||
if next_name not in transforms:
|
||||
continue
|
||||
if not parent_graph.get(next_name, set()) & self_keys:
|
||||
continue
|
||||
|
||||
outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling)
|
||||
inners = transforms.pop(next_name)
|
||||
for parent, outer in outers.items():
|
||||
outer_tf = numpy.concatenate(outer)
|
||||
for path, inner in inners:
|
||||
combined = apply_transforms(outer_tf, inner)
|
||||
transforms[parent].append(((next_name,) + path, combined))
|
||||
|
||||
result = {}
|
||||
for parent, targets in transforms.items():
|
||||
for path, instances in targets:
|
||||
result[(parent,) + path] = instances
|
||||
return result
|
||||
|
||||
def close(self) -> None:
|
||||
self._source.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -21,11 +21,12 @@ from .lazy_write import write as write, writefile as writefile
|
|||
from ..utils import is_gzipped
|
||||
from ...library import (
|
||||
ILibraryView,
|
||||
IMaterializable,
|
||||
LibraryView,
|
||||
PortsLibraryView,
|
||||
dangling_mode_t,
|
||||
)
|
||||
from ...utils import apply_transforms
|
||||
from ...library.utils import _validate_dangling_mode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
|
|
@ -197,7 +198,7 @@ def _expand_aref_row(
|
|||
return rows
|
||||
|
||||
|
||||
class ArrowLibrary(ILibraryView):
|
||||
class ArrowLibrary(ILibraryView, IMaterializable):
|
||||
"""
|
||||
Read-only library backed by the native lazy Arrow scan schema.
|
||||
|
||||
|
|
@ -231,7 +232,7 @@ class ArrowLibrary(ILibraryView):
|
|||
return cls(path=path, payload=payload, source=source)
|
||||
|
||||
def __getitem__(self, key: str) -> Pattern:
|
||||
return self._materialize_pattern(key, persist=True)
|
||||
return self.materialize(key, persist=True)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._payload.cell_order)
|
||||
|
|
@ -299,7 +300,7 @@ class ArrowLibrary(ILibraryView):
|
|||
materialized[name] = self._cache[name]
|
||||
return materialized
|
||||
|
||||
def _materialize_pattern(self, name: str, *, persist: bool) -> Pattern:
|
||||
def materialize(self, name: str, *, persist: bool = True) -> Pattern:
|
||||
return self._materialize_patterns((name,), persist=persist)[name]
|
||||
|
||||
def _raw_children(self, name: str) -> set[str]:
|
||||
|
|
@ -346,6 +347,7 @@ class ArrowLibrary(ILibraryView):
|
|||
self,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, set[str]]:
|
||||
_validate_dangling_mode(dangling)
|
||||
graph: dict[str, set[str]] = {}
|
||||
for name in self._payload.cell_order:
|
||||
if name in self._cache:
|
||||
|
|
@ -366,41 +368,11 @@ class ArrowLibrary(ILibraryView):
|
|||
graph.setdefault(cast('str', child), set())
|
||||
return graph
|
||||
|
||||
def parent_graph(
|
||||
self,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, set[str]]:
|
||||
child_graph = self.child_graph(dangling='include' if dangling == 'include' else 'ignore')
|
||||
existing = set(self.keys())
|
||||
igraph: dict[str, set[str]] = {name: set() for name in child_graph}
|
||||
for parent, children in child_graph.items():
|
||||
for child in children:
|
||||
if child in existing or dangling == 'include':
|
||||
igraph.setdefault(child, set()).add(parent)
|
||||
if dangling == 'error':
|
||||
raw = self.child_graph(dangling='include')
|
||||
dangling_refs = set().union(*(children - existing for children in raw.values()))
|
||||
if dangling_refs:
|
||||
raise self._dangling_refs_error(cast('set[str]', dangling_refs), 'building parent graph')
|
||||
return igraph
|
||||
|
||||
def subtree(
|
||||
self,
|
||||
tops: str | Sequence[str],
|
||||
) -> ILibraryView:
|
||||
if isinstance(tops, str):
|
||||
tops = (tops,)
|
||||
keep = cast('set[str]', self.referenced_patterns(tops) - {None})
|
||||
keep |= set(tops)
|
||||
return self.materialize_many(tuple(keep), persist=True)
|
||||
|
||||
def tops(self) -> list[str]:
|
||||
graph = self.child_graph(dangling='ignore')
|
||||
names = set(graph)
|
||||
not_toplevel: set[str] = set()
|
||||
for children in graph.values():
|
||||
not_toplevel |= children
|
||||
return list(names - not_toplevel)
|
||||
return super().subtree(tops)
|
||||
|
||||
def with_ports_from_data(
|
||||
self,
|
||||
|
|
@ -452,6 +424,7 @@ class ArrowLibrary(ILibraryView):
|
|||
parent_graph: dict[str, set[str]] | None = None,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[str, list[NDArray[numpy.float64]]]:
|
||||
_validate_dangling_mode(dangling)
|
||||
instances: dict[str, list[NDArray[numpy.float64]]] = defaultdict(list)
|
||||
if parent_graph is None:
|
||||
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
|
||||
|
|
@ -479,54 +452,6 @@ class ArrowLibrary(ILibraryView):
|
|||
instances[parent].extend(rows)
|
||||
return instances
|
||||
|
||||
def find_refs_global(
|
||||
self,
|
||||
name: str,
|
||||
order: list[str] | None = None,
|
||||
parent_graph: dict[str, set[str]] | None = None,
|
||||
dangling: dangling_mode_t = 'error',
|
||||
) -> dict[tuple[str, ...], NDArray[numpy.float64]]:
|
||||
graph_mode = 'ignore' if dangling == 'ignore' else 'include'
|
||||
if order is None:
|
||||
order = self.child_order(dangling=graph_mode)
|
||||
if parent_graph is None:
|
||||
parent_graph = self.parent_graph(dangling=graph_mode)
|
||||
|
||||
if name not in self:
|
||||
if name not in parent_graph:
|
||||
return {}
|
||||
if dangling == 'error':
|
||||
raise self._dangling_refs_error({name}, f'finding global refs for {name!r}')
|
||||
if dangling == 'ignore':
|
||||
return {}
|
||||
|
||||
self_keys = set(self.keys())
|
||||
transforms: dict[str, list[tuple[tuple[str, ...], NDArray[numpy.float64]]]]
|
||||
transforms = defaultdict(list)
|
||||
for parent, vals in self.find_refs_local(name, parent_graph=parent_graph, dangling=dangling).items():
|
||||
transforms[parent] = [((name,), numpy.concatenate(vals))]
|
||||
|
||||
for next_name in order:
|
||||
if next_name not in transforms:
|
||||
continue
|
||||
if not parent_graph.get(next_name, set()) & self_keys:
|
||||
continue
|
||||
|
||||
outers = self.find_refs_local(next_name, parent_graph=parent_graph, dangling=dangling)
|
||||
inners = transforms.pop(next_name)
|
||||
for parent, outer in outers.items():
|
||||
outer_tf = numpy.concatenate(outer)
|
||||
for path, inner in inners:
|
||||
combined = apply_transforms(outer_tf, inner)
|
||||
transforms[parent].append(((next_name,) + path, combined))
|
||||
|
||||
result = {}
|
||||
for parent, targets in transforms.items():
|
||||
for path, instances in targets:
|
||||
full_path = (parent,) + path
|
||||
result[full_path] = instances
|
||||
return result
|
||||
|
||||
|
||||
def readfile(
|
||||
filename: str | pathlib.Path,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ or remapped.
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import IO, TYPE_CHECKING, Any, cast
|
||||
from typing import IO, TYPE_CHECKING, Any, Protocol, cast, runtime_checkable
|
||||
import gzip
|
||||
import logging
|
||||
import pathlib
|
||||
|
|
@ -18,8 +18,8 @@ import klamath
|
|||
from . import klamath as gdsii
|
||||
from ..utils import tmpfile
|
||||
from ...error import LibraryError
|
||||
from ...library import ILibraryView, OverlayLibrary
|
||||
from ...library.overlay import _SourceEntry, _materialize_detached_pattern
|
||||
from ...library import IBorrowing, ILibraryView
|
||||
from ...library.overlay import _materialize_detached_pattern
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -30,6 +30,23 @@ if TYPE_CHECKING:
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _GdsInfoSource(Protocol):
|
||||
"""Structural capability for propagating GDS header metadata."""
|
||||
library_info: dict[str, Any]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _GdsRawCellSource(Protocol):
|
||||
"""GDS-specific raw-structure copy-through capability."""
|
||||
|
||||
def source_order(self) -> tuple[str, ...]: ...
|
||||
|
||||
def can_copy_raw_struct(self, name: str) -> bool: ...
|
||||
|
||||
def raw_struct_bytes(self, name: str) -> bytes: ...
|
||||
|
||||
|
||||
def _get_write_info(
|
||||
library: Mapping[str, Pattern] | ILibraryView,
|
||||
*,
|
||||
|
|
@ -42,13 +59,16 @@ def _get_write_info(
|
|||
|
||||
infos: list[dict[str, Any]] = []
|
||||
stack: list[Mapping[str, Pattern] | ILibraryView] = [library]
|
||||
seen: set[int] = set()
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
info = getattr(current, 'library_info', None)
|
||||
if isinstance(info, dict):
|
||||
infos.append(info)
|
||||
if isinstance(current, OverlayLibrary):
|
||||
stack.extend(reversed([layer.library for layer in current._layers]))
|
||||
if id(current) in seen:
|
||||
continue
|
||||
seen.add(id(current))
|
||||
if isinstance(current, _GdsInfoSource) and isinstance(current.library_info, dict):
|
||||
infos.append(current.library_info)
|
||||
if isinstance(current, IBorrowing):
|
||||
stack.extend(reversed(current.borrowed_sources()))
|
||||
|
||||
if infos:
|
||||
unit_pairs = {(info['meters_per_unit'], info['logical_units_per_unit']) for info in infos}
|
||||
|
|
@ -65,20 +85,6 @@ def _get_write_info(
|
|||
return meters_per_unit, logical_units_per_unit, library_name
|
||||
|
||||
|
||||
def _can_copy_raw_cell(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bool:
|
||||
can_copy = getattr(library, 'can_copy_raw_struct', None)
|
||||
if not callable(can_copy):
|
||||
return False
|
||||
return bool(can_copy(name))
|
||||
|
||||
|
||||
def _raw_struct_bytes(library: Mapping[str, Pattern] | ILibraryView, name: str) -> bytes:
|
||||
reader = getattr(library, 'raw_struct_bytes', None)
|
||||
if not callable(reader):
|
||||
raise TypeError('raw_struct_bytes')
|
||||
return cast('bytes', reader(name))
|
||||
|
||||
|
||||
def _write_pattern_struct(stream: IO[bytes], name: str, pat: Pattern) -> None:
|
||||
elements: list[klamath.elements.Element] = []
|
||||
elements += gdsii._shapes_to_elements(pat.shapes)
|
||||
|
|
@ -109,28 +115,10 @@ def write(
|
|||
)
|
||||
header.write(stream)
|
||||
|
||||
if isinstance(library, OverlayLibrary):
|
||||
if isinstance(library, _GdsRawCellSource):
|
||||
for name in library.source_order():
|
||||
entry = library._entries[name]
|
||||
can_copy_overlay = False
|
||||
if isinstance(entry, _SourceEntry) and name == entry.source_name:
|
||||
layer = library._layers[entry.layer_index]
|
||||
children = layer.child_graph.get(entry.source_name, set())
|
||||
can_copy_overlay = (
|
||||
_can_copy_raw_cell(layer.library, entry.source_name)
|
||||
and all(library._effective_target(layer, child) == child for child in children)
|
||||
)
|
||||
if can_copy_overlay:
|
||||
stream.write(_raw_struct_bytes(layer.library, entry.source_name))
|
||||
else:
|
||||
_write_pattern_struct(stream, name, library._materialize_pattern(name, persist=False))
|
||||
klamath.records.ENDLIB.write(stream, None)
|
||||
return
|
||||
|
||||
if hasattr(library, 'raw_struct_bytes'):
|
||||
for name in library.source_order():
|
||||
if _can_copy_raw_cell(library, name):
|
||||
stream.write(_raw_struct_bytes(library, name))
|
||||
if library.can_copy_raw_struct(name):
|
||||
stream.write(library.raw_struct_bytes(name))
|
||||
else:
|
||||
_write_pattern_struct(stream, name, _materialize_detached_pattern(cast('ILibraryView', library), name))
|
||||
klamath.records.ENDLIB.write(stream, None)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue