From 1fdfcbd85daec1e565ff22116342ddaa4b384ec9 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 17 Oct 2024 18:04:25 -0700 Subject: [PATCH 01/11] [wip] add poly_collection shape --- masque/shapes/poly_collection.py | 210 +++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 masque/shapes/poly_collection.py diff --git a/masque/shapes/poly_collection.py b/masque/shapes/poly_collection.py new file mode 100644 index 0000000..d7d1c95 --- /dev/null +++ b/masque/shapes/poly_collection.py @@ -0,0 +1,210 @@ +from typing import Any, cast, Iterable +from collections.abc import Sequence +import copy +import functools + +import numpy +from numpy import pi +from numpy.typing import NDArray, ArrayLike + +from . import Shape, normalized_shape_tuple +from ..error import PatternError +from ..repetition import Repetition +from ..utils import is_scalar, rotation_matrix_2d, annotations_lt, annotations_eq, rep2key +from ..utils import remove_colinear_vertices, remove_duplicate_vertices, annotations_t + + +@functools.total_ordering +class PolyCollection(Shape): + """ + A collection of polygons, consisting of list of vertex arrays (N_m x 2 ndarrays) which specify + implicitly-closed boundaries, and an offset. + + Note that the setter for `PolyCollection.vertex_list` creates a copy of the + passed vertex coordinates. + + A `normalized_form(...)` is available, but can be quite slow with lots of vertices. + """ + __slots__ = ( + '_vertex_lists', + # Inherited + '_offset', '_repetition', '_annotations', + ) + + _vertex_lists: list[NDArray[numpy.float64]] + """ List of ndarrays (N_m x 2) of vertices `[ [[x0, y0], [x1, y1], ...] ]` """ + + # vertex_lists property + @property + def vertex_lists(self) -> Any: # mypy#3004 NDArray[numpy.float64]: + """ + Vertices of the polygons (ist of ndarrays (N_m x 2) `[ [[x0, y0], [x1, y1], ...] ]` + + When setting, note that a copy will be made, + """ + return self._vertex_lists + + @vertex_lists.setter + def vertex_lists(self, val: ArrayLike) -> None: + val = [numpy.array(vv, dtype=float) for vv in val] + for ii, vv in enumerate(val): + if len(vv.shape) < 2 or vv.shape[1] != 2: + raise PatternError(f'vertex_lists contents must be an Nx2 arrays (polygon #{ii} fails)') + if vv.shape[0] < 3: + raise PatternError(f'vertex_lists contents must have at least 3 vertices (Nx2 where N>2) (polygon ${ii} has shape {vv.shape})') + self._vertices = val + + # xs property + @property + def xs(self) -> NDArray[numpy.float64]: + """ + All vertex x coords as a 1D ndarray + """ + return self.vertices[:, 0] + + def __init__( + self, + vertex_lists: Iterable[ArrayLike], + *, + offset: ArrayLike = (0.0, 0.0), + rotation: float = 0.0, + repetition: Repetition | None = None, + annotations: annotations_t | None = None, + raw: bool = False, + ) -> None: + if raw: + assert isinstance(vertex_lists, list) + assert all(isinstance(vv, numpy.ndarray) for vv in vertex_lists) + assert isinstance(offset, numpy.ndarray) + self._vertex_lists = vertex_lists + self._offset = offset + self._repetition = repetition + self._annotations = annotations if annotations is not None else {} + else: + self.vertices = vertices + self.offset = offset + self.repetition = repetition + self.annotations = annotations if annotations is not None else {} + self.rotate(rotation) + + def __deepcopy__(self, memo: dict | None = None) -> 'PolyCollection': + memo = {} if memo is None else memo + new = copy.copy(self) + new._offset = self._offset.copy() + new._vertex_lists = [vv.copy() for vv in self._vertex_lists] + new._annotations = copy.deepcopy(self._annotations) + return new + + def __eq__(self, other: Any) -> bool: + return ( + type(self) is type(other) + and numpy.array_equal(self.offset, other.offset) + and all(numpy.array_equal(ss, oo) for ss, oo in zip(self.vertices, other.vertices)) + and self.repetition == other.repetition + and annotations_eq(self.annotations, other.annotations) + ) + + def __lt__(self, other: Shape) -> bool: + if type(self) is not type(other): + if repr(type(self)) != repr(type(other)): + return repr(type(self)) < repr(type(other)) + return id(type(self)) < id(type(other)) + + other = cast(PolyCollection, other) + for vv, oo in zip(self.vertices, other.vertices): + if not numpy.array_equal(vv, oo): + min_len = min(vv.shape[0], oo.shape[0]) + eq_mask = vv[:min_len] != oo[:min_len] + eq_lt = vv[:min_len] < oo[:min_len] + eq_lt_masked = eq_lt[eq_mask] + if eq_lt_masked.size > 0: + return eq_lt_masked.flat[0] + return vv.shape[0] < oo.shape[0] + if len(self.vertex_lists) != len(other.vertex_lists): + return len(self.vertex_lists) < len(other.vertex_lists): + if not numpy.array_equal(self.offset, other.offset): + return tuple(self.offset) < tuple(other.offset) + if self.repetition != other.repetition: + return rep2key(self.repetition) < rep2key(other.repetition) + return annotations_lt(self.annotations, other.annotations) + + def pop_as_polygon(self, index: int) -> 'Polygon': + """ + Remove one polygon from the list, and return it as a `Polygon` object. + + Args: + index: which polygon to pop + """ + verts = self.vertex_lists.pop(index) + return Polygon( + vertices=verts, + offset=self.offset, + repetition=self.repetition.copy(), + annotations=copy.deepcopy(self.annotations), + ) + + def to_polygons( + self, + num_vertices: int | None = None, # unused # noqa: ARG002 + max_arclen: float | None = None, # unused # noqa: ARG002 + ) -> list['Polygon']: + return [Polygon( + vertices=vv, + offset=self.offset, + repetition=self.repetition.copy(), + annotations=copy.deepcopy(self.annotations), + ) for vv in self.vertex_lists] + + def get_bounds_single(self) -> NDArray[numpy.float64]: # TODO note shape get_bounds doesn't include repetition + mins = [numpy.min(vv, axis=0) for vv self.vertex_lists] + maxs = [numpy.max(vv, axis=0) for vv self.vertex_lists] + return numpy.vstack((self.offset + numpy.min(self.vertex_lists, axis=0), + self.offset + numpy.max(self.vertex_lists, axis=0))) + + def rotate(self, theta: float) -> 'Polygon': + if theta != 0: + for vv in self.vertex_lists: + vv[:] = numpy.dot(rotation_matrix_2d(theta), vv.T).T + return self + + def mirror(self, axis: int = 0) -> 'Polygon': + for vv in self.vertex_lists: + vv[:, axis - 1] *= -1 + return self + + def scale_by(self, c: float) -> 'Polygon': + for vv in self.vertex_lists: + vv *= c + return self + + def normalized_form(self, norm_value: float) -> normalized_shape_tuple: + # Note: this function is going to be pretty slow for many-vertexed polygons, relative to + # other shapes + meanv = numpy.concatenate(self.vertex_lists).mean(axis=0) + zeroed_vertices = [vv - meanv for vv in self.vertex_lists] + offset = meanv + self.offset + + scale = zeroed_vertices.std() + normed_vertices = zeroed_vertices / scale + + _, _, vertex_axis = numpy.linalg.svd(zeroed_vertices) + rotation = numpy.arctan2(vertex_axis[0][1], vertex_axis[0][0]) % (2 * pi) + rotated_vertices = numpy.vstack([numpy.dot(rotation_matrix_2d(-rotation), v) + for v in normed_vertices]) + + # Reorder the vertices so that the one with lowest x, then y, comes first. + x_min = rotated_vertices[:, 0].argmin() + if not is_scalar(x_min): + y_min = rotated_vertices[x_min, 1].argmin() + x_min = cast(Sequence, x_min)[y_min] + reordered_vertices = numpy.roll(rotated_vertices, -x_min, axis=0) + + # TODO: normalize mirroring? + + return ((type(self), reordered_vertices.data.tobytes()), + (offset, scale / norm_value, rotation, False), + lambda: Polygon(reordered_vertices * norm_value)) + + def __repr__(self) -> str: + centroid = self.offset + numpy.concatenate(self.vertex_lists).mean(axis=0) + return f'' From e6d96bb7a539bf29eea17162a24b4a5a131b9a63 Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 02/11] add gdsii_arrow --- masque/file/gdsii_arrow.py | 275 +++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..9accc0b --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,275 @@ +""" +GDSII file format readers and writers using the `klamath` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/debug/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + mlib = Library() + for cell in libarr['cells']: + name = libarr['cell_names'][cell['id'].as_py()].as_py() + pat = read_cell(cell, libarr['cell_names'], raw_mode=raw_mode) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def read_cell( + cellarr: pyarrow.Array, + cell_names: pyarrow.Array, + raw_mode: bool = True, + ) -> Pattern: + """ + TODO + Read elements from a GDS structure and build a Pattern from them. + + Args: + stream: Seekable stream, positioned at a record boundary. + Will be read until an ENDSTR record is consumed. + name: Name of the resulting Pattern + raw_mode: If True, bypass per-shape data validation. Default True. + + Returns: + A pattern containing the elements that were read. + """ + pat = Pattern() + + for refarr in cellarr['refs']: + target = cell_names[refarr['target'].as_py()].as_py() + args = dict( + offset = (refarr['x'].as_py(), refarr['y'].as_py()), + ) + if (mirr := refarr['invert_y']).is_valid: + args['mirrored'] = mirr.as_py() + if (rot := refarr['angle_deg']).is_valid: + args['rotation'] = numpy.deg2rad(rot.as_py()) + if (mag := refarr['mag']).is_valid: + args['scale'] = mag.as_py() + if (rep := refarr['repetition']).is_valid: + repetition = Grid( + a_vector = (rep['x0'].as_py(), rep['y0'].as_py()), + b_vector = (rep['x1'].as_py(), rep['y1'].as_py()), + a_count = rep['count0'].as_py(), + b_count = rep['count1'].as_py(), + ) + args['repetition'] = repetition + ref = Ref(**args) + pat.refs[target].append(ref) + + for bnd in cellarr['boundaries']: + layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) + args = dict( + vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], + ) + + if (props := bnd['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + poly = Polygon(**args) + pat.shapes[layer].append(poly) + + for gpath in cellarr['paths']: + layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) + args = dict( + vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), + ) + + if (gcap := gpath['path_type']).is_valid: + mcap = path_cap_map[gcap.as_py()] + args['cap'] = mcap + if mcap == Path.Cap.SquareCustom: + extensions = [0, 0] + if (ext0 := gpath['extension_start']).is_valid: + extensions[0] = ext0.as_py() + if (ext1 := gpath['extension_end']).is_valid: + extensions[1] = ext1.as_py() + + args['extensions'] = extensions + + if (width := gpath['width']).is_valid: + args['width'] = width.as_py() + else: + args['width'] = 0 + + if (props := gpath['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mpath = Path(**args) + pat.shapes[layer].append(mpath) + + for gtext in cellarr['texts']: + layer = (gtext['layer'].as_py(), gtext['dtype'].as_py()) + args = dict( + offset = (gtext['x'].as_py(), gtext['y'].as_py()), + string = gtext['string'].as_py(), + ) + + if (props := gtext['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mlabel = Label(**args) + pat.labels[layer].append(mlabel) + + return pat + + +def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: + return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From 1eac3baf6a83962bf70d69bd8024071fa6d8447a Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 15 Apr 2025 17:21:49 -0700 Subject: [PATCH 03/11] [pattern] add arg to , useful for whole-library scaling --- masque/pattern.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/masque/pattern.py b/masque/pattern.py index 0ae230d..afd73fc 100644 --- a/masque/pattern.py +++ b/masque/pattern.py @@ -648,21 +648,25 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): cast(Scalable, entry).scale_by(c) return self - def scale_by(self, c: float) -> Self: + def scale_by(self, c: float, scale_refs: bool = True) -> Self: """ Scale this Pattern by the given value - (all shapes and refs and their offsets are scaled, - as are all label and port offsets) + All shapes and (optionally) refs and their offsets are scaled, + as are all label and port offsets. Args: c: factor to scale by + scale_refs: Whether to scale refs. Ref offsets are always scaled, + but it may be desirable to not scale the ref itself (e.g. if + the target cell was also scaled). Returns: self """ for entry in chain_elements(self.shapes, self.refs): cast(Positionable, entry).offset *= c - cast(Scalable, entry).scale_by(c) + if scale_refs or not isinstance(entry, Ref): + cast(Scalable, entry).scale_by(c) rep = cast(Repeatable, entry).repetition if rep: From 284c7e4fd0f3671d538e38e91fcea1d4be5b087b Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 15 Apr 2025 17:25:56 -0700 Subject: [PATCH 04/11] Use quoted first arg for cast() ruff rule TC006 --- masque/builder/utils.py | 8 ++++---- masque/file/dxf.py | 2 +- masque/file/gdsii.py | 2 +- masque/file/oasis.py | 18 +++++++++--------- masque/library.py | 12 ++++++------ masque/pattern.py | 28 ++++++++++++++-------------- masque/repetition.py | 4 ++-- masque/shapes/arc.py | 8 ++++---- masque/shapes/circle.py | 2 +- masque/shapes/ellipse.py | 2 +- masque/shapes/path.py | 4 ++-- masque/shapes/polygon.py | 10 ++++++---- masque/shapes/text.py | 2 +- masque/traits/rotatable.py | 11 ++++++----- 14 files changed, 58 insertions(+), 55 deletions(-) diff --git a/masque/builder/utils.py b/masque/builder/utils.py index 6e3334d..3109f46 100644 --- a/masque/builder/utils.py +++ b/masque/builder/utils.py @@ -169,11 +169,11 @@ def ell( 'emax', 'max_extension', 'min_past_furthest',): if numpy.size(bound) == 2: - bound = cast(Sequence[float], bound) + bound = cast('Sequence[float]', bound) rot_bound = (rot_matrix @ ((bound[0], 0), (0, bound[1])))[0, :] else: - bound = cast(float, bound) + bound = cast('float', bound) rot_bound = numpy.array(bound) if rot_bound < 0: @@ -185,10 +185,10 @@ def ell( offsets += rot_bound.min() - offsets.max() else: if numpy.size(bound) == 2: - bound = cast(Sequence[float], bound) + bound = cast('Sequence[float]', bound) rot_bound = (rot_matrix @ bound)[0] else: - bound = cast(float, bound) + bound = cast('float', bound) neg = (direction + pi / 4) % (2 * pi) > pi rot_bound = -bound if neg else bound diff --git a/masque/file/dxf.py b/masque/file/dxf.py index dc3d6f3..1cf5e88 100644 --- a/masque/file/dxf.py +++ b/masque/file/dxf.py @@ -132,7 +132,7 @@ def writefile( with tmpfile(path) as base_stream: streams: tuple[Any, ...] = (base_stream,) if path.suffix == '.gz': - gz_stream = cast(IO[bytes], gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb')) + gz_stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb')) streams = (gz_stream,) + streams else: gz_stream = base_stream diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index 71ea94f..c323ecf 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -145,7 +145,7 @@ def writefile( with tmpfile(path) as base_stream: streams: tuple[Any, ...] = (base_stream,) if path.suffix == '.gz': - stream = cast(IO[bytes], gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb', compresslevel=6)) streams = (stream,) + streams else: stream = base_stream diff --git a/masque/file/oasis.py b/masque/file/oasis.py index 0e2305a..e64bb4c 100644 --- a/masque/file/oasis.py +++ b/masque/file/oasis.py @@ -190,7 +190,7 @@ def writefile( with tmpfile(path) as base_stream: streams: tuple[Any, ...] = (base_stream,) if path.suffix == '.gz': - stream = cast(IO[bytes], gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb')) + stream = cast('IO[bytes]', gzip.GzipFile(filename='', mtime=0, fileobj=base_stream, mode='wb')) streams += (stream,) else: stream = base_stream @@ -551,7 +551,7 @@ def _shapes_to_elements( circle = fatrec.Circle( layer=layer, datatype=datatype, - radius=cast(int, radius), + radius=cast('int', radius), x=offset[0], y=offset[1], properties=properties, @@ -568,8 +568,8 @@ def _shapes_to_elements( path = fatrec.Path( layer=layer, datatype=datatype, - point_list=cast(Sequence[Sequence[int]], deltas), - half_width=cast(int, half_width), + point_list=cast('Sequence[Sequence[int]]', deltas), + half_width=cast('int', half_width), x=xy[0], y=xy[1], extension_start=extension_start, # TODO implement multiple cap types? @@ -587,7 +587,7 @@ def _shapes_to_elements( datatype=datatype, x=xy[0], y=xy[1], - point_list=cast(list[list[int]], points), + point_list=cast('list[list[int]]', points), properties=properties, repetition=repetition, )) @@ -651,10 +651,10 @@ def repetition_masq2fata( a_count = rint_cast(rep.a_count) b_count = rint_cast(rep.b_count) if rep.b_count is not None else None frep = fatamorgana.GridRepetition( - a_vector=cast(list[int], a_vector), - b_vector=cast(list[int] | None, b_vector), - a_count=cast(int, a_count), - b_count=cast(int | None, b_count), + a_vector=cast('list[int]', a_vector), + b_vector=cast('list[int] | None', b_vector), + a_count=cast('int', a_count), + b_count=cast('int | None', b_count), ) offset = (0, 0) elif isinstance(rep, Arbitrary): diff --git a/masque/library.py b/masque/library.py index 90202dc..e41d27f 100644 --- a/masque/library.py +++ b/masque/library.py @@ -211,7 +211,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): if isinstance(tops, str): tops = (tops,) - keep = cast(set[str], self.referenced_patterns(tops) - {None}) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) keep |= set(tops) filtered = {kk: vv for kk, vv in self.items() if kk in keep} @@ -314,7 +314,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): flatten_single(top) assert None not in flattened.values() - return cast(dict[str, 'Pattern'], flattened) + return cast('dict[str, Pattern]', flattened) def get_name( self, @@ -504,7 +504,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): raise LibraryError('visit_* functions returned a new `Pattern` object' ' but no top-level name was provided in `hierarchy`') - cast(ILibrary, self)[name] = pattern + cast('ILibrary', self)[name] = pattern return self @@ -542,7 +542,7 @@ class ILibraryView(Mapping[str, 'Pattern'], metaclass=ABCMeta): Return: Topologically sorted list of pattern names. """ - return cast(list[str], list(TopologicalSorter(self.child_graph()).static_order())) + return cast('list[str]', list(TopologicalSorter(self.child_graph()).static_order())) def find_refs_local( self, @@ -827,7 +827,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): for old_name in temp: new_name = rename_map.get(old_name, old_name) pat = self[new_name] - pat.refs = map_targets(pat.refs, lambda tt: cast(dict[str | None, str | None], rename_map).get(tt, tt)) + pat.refs = map_targets(pat.refs, lambda tt: cast('dict[str | None, str | None]', rename_map).get(tt, tt)) return rename_map @@ -1047,7 +1047,7 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): if isinstance(tops, str): tops = (tops,) - keep = cast(set[str], self.referenced_patterns(tops) - {None}) + keep = cast('set[str]', self.referenced_patterns(tops) - {None}) keep |= set(tops) new = type(self)() diff --git a/masque/pattern.py b/masque/pattern.py index afd73fc..5bf030a 100644 --- a/masque/pattern.py +++ b/masque/pattern.py @@ -491,7 +491,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): """ pat = self.deepcopy().polygonize().flatten(library=library) polys = [ - cast(Polygon, shape).vertices + cast(Polygon, shape).offset + cast('Polygon', shape).vertices + cast('Polygon', shape).offset for shape in chain_elements(pat.shapes) ] return polys @@ -533,7 +533,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): n_elems = sum(1 for _ in chain_elements(self.shapes, self.labels)) ebounds = numpy.full((n_elems, 2, 2), nan) for ee, entry in enumerate(chain_elements(self.shapes, self.labels)): - maybe_ebounds = cast(Bounded, entry).get_bounds() + maybe_ebounds = cast('Bounded', entry).get_bounds() if maybe_ebounds is not None: ebounds[ee] = maybe_ebounds mask = ~numpy.isnan(ebounds[:, 0, 0]) @@ -631,7 +631,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain(chain_elements(self.shapes, self.labels, self.refs), self.ports.values()): - cast(Positionable, entry).translate(offset) + cast('Positionable', entry).translate(offset) return self def scale_elements(self, c: float) -> Self: @@ -645,7 +645,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain_elements(self.shapes, self.refs): - cast(Scalable, entry).scale_by(c) + cast('Scalable', entry).scale_by(c) return self def scale_by(self, c: float, scale_refs: bool = True) -> Self: @@ -664,18 +664,18 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain_elements(self.shapes, self.refs): - cast(Positionable, entry).offset *= c + cast('Positionable', entry).offset *= c if scale_refs or not isinstance(entry, Ref): - cast(Scalable, entry).scale_by(c) + cast('Scalable', entry).scale_by(c) - rep = cast(Repeatable, entry).repetition + rep = cast('Repeatable', entry).repetition if rep: rep.scale_by(c) for label in chain_elements(self.labels): - cast(Positionable, label).offset *= c + cast('Positionable', label).offset *= c - rep = cast(Repeatable, label).repetition + rep = cast('Repeatable', label).repetition if rep: rep.scale_by(c) @@ -712,8 +712,8 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain(chain_elements(self.shapes, self.refs, self.labels), self.ports.values()): - old_offset = cast(Positionable, entry).offset - cast(Positionable, entry).offset = numpy.dot(rotation_matrix_2d(rotation), old_offset) + old_offset = cast('Positionable', entry).offset + cast('Positionable', entry).offset = numpy.dot(rotation_matrix_2d(rotation), old_offset) return self def rotate_elements(self, rotation: float) -> Self: @@ -727,7 +727,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain(chain_elements(self.shapes, self.refs), self.ports.values()): - cast(Rotatable, entry).rotate(rotation) + cast('Rotatable', entry).rotate(rotation) return self def mirror_element_centers(self, across_axis: int = 0) -> Self: @@ -742,7 +742,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain(chain_elements(self.shapes, self.refs, self.labels), self.ports.values()): - cast(Positionable, entry).offset[across_axis - 1] *= -1 + cast('Positionable', entry).offset[across_axis - 1] *= -1 return self def mirror_elements(self, across_axis: int = 0) -> Self: @@ -758,7 +758,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): self """ for entry in chain(chain_elements(self.shapes, self.refs), self.ports.values()): - cast(Mirrorable, entry).mirror(across_axis) + cast('Mirrorable', entry).mirror(across_axis) return self def mirror(self, across_axis: int = 0) -> Self: diff --git a/masque/repetition.py b/masque/repetition.py index a365909..e6d00fc 100644 --- a/masque/repetition.py +++ b/masque/repetition.py @@ -294,7 +294,7 @@ class Grid(Repetition): def __le__(self, other: Repetition) -> bool: if type(self) is not type(other): return repr(type(self)) < repr(type(other)) - other = cast(Grid, other) + other = cast('Grid', other) if self.a_count != other.a_count: return self.a_count < other.a_count if self.b_count != other.b_count: @@ -357,7 +357,7 @@ class Arbitrary(Repetition): def __le__(self, other: Repetition) -> bool: if type(self) is not type(other): return repr(type(self)) < repr(type(other)) - other = cast(Arbitrary, other) + other = cast('Arbitrary', other) if self.displacements.size != other.displacements.size: return self.displacements.size < other.displacements.size diff --git a/masque/shapes/arc.py b/masque/shapes/arc.py index b3a9b7d..f3f4e1e 100644 --- a/masque/shapes/arc.py +++ b/masque/shapes/arc.py @@ -206,7 +206,7 @@ class Arc(Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Arc, other) + other = cast('Arc', other) if self.width != other.width: return self.width < other.width if not numpy.array_equal(self.radii, other.radii): @@ -233,7 +233,7 @@ class Arc(Shape): r0, r1 = self.radii # Convert from polar angle to ellipse parameter (for [rx*cos(t), ry*sin(t)] representation) - a_ranges = cast(_array2x2_t, self._angles_to_parameters()) + a_ranges = cast('_array2x2_t', self._angles_to_parameters()) # Approximate perimeter via numerical integration @@ -321,7 +321,7 @@ class Arc(Shape): If the extrema are innaccessible due to arc constraints, check the arc endpoints instead. """ - a_ranges = cast(_array2x2_t, self._angles_to_parameters()) + a_ranges = cast('_array2x2_t', self._angles_to_parameters()) mins = [] maxs = [] @@ -432,7 +432,7 @@ class Arc(Shape): [[x2, y2], [x3, y3]]], would create this arc from its corresponding ellipse. ``` """ - a_ranges = cast(_array2x2_t, self._angles_to_parameters()) + a_ranges = cast('_array2x2_t', self._angles_to_parameters()) mins = [] maxs = [] diff --git a/masque/shapes/circle.py b/masque/shapes/circle.py index 5f8ebe0..2d403b4 100644 --- a/masque/shapes/circle.py +++ b/masque/shapes/circle.py @@ -84,7 +84,7 @@ class Circle(Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Circle, other) + other = cast('Circle', other) if not self.radius == other.radius: return self.radius < other.radius if not numpy.array_equal(self.offset, other.offset): diff --git a/masque/shapes/ellipse.py b/masque/shapes/ellipse.py index 9c671d6..0d6a6c5 100644 --- a/masque/shapes/ellipse.py +++ b/masque/shapes/ellipse.py @@ -134,7 +134,7 @@ class Ellipse(Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Ellipse, other) + other = cast('Ellipse', other) if not numpy.array_equal(self.radii, other.radii): return tuple(self.radii) < tuple(other.radii) if not numpy.array_equal(self.offset, other.offset): diff --git a/masque/shapes/path.py b/masque/shapes/path.py index 717e59f..93e85ea 100644 --- a/masque/shapes/path.py +++ b/masque/shapes/path.py @@ -223,7 +223,7 @@ class Path(Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Path, other) + other = cast('Path', other) if self.width != other.width: return self.width < other.width if self.cap != other.cap: @@ -405,7 +405,7 @@ class Path(Shape): x_min = rotated_vertices[:, 0].argmin() if not is_scalar(x_min): y_min = rotated_vertices[x_min, 1].argmin() - x_min = cast(Sequence, x_min)[y_min] + x_min = cast('Sequence', x_min)[y_min] reordered_vertices = numpy.roll(rotated_vertices, -x_min, axis=0) width0 = self.width / norm_value diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index cbcbe63..10fd522 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -1,5 +1,4 @@ -from typing import Any, cast -from collections.abc import Sequence +from typing import Any, cast, TYPE_CHECKING import copy import functools @@ -13,6 +12,9 @@ from ..repetition import Repetition from ..utils import is_scalar, rotation_matrix_2d, annotations_lt, annotations_eq, rep2key from ..utils import remove_colinear_vertices, remove_duplicate_vertices, annotations_t +if TYPE_CHECKING: + from collections.abc import Sequence + @functools.total_ordering class Polygon(Shape): @@ -129,7 +131,7 @@ class Polygon(Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Polygon, other) + other = cast('Polygon', other) if not numpy.array_equal(self.vertices, other.vertices): min_len = min(self.vertices.shape[0], other.vertices.shape[0]) eq_mask = self.vertices[:min_len] != other.vertices[:min_len] @@ -395,7 +397,7 @@ class Polygon(Shape): x_min = rotated_vertices[:, 0].argmin() if not is_scalar(x_min): y_min = rotated_vertices[x_min, 1].argmin() - x_min = cast(Sequence, x_min)[y_min] + x_min = cast('Sequence', x_min)[y_min] reordered_vertices = numpy.roll(rotated_vertices, -x_min, axis=0) # TODO: normalize mirroring? diff --git a/masque/shapes/text.py b/masque/shapes/text.py index e936796..69318ac 100644 --- a/masque/shapes/text.py +++ b/masque/shapes/text.py @@ -115,7 +115,7 @@ class Text(RotatableImpl, Shape): if repr(type(self)) != repr(type(other)): return repr(type(self)) < repr(type(other)) return id(type(self)) < id(type(other)) - other = cast(Text, other) + other = cast('Text', other) if not self.height == other.height: return self.height < other.height if not self.string == other.string: diff --git a/masque/traits/rotatable.py b/masque/traits/rotatable.py index f873ce4..04816f1 100644 --- a/masque/traits/rotatable.py +++ b/masque/traits/rotatable.py @@ -1,14 +1,15 @@ -from typing import Self, cast, Any +from typing import Self, cast, Any, TYPE_CHECKING from abc import ABCMeta, abstractmethod import numpy from numpy import pi from numpy.typing import ArrayLike -from .positionable import Positionable from ..error import MasqueError from ..utils import rotation_matrix_2d +if TYPE_CHECKING: + from .positionable import Positionable _empty_slots = () # Workaround to get mypy to ignore intentionally empty slots for superclass @@ -113,9 +114,9 @@ class PivotableImpl(Pivotable, metaclass=ABCMeta): def rotate_around(self, pivot: ArrayLike, rotation: float) -> Self: pivot = numpy.asarray(pivot, dtype=float) - cast(Positionable, self).translate(-pivot) - cast(Rotatable, self).rotate(rotation) + cast('Positionable', self).translate(-pivot) + cast('Rotatable', self).rotate(rotation) self.offset = numpy.dot(rotation_matrix_2d(rotation), self.offset) # type: ignore # mypy#3004 - cast(Positionable, self).translate(+pivot) + cast('Positionable', self).translate(+pivot) return self From 560c165f2ed5c151bce74f6f55aa0e53843751ec Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 15 Apr 2025 17:26:33 -0700 Subject: [PATCH 05/11] remove deprecated rule from ignore list --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ab4661e..9587a04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,6 @@ lint.ignore = [ "ANN002", # *args "ANN003", # **kwargs "ANN401", # Any - "ANN101", # self: Self "SIM108", # single-line if / else assignment "RET504", # x=y+z; return x "PIE790", # unnecessary pass From c1bfee1dddb55b30d247306759a348cad9783c1f Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 15 Apr 2025 17:34:05 -0700 Subject: [PATCH 06/11] [library] minor stylistic cleanup --- masque/library.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/masque/library.py b/masque/library.py index e41d27f..b52da74 100644 --- a/masque/library.py +++ b/masque/library.py @@ -944,8 +944,8 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): shape_table: dict[tuple, list] = defaultdict(list) for layer, sseq in pat.shapes.items(): - for i, shape in enumerate(sseq): - if any(isinstance(shape, t) for t in exclude_types): + for ii, shape in enumerate(sseq): + if any(isinstance(shape, tt) for tt in exclude_types): continue base_label, values, _func = shape.normalized_form(norm_value) @@ -954,16 +954,16 @@ class ILibrary(ILibraryView, MutableMapping[str, 'Pattern'], metaclass=ABCMeta): if label not in shape_pats: continue - shape_table[label].append((i, values)) + shape_table[label].append((ii, values)) # For repeated shapes, create a `Pattern` holding a normalized shape object, # and add `pat.refs` entries for each occurrence in pat. Also, note down that # we should delete the `pat.shapes` entries for which we made `Ref`s. shapes_to_remove = [] - for label in shape_table: + for label, shape_entries in shape_table.items(): layer = label[-1] target = label2name(label) - for ii, values in shape_table[label]: + for ii, values in shape_entries: offset, scale, rotation, mirror_x = values pat.ref(target=target, offset=offset, scale=scale, rotation=rotation, mirrored=(mirror_x, False)) From 35e28acb89e4d0e460cd1d31be1407dbd80d84b5 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sun, 20 Apr 2025 23:24:01 -0700 Subject: [PATCH 07/11] [polygon] Only call rotate if necessary --- masque/shapes/polygon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/masque/shapes/polygon.py b/masque/shapes/polygon.py index 10fd522..2976271 100644 --- a/masque/shapes/polygon.py +++ b/masque/shapes/polygon.py @@ -107,7 +107,8 @@ class Polygon(Shape): self.offset = offset self.repetition = repetition self.annotations = annotations if annotations is not None else {} - self.rotate(rotation) + if rotation: + self.rotate(rotation) def __deepcopy__(self, memo: dict | None = None) -> 'Polygon': memo = {} if memo is None else memo From de9714041f15e7916a381662962c4910b7f5448a Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 08/11] add gdsii_arrow --- masque/file/gdsii_arrow.py | 275 +++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..9accc0b --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,275 @@ +""" +GDSII file format readers and writers using the `klamath` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/debug/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + mlib = Library() + for cell in libarr['cells']: + name = libarr['cell_names'][cell['id'].as_py()].as_py() + pat = read_cell(cell, libarr['cell_names'], raw_mode=raw_mode) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def read_cell( + cellarr: pyarrow.Array, + cell_names: pyarrow.Array, + raw_mode: bool = True, + ) -> Pattern: + """ + TODO + Read elements from a GDS structure and build a Pattern from them. + + Args: + stream: Seekable stream, positioned at a record boundary. + Will be read until an ENDSTR record is consumed. + name: Name of the resulting Pattern + raw_mode: If True, bypass per-shape data validation. Default True. + + Returns: + A pattern containing the elements that were read. + """ + pat = Pattern() + + for refarr in cellarr['refs']: + target = cell_names[refarr['target'].as_py()].as_py() + args = dict( + offset = (refarr['x'].as_py(), refarr['y'].as_py()), + ) + if (mirr := refarr['invert_y']).is_valid: + args['mirrored'] = mirr.as_py() + if (rot := refarr['angle_deg']).is_valid: + args['rotation'] = numpy.deg2rad(rot.as_py()) + if (mag := refarr['mag']).is_valid: + args['scale'] = mag.as_py() + if (rep := refarr['repetition']).is_valid: + repetition = Grid( + a_vector = (rep['x0'].as_py(), rep['y0'].as_py()), + b_vector = (rep['x1'].as_py(), rep['y1'].as_py()), + a_count = rep['count0'].as_py(), + b_count = rep['count1'].as_py(), + ) + args['repetition'] = repetition + ref = Ref(**args) + pat.refs[target].append(ref) + + for bnd in cellarr['boundaries']: + layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) + args = dict( + vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], + ) + + if (props := bnd['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + poly = Polygon(**args) + pat.shapes[layer].append(poly) + + for gpath in cellarr['paths']: + layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) + args = dict( + vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), + ) + + if (gcap := gpath['path_type']).is_valid: + mcap = path_cap_map[gcap.as_py()] + args['cap'] = mcap + if mcap == Path.Cap.SquareCustom: + extensions = [0, 0] + if (ext0 := gpath['extension_start']).is_valid: + extensions[0] = ext0.as_py() + if (ext1 := gpath['extension_end']).is_valid: + extensions[1] = ext1.as_py() + + args['extensions'] = extensions + + if (width := gpath['width']).is_valid: + args['width'] = width.as_py() + else: + args['width'] = 0 + + if (props := gpath['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mpath = Path(**args) + pat.shapes[layer].append(mpath) + + for gtext in cellarr['texts']: + layer = (gtext['layer'].as_py(), gtext['dtype'].as_py()) + args = dict( + offset = (gtext['x'].as_py(), gtext['y'].as_py()), + string = gtext['string'].as_py(), + ) + + if (props := gtext['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mlabel = Label(**args) + pat.labels[layer].append(mlabel) + + return pat + + +def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: + return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From dc894916944dc7f9bf14da8751a56687b1af2d29 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 19 Apr 2025 17:49:00 -0700 Subject: [PATCH 09/11] actually make use of raw mode --- masque/file/gdsii_arrow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 9accc0b..02f4ed8 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -184,6 +184,8 @@ def read_cell( layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) args = dict( vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], + raw = raw_mode, + offset = numpy.zeros(2), ) if (props := bnd['properties']).is_valid: @@ -196,6 +198,8 @@ def read_cell( layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) args = dict( vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), + offset = numpy.zeros(2), + raw = raw_mode, ) if (gcap := gpath['path_type']).is_valid: From 88bd5e897e0df864d4a9408a21a69e4b23175fd3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sun, 20 Apr 2025 23:28:59 -0700 Subject: [PATCH 10/11] split out _read_to_arrow for ease of debugging --- masque/file/gdsii_arrow.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 02f4ed8..e34d25f 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -60,6 +60,24 @@ def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) +def _read_to_arrow( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> pyarrow.Array: + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + return arrow_arr + + def readfile( filename: str | pathlib.Path, *args, @@ -75,15 +93,7 @@ def readfile( *args: passed to `read()` **kwargs: passed to `read()` """ - path = pathlib.Path(filename) - path.resolve() - ptr_array = ffi.new('struct ArrowArray[]', 1) - ptr_schema = ffi.new('struct ArrowSchema[]', 1) - clib.read_path(str(path).encode(), ptr_array, ptr_schema) - - iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) - iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + arrow_arr = _read_to_arrow(filename) assert len(arrow_arr) == 1 results = read_arrow(arrow_arr[0]) From 76511b95e6eb13a7b9564ce9311443f755f437d3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 21 Apr 2025 19:03:32 -0700 Subject: [PATCH 11/11] gdsii_arrow wip --- masque/file/gdsii_arrow.py | 113 ++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 19 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index e34d25f..0c356c4 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -31,6 +31,7 @@ from pprint import pformat import numpy from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_equal import pyarrow from pyarrow.cffi import ffi @@ -44,7 +45,7 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/debug/libklamath_rs_ext.so') +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') @@ -127,10 +128,41 @@ def read_arrow( """ library_info = _read_header(libarr) + layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_tups = [tuple(pair) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + bnd = libarr['cells'].values.field('boundaries') + boundary = dict( + offsets = bnd.offsets.to_numpy(), + xy_arr = bnd.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = bnd.values.field('xy').offsets.to_numpy() // 2, + layer_tups = layer_tups, + layer_inds = bnd.values.field('layer').to_numpy(), + prop_off = bnd.values.field('properties').offsets.to_numpy(), + prop_key = bnd.values.field('properties').values.field('key').to_numpy(), + prop_val = bnd.values.field('properties').values.field('value').to_pylist(), + ) + + pth = libarr['cells'].values.field('boundaries') + path = dict( + offsets = pth.offsets.to_numpy(), + xy_arr = pth.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = pth.values.field('xy').offsets.to_numpy() // 2, + layer_tups = layer_tups, + layer_inds = pth.values.field('layer').to_numpy(), + prop_off = pth.values.field('properties').offsets.to_numpy(), + prop_key = pth.values.field('properties').values.field('key').to_numpy(), + prop_val = pth.values.field('properties').values.field('value').to_pylist(), + ) + + mlib = Library() - for cell in libarr['cells']: - name = libarr['cell_names'][cell['id'].as_py()].as_py() - pat = read_cell(cell, libarr['cell_names'], raw_mode=raw_mode) + for cc, cell in enumerate(libarr['cells']): + name = cell_names[cell_ids[cc]] + pat = read_cell(cc, cell, libarr['cell_names'], raw_mode=raw_mode, boundary=boundary) mlib[name] = pat return mlib, library_info @@ -149,8 +181,10 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: def read_cell( + cc: int, cellarr: pyarrow.Array, cell_names: pyarrow.Array, + boundary: dict[str, NDArray], raw_mode: bool = True, ) -> Pattern: """ @@ -190,22 +224,10 @@ def read_cell( ref = Ref(**args) pat.refs[target].append(ref) - for bnd in cellarr['boundaries']: - layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) - args = dict( - vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], - raw = raw_mode, - offset = numpy.zeros(2), - ) - - if (props := bnd['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - poly = Polygon(**args) - pat.shapes[layer].append(poly) + _boundaries_to_polygons(pat, cellarr) for gpath in cellarr['paths']: - layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) + layer = (gpath['layer'].as_py(),) args = dict( vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), offset = numpy.zeros(2), @@ -236,7 +258,7 @@ def read_cell( pat.shapes[layer].append(mpath) for gtext in cellarr['texts']: - layer = (gtext['layer'].as_py(), gtext['dtype'].as_py()) + layer = (gtext['layer'].as_py(),) args = dict( offset = (gtext['x'].as_py(), gtext['y'].as_py()), string = gtext['string'].as_py(), @@ -251,6 +273,59 @@ def read_cell( return pat +def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = elem['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + zeros = numpy.zeros((elem_count, 2)) + for ee in range(elem_count): + layer = layer_tups[layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + args = dict(annotations = ann) + + path = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + pat.shapes[layer].append(path) + + +def _boundaries_to_polygons(pat: Pattern, elem: dict[str, Any], cc: int) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = elem['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_slc = slice(elem_off[cc], elem_off[cc + 1] + 1) + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + zeros = numpy.zeros((len(xy_offs) - 1, 2)) + for ee in range(len(xy_offs) - 1): + layer = layer_tups[layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + args = dict(annotations = ann) + + poly = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + pat.shapes[layer].append(poly) + + def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: return {prop['key'].as_py(): prop['value'].as_py() for prop in properties}