From de9714041f15e7916a381662962c4910b7f5448a Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Apr 2025 21:47:54 -0700 Subject: [PATCH 1/8] add gdsii_arrow --- masque/file/gdsii_arrow.py | 275 +++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 masque/file/gdsii_arrow.py diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py new file mode 100644 index 0000000..9accc0b --- /dev/null +++ b/masque/file/gdsii_arrow.py @@ -0,0 +1,275 @@ +""" +GDSII file format readers and writers using the `klamath` library. + +Note that GDSII references follow the same convention as `masque`, + with this order of operations: + 1. Mirroring + 2. Rotation + 3. Scaling + 4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets) + + Scaling, rotation, and mirroring apply to individual instances, not grid + vectors or offsets. + +Notes: + * absolute positioning is not supported + * PLEX is not supported + * ELFLAGS are not supported + * GDS does not support library- or structure-level annotations + * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. + * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) +""" +from typing import IO, cast, Any +from collections.abc import Iterable, Mapping, Callable +import io +import mmap +import logging +import pathlib +import gzip +import string +from pprint import pformat + +import numpy +from numpy.typing import ArrayLike, NDArray +import pyarrow +from pyarrow.cffi import ffi + +from .utils import is_gzipped, tmpfile +from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape +from ..shapes import Polygon, Path +from ..repetition import Grid +from ..utils import layer_t, annotations_t +from ..library import LazyLibrary, Library, ILibrary, ILibraryView + + +logger = logging.getLogger(__name__) + +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/debug/libklamath_rs_ext.so') +ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') + + +path_cap_map = { + 0: Path.Cap.Flush, + 1: Path.Cap.Circle, + 2: Path.Cap.Square, + 4: Path.Cap.SquareCustom, + } + + +def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: + return numpy.rint(val).astype(numpy.int32) + + +def readfile( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> tuple[Library, dict[str, Any]]: + """ + Wrapper for `read()` that takes a filename or path instead of a stream. + + Will automatically decompress gzipped files. + + Args: + filename: Filename to save to. + *args: passed to `read()` + **kwargs: passed to `read()` + """ + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + assert len(arrow_arr) == 1 + + results = read_arrow(arrow_arr[0]) + + return results + + +def read_arrow( + libarr: pyarrow.Array, + raw_mode: bool = True, + ) -> tuple[Library, dict[str, Any]]: + """ + # TODO check GDSII file for cycles! + Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are + translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs + are translated into Ref objects. + + Additional library info is returned in a dict, containing: + 'name': name of the library + 'meters_per_unit': number of meters per database unit (all values are in database units) + 'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns) + per database unit + + Args: + stream: Stream to read from. + raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True. + + Returns: + - dict of pattern_name:Patterns generated from GDSII structures + - dict of GDSII library info + """ + library_info = _read_header(libarr) + + mlib = Library() + for cell in libarr['cells']: + name = libarr['cell_names'][cell['id'].as_py()].as_py() + pat = read_cell(cell, libarr['cell_names'], raw_mode=raw_mode) + mlib[name] = pat + + return mlib, library_info + + +def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: + """ + Read the file header and create the library_info dict. + """ + library_info = dict( + name = libarr['lib_name'], + meters_per_unit = libarr['meters_per_db_unit'], + logical_units_per_unit = libarr['user_units_per_db_unit'], + ) + return library_info + + +def read_cell( + cellarr: pyarrow.Array, + cell_names: pyarrow.Array, + raw_mode: bool = True, + ) -> Pattern: + """ + TODO + Read elements from a GDS structure and build a Pattern from them. + + Args: + stream: Seekable stream, positioned at a record boundary. + Will be read until an ENDSTR record is consumed. + name: Name of the resulting Pattern + raw_mode: If True, bypass per-shape data validation. Default True. + + Returns: + A pattern containing the elements that were read. + """ + pat = Pattern() + + for refarr in cellarr['refs']: + target = cell_names[refarr['target'].as_py()].as_py() + args = dict( + offset = (refarr['x'].as_py(), refarr['y'].as_py()), + ) + if (mirr := refarr['invert_y']).is_valid: + args['mirrored'] = mirr.as_py() + if (rot := refarr['angle_deg']).is_valid: + args['rotation'] = numpy.deg2rad(rot.as_py()) + if (mag := refarr['mag']).is_valid: + args['scale'] = mag.as_py() + if (rep := refarr['repetition']).is_valid: + repetition = Grid( + a_vector = (rep['x0'].as_py(), rep['y0'].as_py()), + b_vector = (rep['x1'].as_py(), rep['y1'].as_py()), + a_count = rep['count0'].as_py(), + b_count = rep['count1'].as_py(), + ) + args['repetition'] = repetition + ref = Ref(**args) + pat.refs[target].append(ref) + + for bnd in cellarr['boundaries']: + layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) + args = dict( + vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], + ) + + if (props := bnd['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + poly = Polygon(**args) + pat.shapes[layer].append(poly) + + for gpath in cellarr['paths']: + layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) + args = dict( + vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), + ) + + if (gcap := gpath['path_type']).is_valid: + mcap = path_cap_map[gcap.as_py()] + args['cap'] = mcap + if mcap == Path.Cap.SquareCustom: + extensions = [0, 0] + if (ext0 := gpath['extension_start']).is_valid: + extensions[0] = ext0.as_py() + if (ext1 := gpath['extension_end']).is_valid: + extensions[1] = ext1.as_py() + + args['extensions'] = extensions + + if (width := gpath['width']).is_valid: + args['width'] = width.as_py() + else: + args['width'] = 0 + + if (props := gpath['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mpath = Path(**args) + pat.shapes[layer].append(mpath) + + for gtext in cellarr['texts']: + layer = (gtext['layer'].as_py(), gtext['dtype'].as_py()) + args = dict( + offset = (gtext['x'].as_py(), gtext['y'].as_py()), + string = gtext['string'].as_py(), + ) + + if (props := gtext['properties']).is_valid: + args['annotations'] = _properties_to_annotations(props) + + mlabel = Label(**args) + pat.labels[layer].append(mlabel) + + return pat + + +def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: + return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} + + +def check_valid_names( + names: Iterable[str], + max_length: int = 32, + ) -> None: + """ + Check all provided names to see if they're valid GDSII cell names. + + Args: + names: Collection of names to check + max_length: Max allowed length + + """ + allowed_chars = set(string.ascii_letters + string.digits + '_?$') + + bad_chars = [ + name for name in names + if not set(name).issubset(allowed_chars) + ] + + bad_lengths = [ + name for name in names + if len(name) > max_length + ] + + if bad_chars: + logger.error('Names contain invalid characters:\n' + pformat(bad_chars)) + + if bad_lengths: + logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars)) + + if bad_chars or bad_lengths: + raise LibraryError('Library contains invalid names, see log above') From dc894916944dc7f9bf14da8751a56687b1af2d29 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sat, 19 Apr 2025 17:49:00 -0700 Subject: [PATCH 2/8] actually make use of raw mode --- masque/file/gdsii_arrow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 9accc0b..02f4ed8 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -184,6 +184,8 @@ def read_cell( layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) args = dict( vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], + raw = raw_mode, + offset = numpy.zeros(2), ) if (props := bnd['properties']).is_valid: @@ -196,6 +198,8 @@ def read_cell( layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) args = dict( vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), + offset = numpy.zeros(2), + raw = raw_mode, ) if (gcap := gpath['path_type']).is_valid: From 88bd5e897e0df864d4a9408a21a69e4b23175fd3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sun, 20 Apr 2025 23:28:59 -0700 Subject: [PATCH 3/8] split out _read_to_arrow for ease of debugging --- masque/file/gdsii_arrow.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 02f4ed8..e34d25f 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -60,6 +60,24 @@ def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) +def _read_to_arrow( + filename: str | pathlib.Path, + *args, + **kwargs, + ) -> pyarrow.Array: + path = pathlib.Path(filename) + path.resolve() + ptr_array = ffi.new('struct ArrowArray[]', 1) + ptr_schema = ffi.new('struct ArrowSchema[]', 1) + clib.read_path(str(path).encode(), ptr_array, ptr_schema) + + iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) + iptr_array = int(ffi.cast('uintptr_t', ptr_array)) + arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + + return arrow_arr + + def readfile( filename: str | pathlib.Path, *args, @@ -75,15 +93,7 @@ def readfile( *args: passed to `read()` **kwargs: passed to `read()` """ - path = pathlib.Path(filename) - path.resolve() - ptr_array = ffi.new('struct ArrowArray[]', 1) - ptr_schema = ffi.new('struct ArrowSchema[]', 1) - clib.read_path(str(path).encode(), ptr_array, ptr_schema) - - iptr_schema = int(ffi.cast('uintptr_t', ptr_schema)) - iptr_array = int(ffi.cast('uintptr_t', ptr_array)) - arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema) + arrow_arr = _read_to_arrow(filename) assert len(arrow_arr) == 1 results = read_arrow(arrow_arr[0]) From 76511b95e6eb13a7b9564ce9311443f755f437d3 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 21 Apr 2025 19:03:32 -0700 Subject: [PATCH 4/8] gdsii_arrow wip --- masque/file/gdsii_arrow.py | 113 ++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 19 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index e34d25f..0c356c4 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -31,6 +31,7 @@ from pprint import pformat import numpy from numpy.typing import ArrayLike, NDArray +from numpy.testing import assert_equal import pyarrow from pyarrow.cffi import ffi @@ -44,7 +45,7 @@ from ..library import LazyLibrary, Library, ILibrary, ILibraryView logger = logging.getLogger(__name__) -clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/debug/libklamath_rs_ext.so') +clib = ffi.dlopen('/home/jan/projects/klamath-rs/target/release/libklamath_rs_ext.so') ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);') @@ -127,10 +128,41 @@ def read_arrow( """ library_info = _read_header(libarr) + layer_names_np = libarr['layers'].values.to_numpy().view('i2').reshape((-1, 2)) + layer_tups = [tuple(pair) for pair in layer_names_np] + + cell_ids = libarr['cells'].values.field('id').to_numpy() + cell_names = libarr['cell_names'].as_py() + + bnd = libarr['cells'].values.field('boundaries') + boundary = dict( + offsets = bnd.offsets.to_numpy(), + xy_arr = bnd.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = bnd.values.field('xy').offsets.to_numpy() // 2, + layer_tups = layer_tups, + layer_inds = bnd.values.field('layer').to_numpy(), + prop_off = bnd.values.field('properties').offsets.to_numpy(), + prop_key = bnd.values.field('properties').values.field('key').to_numpy(), + prop_val = bnd.values.field('properties').values.field('value').to_pylist(), + ) + + pth = libarr['cells'].values.field('boundaries') + path = dict( + offsets = pth.offsets.to_numpy(), + xy_arr = pth.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = pth.values.field('xy').offsets.to_numpy() // 2, + layer_tups = layer_tups, + layer_inds = pth.values.field('layer').to_numpy(), + prop_off = pth.values.field('properties').offsets.to_numpy(), + prop_key = pth.values.field('properties').values.field('key').to_numpy(), + prop_val = pth.values.field('properties').values.field('value').to_pylist(), + ) + + mlib = Library() - for cell in libarr['cells']: - name = libarr['cell_names'][cell['id'].as_py()].as_py() - pat = read_cell(cell, libarr['cell_names'], raw_mode=raw_mode) + for cc, cell in enumerate(libarr['cells']): + name = cell_names[cell_ids[cc]] + pat = read_cell(cc, cell, libarr['cell_names'], raw_mode=raw_mode, boundary=boundary) mlib[name] = pat return mlib, library_info @@ -149,8 +181,10 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]: def read_cell( + cc: int, cellarr: pyarrow.Array, cell_names: pyarrow.Array, + boundary: dict[str, NDArray], raw_mode: bool = True, ) -> Pattern: """ @@ -190,22 +224,10 @@ def read_cell( ref = Ref(**args) pat.refs[target].append(ref) - for bnd in cellarr['boundaries']: - layer = (bnd['layer'].as_py(), bnd['dtype'].as_py()) - args = dict( - vertices = bnd['xy'].values.to_numpy().reshape((-1, 2))[:-1], - raw = raw_mode, - offset = numpy.zeros(2), - ) - - if (props := bnd['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - poly = Polygon(**args) - pat.shapes[layer].append(poly) + _boundaries_to_polygons(pat, cellarr) for gpath in cellarr['paths']: - layer = (gpath['layer'].as_py(), gpath['dtype'].as_py()) + layer = (gpath['layer'].as_py(),) args = dict( vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), offset = numpy.zeros(2), @@ -236,7 +258,7 @@ def read_cell( pat.shapes[layer].append(mpath) for gtext in cellarr['texts']: - layer = (gtext['layer'].as_py(), gtext['dtype'].as_py()) + layer = (gtext['layer'].as_py(),) args = dict( offset = (gtext['x'].as_py(), gtext['y'].as_py()), string = gtext['string'].as_py(), @@ -251,6 +273,59 @@ def read_cell( return pat +def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = elem['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + zeros = numpy.zeros((elem_count, 2)) + for ee in range(elem_count): + layer = layer_tups[layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + args = dict(annotations = ann) + + path = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + pat.shapes[layer].append(path) + + +def _boundaries_to_polygons(pat: Pattern, elem: dict[str, Any], cc: int) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy_val = elem['xy_arr'] + layer_tups = elem['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_slc = slice(elem_off[cc], elem_off[cc + 1] + 1) + xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + zeros = numpy.zeros((len(xy_offs) - 1, 2)) + for ee in range(len(xy_offs) - 1): + layer = layer_tups[layer_inds[ee]] + vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + args = dict(annotations = ann) + + poly = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + pat.shapes[layer].append(poly) + + def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: return {prop['key'].as_py(): prop['value'].as_py() for prop in properties} From e89d912ce8c71211904fa4500172cc48524a3859 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 21 Apr 2025 20:26:34 -0700 Subject: [PATCH 5/8] allow annotations to be None breaking change, but properties are seldom used by anyone afaik --- masque/file/gdsii.py | 11 +++++++++-- masque/file/oasis.py | 2 ++ masque/pattern.py | 15 +++++++++------ masque/utils/types.py | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index c323ecf..10f5a9a 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -21,6 +21,7 @@ Notes: """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable +from types import MappingProxyType import io import mmap import logging @@ -52,6 +53,8 @@ path_cap_map = { 4: Path.Cap.SquareCustom, } +RO_EMPTY_DICT: Mapping[int, bytes] = MappingProxyType({}) + def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) @@ -399,11 +402,15 @@ def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.R return grefs -def _properties_to_annotations(properties: dict[int, bytes]) -> annotations_t: +def _properties_to_annotations(properties: Mapping[int, bytes]) -> annotations_t: + if not properties: + return None return {str(k): [v.decode()] for k, v in properties.items()} -def _annotations_to_properties(annotations: annotations_t, max_len: int = 126) -> dict[int, bytes]: +def _annotations_to_properties(annotations: annotations_t, max_len: int = 126) -> Mapping[int, bytes]: + if annotations is None: + return RO_EMPTY_DICT cum_len = 0 props = {} for key, vals in annotations.items(): diff --git a/masque/file/oasis.py b/masque/file/oasis.py index e64bb4c..642ad44 100644 --- a/masque/file/oasis.py +++ b/masque/file/oasis.py @@ -671,6 +671,8 @@ def repetition_masq2fata( def annotations_to_properties(annotations: annotations_t) -> list[fatrec.Property]: #TODO determine is_standard based on key? + if annotations is None: + return [] properties = [] for key, values in annotations.items(): vals = [AString(v) if isinstance(v, str) else v diff --git a/masque/pattern.py b/masque/pattern.py index 5bf030a..f77c64f 100644 --- a/masque/pattern.py +++ b/masque/pattern.py @@ -332,7 +332,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): )) self.ports = dict(sorted(self.ports.items())) - self.annotations = dict(sorted(self.annotations.items())) + self.annotations = dict(sorted(self.annotations.items())) if self.annotations is not None else None return self @@ -354,10 +354,13 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): for layer, lseq in other_pattern.labels.items(): self.labels[layer].extend(lseq) - annotation_conflicts = set(self.annotations.keys()) & set(other_pattern.annotations.keys()) - if annotation_conflicts: - raise PatternError(f'Annotation keys overlap: {annotation_conflicts}') - self.annotations.update(other_pattern.annotations) + if other_pattern.annotations is not None: + if self.annotations is None: + self.annotations = {} + annotation_conflicts = set(self.annotations.keys()) & set(other_pattern.annotations.keys()) + if annotation_conflicts: + raise PatternError(f'Annotation keys overlap: {annotation_conflicts}') + self.annotations.update(other_pattern.annotations) port_conflicts = set(self.ports.keys()) & set(other_pattern.ports.keys()) if port_conflicts: @@ -415,7 +418,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): elif default_keep: pat.refs = copy.copy(self.refs) - if annotations is not None: + if annotations is not None and self.annotations is not None: pat.annotations = {k: v for k, v in self.annotations.items() if annotations(k, v)} elif default_keep: pat.annotations = copy.copy(self.annotations) diff --git a/masque/utils/types.py b/masque/utils/types.py index 4060ac4..c06b7b4 100644 --- a/masque/utils/types.py +++ b/masque/utils/types.py @@ -5,7 +5,7 @@ from typing import Protocol layer_t = int | tuple[int, int] | str -annotations_t = dict[str, list[int | float | str]] +annotations_t = dict[str, list[int | float | str]] | None class SupportsBool(Protocol): From 79f20881808626adb64e7458d964abcda52e266e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:19:59 -0700 Subject: [PATCH 6/8] [utils.curves] ignore re-import of trapeziod --- masque/utils/curves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/masque/utils/curves.py b/masque/utils/curves.py index 1e41c35..8b3fcc4 100644 --- a/masque/utils/curves.py +++ b/masque/utils/curves.py @@ -5,7 +5,7 @@ from numpy import pi try: from numpy import trapezoid except ImportError: - from numpy import trapz as trapezoid + from numpy import trapz as trapezoid # type:ignore def bezier( From 4e40e3f82917f8b45d29ff3f4e40f06bfe628ed6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:20:46 -0700 Subject: [PATCH 7/8] [gdsii_arrow] use direct access for all element types --- masque/file/gdsii_arrow.py | 279 +++++++++++++++++++++++-------------- 1 file changed, 176 insertions(+), 103 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 0c356c4..8051726 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -134,35 +134,76 @@ def read_arrow( cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() - bnd = libarr['cells'].values.field('boundaries') - boundary = dict( - offsets = bnd.offsets.to_numpy(), - xy_arr = bnd.values.field('xy').values.to_numpy().reshape((-1, 2)), - xy_off = bnd.values.field('xy').offsets.to_numpy() // 2, - layer_tups = layer_tups, - layer_inds = bnd.values.field('layer').to_numpy(), - prop_off = bnd.values.field('properties').offsets.to_numpy(), - prop_key = bnd.values.field('properties').values.field('key').to_numpy(), - prop_val = bnd.values.field('properties').values.field('value').to_pylist(), + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + rf = libarr['cells'].values.field('refs') + refs = dict( + offsets = rf.offsets.to_numpy(), + targets = rf.values.field('target').to_numpy(), + xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), + angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + scale = rf.values.field('mag').fill_null(1).to_numpy(), + rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), + rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + prop_off = rf.values.field('properties').offsets.to_numpy(), + prop_key = rf.values.field('properties').values.field('key').to_numpy(), + prop_val = rf.values.field('properties').values.field('value').to_pylist(), ) - pth = libarr['cells'].values.field('boundaries') - path = dict( - offsets = pth.offsets.to_numpy(), - xy_arr = pth.values.field('xy').values.to_numpy().reshape((-1, 2)), - xy_off = pth.values.field('xy').offsets.to_numpy() // 2, - layer_tups = layer_tups, - layer_inds = pth.values.field('layer').to_numpy(), - prop_off = pth.values.field('properties').offsets.to_numpy(), - prop_key = pth.values.field('properties').values.field('key').to_numpy(), - prop_val = pth.values.field('properties').values.field('value').to_pylist(), + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), ) + elements = dict( + boundaries = get_geom(libarr, 'boundaries'), + paths = get_geom(libarr, 'paths'), + boxes = get_geom(libarr, 'boxes'), + nodes = get_geom(libarr, 'nodes'), + texts = texts, + refs = refs, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').to_numpy(), + path_type = paths.values.field('path_type').to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').to_numpy(zero_copy_only=False), + paths.values.field('extension_end').to_numpy(zero_copy_only=False), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + raw_mode = raw_mode, + ) mlib = Library() for cc, cell in enumerate(libarr['cells']): name = cell_names[cell_ids[cc]] - pat = read_cell(cc, cell, libarr['cell_names'], raw_mode=raw_mode, boundary=boundary) + pat = read_cell(cc, cell, libarr['cell_names'], global_args=global_args, elements=elements) mlib[name] = pat return mlib, library_info @@ -184,8 +225,8 @@ def read_cell( cc: int, cellarr: pyarrow.Array, cell_names: pyarrow.Array, - boundary: dict[str, NDArray], - raw_mode: bool = True, + elements: dict[str, Any], + global_args: dict[str, Any], ) -> Pattern: """ TODO @@ -202,81 +243,96 @@ def read_cell( """ pat = Pattern() - for refarr in cellarr['refs']: - target = cell_names[refarr['target'].as_py()].as_py() - args = dict( - offset = (refarr['x'].as_py(), refarr['y'].as_py()), - ) - if (mirr := refarr['invert_y']).is_valid: - args['mirrored'] = mirr.as_py() - if (rot := refarr['angle_deg']).is_valid: - args['rotation'] = numpy.deg2rad(rot.as_py()) - if (mag := refarr['mag']).is_valid: - args['scale'] = mag.as_py() - if (rep := refarr['repetition']).is_valid: - repetition = Grid( - a_vector = (rep['x0'].as_py(), rep['y0'].as_py()), - b_vector = (rep['x1'].as_py(), rep['y1'].as_py()), - a_count = rep['count0'].as_py(), - b_count = rep['count1'].as_py(), - ) - args['repetition'] = repetition - ref = Ref(**args) - pat.refs[target].append(ref) - - _boundaries_to_polygons(pat, cellarr) - - for gpath in cellarr['paths']: - layer = (gpath['layer'].as_py(),) - args = dict( - vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), - offset = numpy.zeros(2), - raw = raw_mode, - ) - - if (gcap := gpath['path_type']).is_valid: - mcap = path_cap_map[gcap.as_py()] - args['cap'] = mcap - if mcap == Path.Cap.SquareCustom: - extensions = [0, 0] - if (ext0 := gpath['extension_start']).is_valid: - extensions[0] = ext0.as_py() - if (ext1 := gpath['extension_end']).is_valid: - extensions[1] = ext1.as_py() - - args['extensions'] = extensions - - if (width := gpath['width']).is_valid: - args['width'] = width.as_py() - else: - args['width'] = 0 - - if (props := gpath['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - mpath = Path(**args) - pat.shapes[layer].append(mpath) - - for gtext in cellarr['texts']: - layer = (gtext['layer'].as_py(),) - args = dict( - offset = (gtext['x'].as_py(), gtext['y'].as_py()), - string = gtext['string'].as_py(), - ) - - if (props := gtext['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - mlabel = Label(**args) - pat.labels[layer].append(mlabel) + _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) return pat -def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: +def _grefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + targets = elem['targets'] + + rep_valid = elem['rep_valid'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + for ee in range(elem_count): + target = cell_names[targets[ee]] + offset = xy[ee] + mirr = elem['invert_y'][ee] + rot = elem['angle_rad'][ee] + mag = elem['scale'][ee] + + rep: None | Grid = None + if rep_valid[ee]: + a_vector = elem['rep_xy0'][ee] + b_vector = elem['rep_xy1'][ee] + a_count, b_count = elem['rep_counts'][ee] + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + + annotations: None | dict[int, str] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + pat.refs[target].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + for ee in range(elem_count): + layer = layer_tups[layer_inds[ee]] + offset = xy[ee] + string = elem['string'][ee] + + annotations: None | dict[int, str] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + + mlabel = Label(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: elem_off = elem['offsets'] # which elements belong to each cell xy_val = elem['xy_arr'] - layer_tups = elem['layer_tups'] + layer_tups = global_args['layer_tups'] layer_inds = elem['layer_inds'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] @@ -287,42 +343,59 @@ def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: prop_offs = elem['prop_off'][elem_slc] # which props belong to each element zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] for ee in range(elem_count): + elem_ind = elem_off[cc] + ee layer = layer_tups[layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem['width'][elem_ind] + cap_int = elem['path_type'][elem_ind] + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem['extensions'][elem_ind] + else: + cap_extensions = None + annotations: None | dict[int, str] = None prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] if prop_ii < prop_ff: - ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - args = dict(annotations = ann) + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - path = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) -def _boundaries_to_polygons(pat: Pattern, elem: dict[str, Any], cc: int) -> None: +def _boundaries_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: elem_off = elem['offsets'] # which elements belong to each cell xy_val = elem['xy_arr'] - layer_tups = elem['layer_tups'] + layer_tups = global_args['layer_tups'] layer_inds = elem['layer_inds'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] - elem_slc = slice(elem_off[cc], elem_off[cc + 1] + 1) + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - zeros = numpy.zeros((len(xy_offs) - 1, 2)) - for ee in range(len(xy_offs) - 1): + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for ee in range(elem_count): layer = layer_tups[layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + annotations: None | dict[int, str] = None prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] if prop_ii < prop_ff: - ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - args = dict(annotations = ann) + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - poly = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) pat.shapes[layer].append(poly) From 7336545f0702ea8b428c99b847afa0c970000916 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:22:01 -0700 Subject: [PATCH 8/8] [gdsii_arrow] add some TODO notes --- masque/file/gdsii_arrow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 8051726..763c438 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -1,5 +1,5 @@ """ -GDSII file format readers and writers using the `klamath` library. +GDSII file format readers and writers using the `TODO` library. Note that GDSII references follow the same convention as `masque`, with this order of operations: @@ -18,6 +18,9 @@ Notes: * GDS does not support library- or structure-level annotations * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable