From e89d912ce8c71211904fa4500172cc48524a3859 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Mon, 21 Apr 2025 20:26:34 -0700 Subject: [PATCH 1/4] allow annotations to be None breaking change, but properties are seldom used by anyone afaik --- masque/file/gdsii.py | 11 +++++++++-- masque/file/oasis.py | 2 ++ masque/pattern.py | 15 +++++++++------ masque/utils/types.py | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/masque/file/gdsii.py b/masque/file/gdsii.py index c323ecf..10f5a9a 100644 --- a/masque/file/gdsii.py +++ b/masque/file/gdsii.py @@ -21,6 +21,7 @@ Notes: """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable +from types import MappingProxyType import io import mmap import logging @@ -52,6 +53,8 @@ path_cap_map = { 4: Path.Cap.SquareCustom, } +RO_EMPTY_DICT: Mapping[int, bytes] = MappingProxyType({}) + def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]: return numpy.rint(val).astype(numpy.int32) @@ -399,11 +402,15 @@ def _mrefs_to_grefs(refs: dict[str | None, list[Ref]]) -> list[klamath.library.R return grefs -def _properties_to_annotations(properties: dict[int, bytes]) -> annotations_t: +def _properties_to_annotations(properties: Mapping[int, bytes]) -> annotations_t: + if not properties: + return None return {str(k): [v.decode()] for k, v in properties.items()} -def _annotations_to_properties(annotations: annotations_t, max_len: int = 126) -> dict[int, bytes]: +def _annotations_to_properties(annotations: annotations_t, max_len: int = 126) -> Mapping[int, bytes]: + if annotations is None: + return RO_EMPTY_DICT cum_len = 0 props = {} for key, vals in annotations.items(): diff --git a/masque/file/oasis.py b/masque/file/oasis.py index e64bb4c..642ad44 100644 --- a/masque/file/oasis.py +++ b/masque/file/oasis.py @@ -671,6 +671,8 @@ def repetition_masq2fata( def annotations_to_properties(annotations: annotations_t) -> list[fatrec.Property]: #TODO determine is_standard based on key? + if annotations is None: + return [] properties = [] for key, values in annotations.items(): vals = [AString(v) if isinstance(v, str) else v diff --git a/masque/pattern.py b/masque/pattern.py index 5bf030a..f77c64f 100644 --- a/masque/pattern.py +++ b/masque/pattern.py @@ -332,7 +332,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): )) self.ports = dict(sorted(self.ports.items())) - self.annotations = dict(sorted(self.annotations.items())) + self.annotations = dict(sorted(self.annotations.items())) if self.annotations is not None else None return self @@ -354,10 +354,13 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): for layer, lseq in other_pattern.labels.items(): self.labels[layer].extend(lseq) - annotation_conflicts = set(self.annotations.keys()) & set(other_pattern.annotations.keys()) - if annotation_conflicts: - raise PatternError(f'Annotation keys overlap: {annotation_conflicts}') - self.annotations.update(other_pattern.annotations) + if other_pattern.annotations is not None: + if self.annotations is None: + self.annotations = {} + annotation_conflicts = set(self.annotations.keys()) & set(other_pattern.annotations.keys()) + if annotation_conflicts: + raise PatternError(f'Annotation keys overlap: {annotation_conflicts}') + self.annotations.update(other_pattern.annotations) port_conflicts = set(self.ports.keys()) & set(other_pattern.ports.keys()) if port_conflicts: @@ -415,7 +418,7 @@ class Pattern(PortList, AnnotatableImpl, Mirrorable): elif default_keep: pat.refs = copy.copy(self.refs) - if annotations is not None: + if annotations is not None and self.annotations is not None: pat.annotations = {k: v for k, v in self.annotations.items() if annotations(k, v)} elif default_keep: pat.annotations = copy.copy(self.annotations) diff --git a/masque/utils/types.py b/masque/utils/types.py index 4060ac4..c06b7b4 100644 --- a/masque/utils/types.py +++ b/masque/utils/types.py @@ -5,7 +5,7 @@ from typing import Protocol layer_t = int | tuple[int, int] | str -annotations_t = dict[str, list[int | float | str]] +annotations_t = dict[str, list[int | float | str]] | None class SupportsBool(Protocol): From 79f20881808626adb64e7458d964abcda52e266e Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:19:59 -0700 Subject: [PATCH 2/4] [utils.curves] ignore re-import of trapeziod --- masque/utils/curves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/masque/utils/curves.py b/masque/utils/curves.py index 1e41c35..8b3fcc4 100644 --- a/masque/utils/curves.py +++ b/masque/utils/curves.py @@ -5,7 +5,7 @@ from numpy import pi try: from numpy import trapezoid except ImportError: - from numpy import trapz as trapezoid + from numpy import trapz as trapezoid # type:ignore def bezier( From 4e40e3f82917f8b45d29ff3f4e40f06bfe628ed6 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:20:46 -0700 Subject: [PATCH 3/4] [gdsii_arrow] use direct access for all element types --- masque/file/gdsii_arrow.py | 279 +++++++++++++++++++++++-------------- 1 file changed, 176 insertions(+), 103 deletions(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 0c356c4..8051726 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -134,35 +134,76 @@ def read_arrow( cell_ids = libarr['cells'].values.field('id').to_numpy() cell_names = libarr['cell_names'].as_py() - bnd = libarr['cells'].values.field('boundaries') - boundary = dict( - offsets = bnd.offsets.to_numpy(), - xy_arr = bnd.values.field('xy').values.to_numpy().reshape((-1, 2)), - xy_off = bnd.values.field('xy').offsets.to_numpy() // 2, - layer_tups = layer_tups, - layer_inds = bnd.values.field('layer').to_numpy(), - prop_off = bnd.values.field('properties').offsets.to_numpy(), - prop_key = bnd.values.field('properties').values.field('key').to_numpy(), - prop_val = bnd.values.field('properties').values.field('value').to_pylist(), + def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]: + el = libarr['cells'].values.field(geom_type) + elem = dict( + offsets = el.offsets.to_numpy(), + xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)), + xy_off = el.values.field('xy').offsets.to_numpy() // 2, + layer_inds = el.values.field('layer').to_numpy(), + prop_off = el.values.field('properties').offsets.to_numpy(), + prop_key = el.values.field('properties').values.field('key').to_numpy(), + prop_val = el.values.field('properties').values.field('value').to_pylist(), + ) + return elem + + rf = libarr['cells'].values.field('refs') + refs = dict( + offsets = rf.offsets.to_numpy(), + targets = rf.values.field('target').to_numpy(), + xy = rf.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False), + angle_rad = numpy.rad2deg(rf.values.field('angle_deg').fill_null(0).to_numpy()), + scale = rf.values.field('mag').fill_null(1).to_numpy(), + rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False), + rep_xy0 = rf.values.field('repetition').field('xy0').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_xy1 = rf.values.field('repetition').field('xy1').fill_null(0).to_numpy().view('i4').reshape((-1, 2)), + rep_counts = rf.values.field('repetition').field('counts').fill_null(0).to_numpy().view('i2').reshape((-1, 2)), + prop_off = rf.values.field('properties').offsets.to_numpy(), + prop_key = rf.values.field('properties').values.field('key').to_numpy(), + prop_val = rf.values.field('properties').values.field('value').to_pylist(), ) - pth = libarr['cells'].values.field('boundaries') - path = dict( - offsets = pth.offsets.to_numpy(), - xy_arr = pth.values.field('xy').values.to_numpy().reshape((-1, 2)), - xy_off = pth.values.field('xy').offsets.to_numpy() // 2, - layer_tups = layer_tups, - layer_inds = pth.values.field('layer').to_numpy(), - prop_off = pth.values.field('properties').offsets.to_numpy(), - prop_key = pth.values.field('properties').values.field('key').to_numpy(), - prop_val = pth.values.field('properties').values.field('value').to_pylist(), + txt = libarr['cells'].values.field('texts') + texts = dict( + offsets = txt.offsets.to_numpy(), + layer_inds = txt.values.field('layer').to_numpy(), + xy = txt.values.field('xy').to_numpy().view('i4').reshape((-1, 2)), + string = txt.values.field('string').to_pylist(), + prop_off = txt.values.field('properties').offsets.to_numpy(), + prop_key = txt.values.field('properties').values.field('key').to_numpy(), + prop_val = txt.values.field('properties').values.field('value').to_pylist(), ) + elements = dict( + boundaries = get_geom(libarr, 'boundaries'), + paths = get_geom(libarr, 'paths'), + boxes = get_geom(libarr, 'boxes'), + nodes = get_geom(libarr, 'nodes'), + texts = texts, + refs = refs, + ) + + paths = libarr['cells'].values.field('paths') + elements['paths'].update(dict( + width = paths.values.field('width').to_numpy(), + path_type = paths.values.field('path_type').to_numpy(), + extensions = numpy.stack(( + paths.values.field('extension_start').to_numpy(zero_copy_only=False), + paths.values.field('extension_end').to_numpy(zero_copy_only=False), + ), axis=-1), + )) + + global_args = dict( + cell_names = cell_names, + layer_tups = layer_tups, + raw_mode = raw_mode, + ) mlib = Library() for cc, cell in enumerate(libarr['cells']): name = cell_names[cell_ids[cc]] - pat = read_cell(cc, cell, libarr['cell_names'], raw_mode=raw_mode, boundary=boundary) + pat = read_cell(cc, cell, libarr['cell_names'], global_args=global_args, elements=elements) mlib[name] = pat return mlib, library_info @@ -184,8 +225,8 @@ def read_cell( cc: int, cellarr: pyarrow.Array, cell_names: pyarrow.Array, - boundary: dict[str, NDArray], - raw_mode: bool = True, + elements: dict[str, Any], + global_args: dict[str, Any], ) -> Pattern: """ TODO @@ -202,81 +243,96 @@ def read_cell( """ pat = Pattern() - for refarr in cellarr['refs']: - target = cell_names[refarr['target'].as_py()].as_py() - args = dict( - offset = (refarr['x'].as_py(), refarr['y'].as_py()), - ) - if (mirr := refarr['invert_y']).is_valid: - args['mirrored'] = mirr.as_py() - if (rot := refarr['angle_deg']).is_valid: - args['rotation'] = numpy.deg2rad(rot.as_py()) - if (mag := refarr['mag']).is_valid: - args['scale'] = mag.as_py() - if (rep := refarr['repetition']).is_valid: - repetition = Grid( - a_vector = (rep['x0'].as_py(), rep['y0'].as_py()), - b_vector = (rep['x1'].as_py(), rep['y1'].as_py()), - a_count = rep['count0'].as_py(), - b_count = rep['count1'].as_py(), - ) - args['repetition'] = repetition - ref = Ref(**args) - pat.refs[target].append(ref) - - _boundaries_to_polygons(pat, cellarr) - - for gpath in cellarr['paths']: - layer = (gpath['layer'].as_py(),) - args = dict( - vertices = gpath['xy'].values.to_numpy().reshape((-1, 2)), - offset = numpy.zeros(2), - raw = raw_mode, - ) - - if (gcap := gpath['path_type']).is_valid: - mcap = path_cap_map[gcap.as_py()] - args['cap'] = mcap - if mcap == Path.Cap.SquareCustom: - extensions = [0, 0] - if (ext0 := gpath['extension_start']).is_valid: - extensions[0] = ext0.as_py() - if (ext1 := gpath['extension_end']).is_valid: - extensions[1] = ext1.as_py() - - args['extensions'] = extensions - - if (width := gpath['width']).is_valid: - args['width'] = width.as_py() - else: - args['width'] = 0 - - if (props := gpath['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - mpath = Path(**args) - pat.shapes[layer].append(mpath) - - for gtext in cellarr['texts']: - layer = (gtext['layer'].as_py(),) - args = dict( - offset = (gtext['x'].as_py(), gtext['y'].as_py()), - string = gtext['string'].as_py(), - ) - - if (props := gtext['properties']).is_valid: - args['annotations'] = _properties_to_annotations(props) - - mlabel = Label(**args) - pat.labels[layer].append(mlabel) + _boundaries_to_polygons(pat, global_args, elements['boundaries'], cc) + _gpaths_to_mpaths(pat, global_args, elements['paths'], cc) + _grefs_to_mrefs(pat, global_args, elements['refs'], cc) + _texts_to_labels(pat, global_args, elements['texts'], cc) return pat -def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: +def _grefs_to_mrefs( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + cell_names = global_args['cell_names'] + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + targets = elem['targets'] + + rep_valid = elem['rep_valid'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + for ee in range(elem_count): + target = cell_names[targets[ee]] + offset = xy[ee] + mirr = elem['invert_y'][ee] + rot = elem['angle_rad'][ee] + mag = elem['scale'][ee] + + rep: None | Grid = None + if rep_valid[ee]: + a_vector = elem['rep_xy0'][ee] + b_vector = elem['rep_xy1'][ee] + a_count, b_count = elem['rep_counts'][ee] + rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count) + + annotations: None | dict[int, str] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + + ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations) + pat.refs[target].append(ref) + + +def _texts_to_labels( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: + elem_off = elem['offsets'] # which elements belong to each cell + xy = elem['xy'] + layer_tups = global_args['layer_tups'] + layer_inds = elem['layer_inds'] + prop_key = elem['prop_key'] + prop_val = elem['prop_val'] + + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem + prop_offs = elem['prop_off'][elem_slc] # which props belong to each element + + for ee in range(elem_count): + layer = layer_tups[layer_inds[ee]] + offset = xy[ee] + string = elem['string'][ee] + + annotations: None | dict[int, str] = None + prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] + if prop_ii < prop_ff: + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} + + mlabel = Label(string=string, offset=offset, annotations=annotations) + pat.labels[layer].append(mlabel) + + +def _gpaths_to_mpaths( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: elem_off = elem['offsets'] # which elements belong to each cell xy_val = elem['xy_arr'] - layer_tups = elem['layer_tups'] + layer_tups = global_args['layer_tups'] layer_inds = elem['layer_inds'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] @@ -287,42 +343,59 @@ def _paths_to_paths(pat: Pattern, paths: dict[str, Any], cc: int) -> None: prop_offs = elem['prop_off'][elem_slc] # which props belong to each element zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] for ee in range(elem_count): + elem_ind = elem_off[cc] + ee layer = layer_tups[layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]] + width = elem['width'][elem_ind] + cap_int = elem['path_type'][elem_ind] + cap = path_cap_map[cap_int] + if cap_int == 4: + cap_extensions = elem['extensions'][elem_ind] + else: + cap_extensions = None + annotations: None | dict[int, str] = None prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] if prop_ii < prop_ff: - ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - args = dict(annotations = ann) + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - path = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode, + width=width, cap=cap,cap_extensions=cap_extensions) pat.shapes[layer].append(path) -def _boundaries_to_polygons(pat: Pattern, elem: dict[str, Any], cc: int) -> None: +def _boundaries_to_polygons( + pat: Pattern, + global_args: dict[str, Any], + elem: dict[str, Any], + cc: int, + ) -> None: elem_off = elem['offsets'] # which elements belong to each cell xy_val = elem['xy_arr'] - layer_tups = elem['layer_tups'] + layer_tups = global_args['layer_tups'] layer_inds = elem['layer_inds'] prop_key = elem['prop_key'] prop_val = elem['prop_val'] - elem_slc = slice(elem_off[cc], elem_off[cc + 1] + 1) + elem_count = elem_off[cc + 1] - elem_off[cc] + elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element prop_offs = elem['prop_off'][elem_slc] # which props belong to each element - zeros = numpy.zeros((len(xy_offs) - 1, 2)) - for ee in range(len(xy_offs) - 1): + zeros = numpy.zeros((elem_count, 2)) + raw_mode = global_args['raw_mode'] + for ee in range(elem_count): layer = layer_tups[layer_inds[ee]] vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point + annotations: None | dict[int, str] = None prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1] if prop_ii < prop_ff: - ann = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - args = dict(annotations = ann) + annotations = {prop_key[off]: prop_val[off] for off in range(prop_ii, prop_ff)} - poly = Polygon(vertices=vertices, offset=zeros[ee], raw=raw_mode) + poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode) pat.shapes[layer].append(poly) From 7336545f0702ea8b428c99b847afa0c970000916 Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Tue, 22 Apr 2025 20:22:01 -0700 Subject: [PATCH 4/4] [gdsii_arrow] add some TODO notes --- masque/file/gdsii_arrow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/masque/file/gdsii_arrow.py b/masque/file/gdsii_arrow.py index 8051726..763c438 100644 --- a/masque/file/gdsii_arrow.py +++ b/masque/file/gdsii_arrow.py @@ -1,5 +1,5 @@ """ -GDSII file format readers and writers using the `klamath` library. +GDSII file format readers and writers using the `TODO` library. Note that GDSII references follow the same convention as `masque`, with this order of operations: @@ -18,6 +18,9 @@ Notes: * GDS does not support library- or structure-level annotations * GDS creation/modification/access times are set to 1900-01-01 for reproducibility. * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01) + + TODO writing + TODO warn on boxes, nodes """ from typing import IO, cast, Any from collections.abc import Iterable, Mapping, Callable