masque/masque/file/gdsii_arrow.py

552 lines
20 KiB
Python
Raw Normal View History

2025-04-13 21:47:54 -07:00
# ruff: noqa: ARG001, F401
"""
GDSII file format readers and writers using the `TODO` library.
Note that GDSII references follow the same convention as `masque`,
with this order of operations:
1. Mirroring
2. Rotation
3. Scaling
4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets)
Scaling, rotation, and mirroring apply to individual instances, not grid
vectors or offsets.
Notes:
* absolute positioning is not supported
* PLEX is not supported
* ELFLAGS are not supported
* GDS does not support library- or structure-level annotations
* GDS creation/modification/access times are set to 1900-01-01 for reproducibility.
* Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01)
TODO writing
TODO warn on boxes, nodes
"""
from typing import IO, cast, Any
from collections.abc import Iterable, Mapping, Callable
2026-04-02 13:21:59 -07:00
from importlib.machinery import EXTENSION_SUFFIXES
import importlib.util
2025-04-13 21:47:54 -07:00
import io
import mmap
import logging
2026-04-02 13:21:59 -07:00
import os
2025-04-13 21:47:54 -07:00
import pathlib
import gzip
import string
2026-04-02 13:21:59 -07:00
import sys
2025-04-13 21:47:54 -07:00
from pprint import pformat
import numpy
from numpy.typing import ArrayLike, NDArray
from numpy.testing import assert_equal
import pyarrow
from pyarrow.cffi import ffi
from .utils import is_gzipped, tmpfile
from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape
from ..shapes import Polygon, Path, PolyCollection
from ..repetition import Grid
from ..utils import layer_t, annotations_t
from ..library import LazyLibrary, Library, ILibrary, ILibraryView
logger = logging.getLogger(__name__)
ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchema* schema);')
2026-04-02 13:21:59 -07:00
clib: Any | None = None
2025-04-13 21:47:54 -07:00
path_cap_map = {
0: Path.Cap.Flush,
1: Path.Cap.Circle,
2: Path.Cap.Square,
4: Path.Cap.SquareCustom,
}
def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]:
return numpy.rint(val).astype(numpy.int32)
2026-04-02 13:21:59 -07:00
def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]:
layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16)
dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16)
return numpy.stack((layer, dtype), axis=-1)
def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]:
a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64)
b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64)
return numpy.stack((a_count, b_count), axis=-1)
def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]:
xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32)
yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32)
return numpy.stack((xx, yy), axis=-1)
def _local_library_filename() -> str:
if sys.platform.startswith('linux'):
return 'libklamath_rs_ext.so'
if sys.platform == 'darwin':
return 'libklamath_rs_ext.dylib'
if sys.platform == 'win32':
return 'klamath_rs_ext.dll'
raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}')
def _installed_library_candidates() -> list[pathlib.Path]:
candidates: list[pathlib.Path] = []
try:
spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext')
except ModuleNotFoundError:
spec = None
if spec is not None and spec.origin is not None:
candidates.append(pathlib.Path(spec.origin))
try:
pkg_spec = importlib.util.find_spec('klamath_rs_ext')
except ModuleNotFoundError:
pkg_spec = None
if pkg_spec is not None and pkg_spec.submodule_search_locations is not None:
for location in pkg_spec.submodule_search_locations:
pkg_dir = pathlib.Path(location)
for suffix in EXTENSION_SUFFIXES:
candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}')))
return candidates
def _repo_library_candidates() -> list[pathlib.Path]:
repo_root = pathlib.Path(__file__).resolve().parents[2]
library_name = _local_library_filename()
return [
repo_root / 'klamath-rs' / 'target' / 'release' / library_name,
repo_root / 'klamath-rs' / 'target' / 'debug' / library_name,
]
def find_klamath_rs_library() -> pathlib.Path | None:
env_path = os.environ.get('KLAMATH_RS_EXT_LIB')
if env_path:
candidate = pathlib.Path(env_path).expanduser()
if candidate.exists():
return candidate.resolve()
seen: set[pathlib.Path] = set()
for candidate in _installed_library_candidates() + _repo_library_candidates():
resolved = candidate.expanduser()
if resolved in seen:
continue
seen.add(resolved)
if resolved.exists():
return resolved.resolve()
return None
def is_available() -> bool:
return find_klamath_rs_library() is not None
def _get_clib() -> Any:
global clib
if clib is None:
lib_path = find_klamath_rs_library()
if lib_path is None:
raise ImportError(
'Could not locate klamath_rs_ext shared library. '
'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` '
'or set KLAMATH_RS_EXT_LIB to the built library path.'
)
clib = ffi.dlopen(str(lib_path))
return clib
def _read_annotations(
prop_offs: NDArray[numpy.integer[Any]],
prop_key: NDArray[numpy.integer[Any]],
prop_val: list[str],
ee: int,
) -> annotations_t:
prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1]
if prop_ii >= prop_ff:
return None
return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)}
2025-04-13 21:47:54 -07:00
def _read_to_arrow(
filename: str | pathlib.Path,
*args,
**kwargs,
) -> pyarrow.Array:
2026-04-02 13:21:59 -07:00
path = pathlib.Path(filename).expanduser().resolve()
2025-04-13 21:47:54 -07:00
ptr_array = ffi.new('struct ArrowArray[]', 1)
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
2026-04-02 13:21:59 -07:00
_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema)
2025-04-13 21:47:54 -07:00
iptr_schema = int(ffi.cast('uintptr_t', ptr_schema))
iptr_array = int(ffi.cast('uintptr_t', ptr_array))
arrow_arr = pyarrow.Array._import_from_c(iptr_array, iptr_schema)
return arrow_arr
def readfile(
filename: str | pathlib.Path,
*args,
**kwargs,
) -> tuple[Library, dict[str, Any]]:
"""
Wrapper for `read()` that takes a filename or path instead of a stream.
Will automatically decompress gzipped files.
Args:
filename: Filename to save to.
*args: passed to `read()`
**kwargs: passed to `read()`
"""
arrow_arr = _read_to_arrow(filename)
assert len(arrow_arr) == 1
results = read_arrow(arrow_arr[0])
return results
def read_arrow(
libarr: pyarrow.Array,
raw_mode: bool = True,
) -> tuple[Library, dict[str, Any]]:
"""
# TODO check GDSII file for cycles!
Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are
translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs
are translated into Ref objects.
Additional library info is returned in a dict, containing:
'name': name of the library
'meters_per_unit': number of meters per database unit (all values are in database units)
'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns)
per database unit
Args:
stream: Stream to read from.
raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True.
Returns:
- dict of pattern_name:Patterns generated from GDSII structures
- dict of GDSII library info
"""
library_info = _read_header(libarr)
2026-04-02 13:21:59 -07:00
layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy())
2025-04-13 21:47:54 -07:00
layer_tups = [tuple(pair) for pair in layer_names_np]
cell_ids = libarr['cells'].values.field('id').to_numpy()
cell_names = libarr['cell_names'].as_py()
def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]:
el = libarr['cells'].values.field(geom_type)
elem = dict(
offsets = el.offsets.to_numpy(),
xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)),
xy_off = el.values.field('xy').offsets.to_numpy() // 2,
layer_inds = el.values.field('layer').to_numpy(),
prop_off = el.values.field('properties').offsets.to_numpy(),
prop_key = el.values.field('properties').values.field('key').to_numpy(),
prop_val = el.values.field('properties').values.field('value').to_pylist(),
)
return elem
rf = libarr['cells'].values.field('refs')
refs = dict(
offsets = rf.offsets.to_numpy(),
targets = rf.values.field('target').to_numpy(),
2026-04-02 13:21:59 -07:00
xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()),
2025-04-13 21:47:54 -07:00
invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False),
2026-04-02 13:21:59 -07:00
angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()),
2025-04-13 21:47:54 -07:00
scale = rf.values.field('mag').fill_null(1).to_numpy(),
rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False),
2026-04-02 13:21:59 -07:00
rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()),
rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()),
rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()),
2025-04-13 21:47:54 -07:00
prop_off = rf.values.field('properties').offsets.to_numpy(),
prop_key = rf.values.field('properties').values.field('key').to_numpy(),
prop_val = rf.values.field('properties').values.field('value').to_pylist(),
)
txt = libarr['cells'].values.field('texts')
texts = dict(
offsets = txt.offsets.to_numpy(),
layer_inds = txt.values.field('layer').to_numpy(),
2026-04-02 13:21:59 -07:00
xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()),
2025-04-13 21:47:54 -07:00
string = txt.values.field('string').to_pylist(),
prop_off = txt.values.field('properties').offsets.to_numpy(),
prop_key = txt.values.field('properties').values.field('key').to_numpy(),
prop_val = txt.values.field('properties').values.field('value').to_pylist(),
)
elements = dict(
boundaries = get_geom(libarr, 'boundaries'),
paths = get_geom(libarr, 'paths'),
boxes = get_geom(libarr, 'boxes'),
nodes = get_geom(libarr, 'nodes'),
texts = texts,
refs = refs,
)
paths = libarr['cells'].values.field('paths')
elements['paths'].update(dict(
width = paths.values.field('width').fill_null(0).to_numpy(),
path_type = paths.values.field('path_type').fill_null(0).to_numpy(),
extensions = numpy.stack((
paths.values.field('extension_start').fill_null(0).to_numpy(),
paths.values.field('extension_end').fill_null(0).to_numpy(),
), axis=-1),
))
global_args = dict(
cell_names = cell_names,
layer_tups = layer_tups,
raw_mode = raw_mode,
)
mlib = Library()
for cc in range(len(libarr['cells'])):
name = cell_names[cell_ids[cc]]
pat = Pattern()
_boundaries_to_polygons(pat, global_args, elements['boundaries'], cc)
_gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
_grefs_to_mrefs(pat, global_args, elements['refs'], cc)
_texts_to_labels(pat, global_args, elements['texts'], cc)
mlib[name] = pat
return mlib, library_info
def _read_header(libarr: pyarrow.Array) -> dict[str, Any]:
"""
Read the file header and create the library_info dict.
"""
library_info = dict(
2026-04-02 13:21:59 -07:00
name = libarr['lib_name'].as_py(),
meters_per_unit = libarr['meters_per_db_unit'].as_py(),
logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(),
2025-04-13 21:47:54 -07:00
)
return library_info
def _grefs_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets'] # which elements belong to each cell
xy = elem['xy']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
targets = elem['targets']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
2026-04-02 13:21:59 -07:00
elem_targets = targets[elem_slc][:elem_count]
elem_xy = xy[elem_slc][:elem_count]
2025-04-13 21:47:54 -07:00
elem_invert_y = elem['invert_y'][elem_slc][:elem_count]
elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count]
elem_scale = elem['scale'][elem_slc][:elem_count]
elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count]
elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count]
elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count]
rep_valid = elem['rep_valid'][elem_slc][:elem_count]
for ee in range(elem_count):
2026-04-02 13:21:59 -07:00
target = cell_names[elem_targets[ee]]
offset = elem_xy[ee]
2025-04-13 21:47:54 -07:00
mirr = elem_invert_y[ee]
rot = elem_angle_rad[ee]
mag = elem_scale[ee]
rep: None | Grid = None
if rep_valid[ee]:
a_vector = elem_rep_xy0[ee]
b_vector = elem_rep_xy1[ee]
a_count, b_count = elem_rep_counts[ee]
rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
2026-04-02 13:21:59 -07:00
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
2025-04-13 21:47:54 -07:00
ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
pat.refs[target].append(ref)
def _texts_to_labels(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets'] # which elements belong to each cell
xy = elem['xy']
layer_tups = global_args['layer_tups']
layer_inds = elem['layer_inds']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
2026-04-02 13:21:59 -07:00
elem_xy = xy[elem_slc][:elem_count]
2025-04-13 21:47:54 -07:00
elem_layer_inds = layer_inds[elem_slc][:elem_count]
elem_strings = elem['string'][elem_slc][:elem_count]
for ee in range(elem_count):
layer = layer_tups[elem_layer_inds[ee]]
2026-04-02 13:21:59 -07:00
offset = elem_xy[ee]
2025-04-13 21:47:54 -07:00
string = elem_strings[ee]
2026-04-02 13:21:59 -07:00
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
2025-04-13 21:47:54 -07:00
mlabel = Label(string=string, offset=offset, annotations=annotations)
pat.labels[layer].append(mlabel)
def _gpaths_to_mpaths(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets'] # which elements belong to each cell
xy_val = elem['xy_arr']
layer_tups = global_args['layer_tups']
layer_inds = elem['layer_inds']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
elem_layer_inds = layer_inds[elem_slc][:elem_count]
elem_widths = elem['width'][elem_slc][:elem_count]
elem_path_types = elem['path_type'][elem_slc][:elem_count]
elem_extensions = elem['extensions'][elem_slc][:elem_count]
zeros = numpy.zeros((elem_count, 2))
raw_mode = global_args['raw_mode']
for ee in range(elem_count):
layer = layer_tups[elem_layer_inds[ee]]
vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]]
width = elem_widths[ee]
cap_int = elem_path_types[ee]
cap = path_cap_map[cap_int]
if cap_int == 4:
cap_extensions = elem_extensions[ee]
else:
cap_extensions = None
2026-04-02 13:21:59 -07:00
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
2025-04-13 21:47:54 -07:00
path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode,
width=width, cap=cap,cap_extensions=cap_extensions)
pat.shapes[layer].append(path)
def _boundaries_to_polygons(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets'] # which elements belong to each cell
xy_val = elem['xy_arr']
layer_inds = elem['layer_inds']
layer_tups = global_args['layer_tups']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element
xy_counts = xy_offs[1:] - xy_offs[:-1]
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
prop_counts = prop_offs[1:] - prop_offs[:-1]
elem_layer_inds = layer_inds[elem_slc][:elem_count]
order = numpy.argsort(elem_layer_inds, stable=True)
unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True)
zeros = numpy.zeros((elem_count, 2))
raw_mode = global_args['raw_mode']
for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True):
ee_inds = order[ff:ff + nn]
layer = layer_tups[layer_ind]
propless_mask = prop_counts[ee_inds] == 0
poly_count_on_layer = propless_mask.sum()
if poly_count_on_layer == 1:
propless_mask[:] = 0 # Never make a 1-element collection
elif poly_count_on_layer > 1:
propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point
vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64)
vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts]))
for ii, ee in enumerate(ee_inds[propless_mask]):
vo = vertex_offsets[ii]
vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1]
polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee])
pat.shapes[layer].append(polys)
# Handle single polygons
for ee in ee_inds[~propless_mask]:
layer = layer_tups[elem_layer_inds[ee]]
vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point
2026-04-02 13:21:59 -07:00
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
2025-04-13 21:47:54 -07:00
poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode)
pat.shapes[layer].append(poly)
#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t:
# return {prop['key'].as_py(): prop['value'].as_py() for prop in properties}
def check_valid_names(
names: Iterable[str],
max_length: int = 32,
) -> None:
"""
Check all provided names to see if they're valid GDSII cell names.
Args:
names: Collection of names to check
max_length: Max allowed length
"""
allowed_chars = set(string.ascii_letters + string.digits + '_?$')
bad_chars = [
name for name in names
if not set(name).issubset(allowed_chars)
]
bad_lengths = [
name for name in names
if len(name) > max_length
]
if bad_chars:
logger.error('Names contain invalid characters:\n' + pformat(bad_chars))
if bad_lengths:
logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars))
if bad_chars or bad_lengths:
raise LibraryError('Library contains invalid names, see log above')