[gdsii_arrow] remove non-raw-mode arrow option; fix gzip wrapper
This commit is contained in:
parent
fd01e369e1
commit
f3a60da30b
3 changed files with 158 additions and 81 deletions
|
|
@ -27,7 +27,6 @@ from typing import IO, cast, Any
|
|||
from collections.abc import Iterable, Mapping, Callable
|
||||
from importlib.machinery import EXTENSION_SUFFIXES
|
||||
import importlib.util
|
||||
import io
|
||||
import mmap
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -35,11 +34,11 @@ import pathlib
|
|||
import gzip
|
||||
import string
|
||||
import sys
|
||||
import tempfile
|
||||
from pprint import pformat
|
||||
|
||||
import numpy
|
||||
from numpy.typing import ArrayLike, NDArray
|
||||
from numpy.testing import assert_equal
|
||||
import pyarrow
|
||||
from pyarrow.cffi import ffi
|
||||
|
||||
|
|
@ -70,8 +69,6 @@ ffi.cdef(
|
|||
|
||||
clib: Any | None = None
|
||||
|
||||
ZERO_OFFSET = numpy.zeros(2)
|
||||
|
||||
|
||||
path_cap_map = {
|
||||
0: Path.Cap.Flush,
|
||||
|
|
@ -195,13 +192,22 @@ def _read_annotations(
|
|||
|
||||
def _read_to_arrow(
|
||||
filename: str | pathlib.Path,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> pyarrow.Array:
|
||||
path = pathlib.Path(filename).expanduser().resolve()
|
||||
ptr_array = ffi.new('struct ArrowArray[]', 1)
|
||||
ptr_schema = ffi.new('struct ArrowSchema[]', 1)
|
||||
_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema)
|
||||
if is_gzipped(path):
|
||||
with gzip.open(path, mode='rb') as src:
|
||||
data = src.read()
|
||||
with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream:
|
||||
tmp_stream.write(data)
|
||||
tmp_name = tmp_stream.name
|
||||
try:
|
||||
_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema)
|
||||
finally:
|
||||
pathlib.Path(tmp_name).unlink(missing_ok=True)
|
||||
else:
|
||||
_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema)
|
||||
return _import_arrow_array(ptr_array, ptr_schema)
|
||||
|
||||
|
||||
|
|
@ -236,18 +242,14 @@ def _read_selected_cells_to_arrow(
|
|||
|
||||
def readfile(
|
||||
filename: str | pathlib.Path,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> tuple[Library, dict[str, Any]]:
|
||||
"""
|
||||
Wrapper for `read()` that takes a filename or path instead of a stream.
|
||||
Read a GDSII file from a path into `masque.Library` / `Pattern` objects.
|
||||
|
||||
Will automatically decompress gzipped files.
|
||||
|
||||
Args:
|
||||
filename: Filename to save to.
|
||||
*args: passed to `read()`
|
||||
**kwargs: passed to `read()`
|
||||
filename: Filename to read.
|
||||
|
||||
For callers that can consume Arrow directly, prefer `readfile_arrow()`
|
||||
to skip Python `Pattern` construction entirely.
|
||||
|
|
@ -284,7 +286,6 @@ def readfile_arrow(
|
|||
|
||||
def read_arrow(
|
||||
libarr: pyarrow.Array,
|
||||
raw_mode: bool = True,
|
||||
) -> tuple[Library, dict[str, Any]]:
|
||||
"""
|
||||
# TODO check GDSII file for cycles!
|
||||
|
|
@ -299,8 +300,7 @@ def read_arrow(
|
|||
per database unit
|
||||
|
||||
Args:
|
||||
stream: Stream to read from.
|
||||
raw_mode: If True, constructs shapes in raw mode, bypassing most data validation, Default True.
|
||||
libarr: Arrow library payload as returned by `readfile_arrow()`.
|
||||
|
||||
Returns:
|
||||
- dict of pattern_name:Patterns generated from GDSII structures
|
||||
|
|
@ -436,7 +436,6 @@ def read_arrow(
|
|||
global_args = dict(
|
||||
cell_names = cell_names,
|
||||
layer_tups = layer_tups,
|
||||
raw_mode = raw_mode,
|
||||
)
|
||||
|
||||
mlib = Library()
|
||||
|
|
@ -488,7 +487,6 @@ def _srefs_to_mrefs(
|
|||
elem_invert_y = elem['invert_y'][start:stop]
|
||||
elem_angle_rad = elem['angle_rad'][start:stop]
|
||||
elem_scale = elem['scale'][start:stop]
|
||||
raw_mode = global_args['raw_mode']
|
||||
|
||||
_append_plain_refs_sorted(
|
||||
pat=pat,
|
||||
|
|
@ -498,7 +496,6 @@ def _srefs_to_mrefs(
|
|||
elem_invert_y=elem_invert_y,
|
||||
elem_angle_rad=elem_angle_rad,
|
||||
elem_scale=elem_scale,
|
||||
raw_mode=raw_mode,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -511,14 +508,11 @@ def _append_plain_refs_sorted(
|
|||
elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
|
||||
elem_angle_rad: NDArray[numpy.floating[Any]],
|
||||
elem_scale: NDArray[numpy.floating[Any]],
|
||||
raw_mode: bool,
|
||||
) -> None:
|
||||
elem_count = len(elem_targets)
|
||||
if elem_count == 0:
|
||||
return
|
||||
|
||||
make_ref = Ref._from_raw if raw_mode else Ref
|
||||
|
||||
target_start = 0
|
||||
while target_start < elem_count:
|
||||
target_id = int(elem_targets[target_start])
|
||||
|
|
@ -528,7 +522,7 @@ def _append_plain_refs_sorted(
|
|||
|
||||
append_refs = pat.refs[cell_names[target_id]].extend
|
||||
append_refs(
|
||||
make_ref(
|
||||
Ref._from_raw(
|
||||
offset=elem_xy[ee],
|
||||
mirrored=elem_invert_y[ee],
|
||||
rotation=elem_angle_rad[ee],
|
||||
|
|
@ -564,10 +558,6 @@ def _arefs_to_mrefs(
|
|||
elem_xy0 = elem['xy0'][start:stop]
|
||||
elem_xy1 = elem['xy1'][start:stop]
|
||||
elem_counts = elem['counts'][start:stop]
|
||||
raw_mode = global_args['raw_mode']
|
||||
|
||||
make_ref = Ref._from_raw if raw_mode else Ref
|
||||
make_grid = Grid._from_raw if raw_mode else Grid
|
||||
|
||||
if len(elem_targets) == 0:
|
||||
return
|
||||
|
|
@ -581,12 +571,12 @@ def _arefs_to_mrefs(
|
|||
append_ref = pat.refs[cell_names[target_id]].append
|
||||
assert append_ref is not None
|
||||
a_count, b_count = elem_counts[ee]
|
||||
append_ref(make_ref(
|
||||
append_ref(Ref._from_raw(
|
||||
offset=elem_xy[ee],
|
||||
mirrored=elem_invert_y[ee],
|
||||
rotation=elem_angle_rad[ee],
|
||||
scale=elem_scale[ee],
|
||||
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
|
||||
repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
|
||||
annotations=None,
|
||||
))
|
||||
|
||||
|
|
@ -613,13 +603,10 @@ def _sref_props_to_mrefs(
|
|||
elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
|
||||
elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
|
||||
elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
|
||||
raw_mode = global_args['raw_mode']
|
||||
|
||||
make_ref = Ref._from_raw if raw_mode else Ref
|
||||
|
||||
for ee in range(elem_count):
|
||||
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
|
||||
ref = make_ref(
|
||||
ref = Ref._from_raw(
|
||||
offset=elem_xy[ee],
|
||||
mirrored=elem_invert_y[ee],
|
||||
rotation=elem_angle_rad[ee],
|
||||
|
|
@ -655,20 +642,16 @@ def _aref_props_to_mrefs(
|
|||
elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]]
|
||||
elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]]
|
||||
elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]]
|
||||
raw_mode = global_args['raw_mode']
|
||||
|
||||
make_ref = Ref._from_raw if raw_mode else Ref
|
||||
make_grid = Grid._from_raw if raw_mode else Grid
|
||||
|
||||
for ee in range(elem_count):
|
||||
a_count, b_count = elem_counts[ee]
|
||||
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
|
||||
ref = make_ref(
|
||||
ref = Ref._from_raw(
|
||||
offset=elem_xy[ee],
|
||||
mirrored=elem_invert_y[ee],
|
||||
rotation=elem_angle_rad[ee],
|
||||
scale=elem_scale[ee],
|
||||
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
|
||||
repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
|
||||
annotations=annotations,
|
||||
)
|
||||
pat.refs[cell_names[int(elem_targets[ee])]].append(ref)
|
||||
|
|
@ -693,7 +676,6 @@ def _texts_to_labels(
|
|||
elem_xy = xy[elem_slc][:elem_count]
|
||||
elem_layer_inds = layer_inds[elem_slc][:elem_count]
|
||||
elem_strings = elem['string'][elem_slc][:elem_count]
|
||||
raw_mode = global_args['raw_mode']
|
||||
|
||||
for ee in range(elem_count):
|
||||
layer = layer_tups[int(elem_layer_inds[ee])]
|
||||
|
|
@ -701,10 +683,7 @@ def _texts_to_labels(
|
|||
string = elem_strings[ee]
|
||||
|
||||
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
|
||||
if raw_mode:
|
||||
mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations)
|
||||
else:
|
||||
mlabel = Label(string=string, offset=offset, annotations=annotations)
|
||||
mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations)
|
||||
pat.labels[layer].append(mlabel)
|
||||
|
||||
|
||||
|
|
@ -730,7 +709,6 @@ def _gpaths_to_mpaths(
|
|||
elem_path_types = elem['path_type'][elem_slc][:elem_count]
|
||||
elem_extensions = elem['extensions'][elem_slc][:elem_count]
|
||||
|
||||
raw_mode = global_args['raw_mode']
|
||||
for ee in range(elem_count):
|
||||
layer = layer_tups[int(elem_layer_inds[ee])]
|
||||
vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]]
|
||||
|
|
@ -743,23 +721,13 @@ def _gpaths_to_mpaths(
|
|||
cap_extensions = None
|
||||
|
||||
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
|
||||
if raw_mode:
|
||||
path = Path._from_raw(
|
||||
vertices=vertices,
|
||||
width=width,
|
||||
cap=cap,
|
||||
cap_extensions=cap_extensions,
|
||||
annotations=annotations,
|
||||
)
|
||||
else:
|
||||
path = Path(
|
||||
vertices=vertices,
|
||||
width=width,
|
||||
cap=cap,
|
||||
cap_extensions=cap_extensions,
|
||||
offset=ZERO_OFFSET,
|
||||
annotations=annotations,
|
||||
)
|
||||
path = Path._from_raw(
|
||||
vertices=vertices,
|
||||
width=width,
|
||||
cap=cap,
|
||||
cap_extensions=cap_extensions,
|
||||
annotations=annotations,
|
||||
)
|
||||
pat.shapes[layer].append(path)
|
||||
|
||||
|
||||
|
|
@ -786,23 +754,16 @@ def _boundary_batches_to_polygons(
|
|||
elem_poly_off = poly_off[elem_slc]
|
||||
elem_layer_inds = layer_inds[elem_slc][:batch_count]
|
||||
|
||||
raw_mode = global_args['raw_mode']
|
||||
for bb in range(batch_count):
|
||||
layer = layer_tups[int(elem_layer_inds[bb])]
|
||||
vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
|
||||
vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]]
|
||||
|
||||
if vertex_offsets.size == 1:
|
||||
if raw_mode:
|
||||
poly = Polygon._from_raw(vertices=vertices, annotations=None)
|
||||
else:
|
||||
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None)
|
||||
poly = Polygon._from_raw(vertices=vertices, annotations=None)
|
||||
pat.shapes[layer].append(poly)
|
||||
else:
|
||||
if raw_mode:
|
||||
polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
|
||||
else:
|
||||
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None)
|
||||
polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
|
||||
pat.shapes[layer].append(polys)
|
||||
|
||||
|
||||
|
|
@ -826,14 +787,10 @@ def _rect_batches_to_rectcollections(
|
|||
elem_rect_off = rect_off[elem_slc]
|
||||
elem_layer_inds = layer_inds[elem_slc][:batch_count]
|
||||
|
||||
raw_mode = global_args['raw_mode']
|
||||
for bb in range(batch_count):
|
||||
layer = layer_tups[int(elem_layer_inds[bb])]
|
||||
rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]]
|
||||
if raw_mode:
|
||||
rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
|
||||
else:
|
||||
rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None)
|
||||
rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
|
||||
pat.shapes[layer].append(rect_collection)
|
||||
|
||||
|
||||
|
|
@ -860,15 +817,11 @@ def _boundary_props_to_polygons(
|
|||
prop_offs = elem['prop_off'][elem_slc]
|
||||
elem_layer_inds = layer_inds[elem_slc][:elem_count]
|
||||
|
||||
raw_mode = global_args['raw_mode']
|
||||
for ee in range(elem_count):
|
||||
layer = layer_tups[int(elem_layer_inds[ee])]
|
||||
vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
|
||||
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
|
||||
if raw_mode:
|
||||
poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
|
||||
else:
|
||||
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations)
|
||||
poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
|
||||
pat.shapes[layer].append(poly)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue