jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[gdsii_arrow] more performance work

Browse source
This commit is contained in:
Jan Petykiewicz 2026-04-02 19:42:52 -07:00
commit 85aed4388e
4 changed files with 558 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

356
masque/file/gdsii_arrow.py
View file

@ -45,7 +45,7 @@ from pyarrow.cffi import ffi
from .utils import is_gzipped, tmpfile
from .. import Pattern, Ref, PatternError, LibraryError, Label, Shape
from ..shapes import Polygon, Path, PolyCollection
from ..shapes import Polygon, Path, PolyCollection, RectCollection
from ..repetition import Grid
from ..utils import layer_t, annotations_t
from ..library import LazyLibrary, Library, ILibrary, ILibraryView
@ -276,6 +276,15 @@ def read_arrow(
poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
)
def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]:
batches = libarr['cells'].values.field('rect_batches')
return dict(
offsets = batches.offsets.to_numpy(),
layer_inds = batches.values.field('layer').to_numpy(),
rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
)
def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]:
boundaries = libarr['cells'].values.field('boundary_props')
return dict(
@ -288,22 +297,46 @@ def read_arrow(
prop_val = boundaries.values.field('properties').values.field('value').to_pylist(),
)
rf = libarr['cells'].values.field('refs')
refs = dict(
offsets = rf.offsets.to_numpy(),
targets = rf.values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(rf.values.field('xy').to_numpy()),
invert_y = rf.values.field('invert_y').fill_null(False).to_numpy(zero_copy_only=False),
angle_rad = numpy.deg2rad(rf.values.field('angle_deg').fill_null(0).to_numpy()),
scale = rf.values.field('mag').fill_null(1).to_numpy(),
rep_valid = rf.values.field('repetition').is_valid().to_numpy(zero_copy_only=False),
rep_xy0 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy0').fill_null(0).to_numpy()),
rep_xy1 = _packed_xy_u64_to_pairs(rf.values.field('repetition').field('xy1').fill_null(0).to_numpy()),
rep_counts = _packed_counts_u32_to_pairs(rf.values.field('repetition').field('counts').fill_null(0).to_numpy()),
prop_off = rf.values.field('properties').offsets.to_numpy(),
prop_key = rf.values.field('properties').values.field('key').to_numpy(),
prop_val = rf.values.field('properties').values.field('value').to_pylist(),
)
def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
refs = libarr['cells'].values.field(geom_type)
values = refs.values
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
refs = libarr['cells'].values.field(geom_type)
values = refs.values
elem = dict(
offsets = refs.offsets.to_numpy(),
targets = values.field('target').to_numpy(),
xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
angle_rad = values.field('angle_rad').to_numpy(),
scale = values.field('scale').to_numpy(),
prop_off = values.field('properties').offsets.to_numpy(),
prop_key = values.field('properties').values.field('key').to_numpy(),
prop_val = values.field('properties').values.field('value').to_pylist(),
)
if has_repetition:
elem.update(dict(
xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
))
return elem
txt = libarr['cells'].values.field('texts')
texts = dict(
@ -317,11 +350,15 @@ def read_arrow(
)
elements = dict(
srefs = get_refs(libarr, 'srefs', has_repetition=False),
arefs = get_refs(libarr, 'arefs', has_repetition=True),
sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False),
aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True),
rect_batches = get_rect_batches(libarr),
boundary_batches = get_boundary_batches(libarr),
boundary_props = get_boundary_props(libarr),
paths = get_geom(libarr, 'paths'),
texts = texts,
refs = refs,
)
paths = libarr['cells'].values.field('paths')
@ -344,10 +381,14 @@ def read_arrow(
for cc in range(len(libarr['cells'])):
name = cell_names[cell_ids[cc]]
pat = Pattern()
_rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc)
_boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc)
_boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc)
_gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
_grefs_to_mrefs(pat, global_args, elements['refs'], cc)
_srefs_to_mrefs(pat, global_args, elements['srefs'], cc)
_arefs_to_mrefs(pat, global_args, elements['arefs'], cc)
_sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc)
_aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc)
_texts_to_labels(pat, global_args, elements['texts'], cc)
mlib[name] = pat
@ -366,57 +407,208 @@ def _read_header(libarr: pyarrow.Array) -> dict[str, Any]:
return library_info
def _grefs_to_mrefs(
def _srefs_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets'] # which elements belong to each cell
xy = elem['xy']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
targets = elem['targets']
elem_off = elem['offsets']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
elem_targets = targets[elem_slc][:elem_count]
elem_xy = xy[elem_slc][:elem_count]
elem_invert_y = elem['invert_y'][elem_slc][:elem_count]
elem_angle_rad = elem['angle_rad'][elem_slc][:elem_count]
elem_scale = elem['scale'][elem_slc][:elem_count]
elem_rep_xy0 = elem['rep_xy0'][elem_slc][:elem_count]
elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count]
elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count]
rep_valid = elem['rep_valid'][elem_slc][:elem_count]
if elem_count == 0:
return
start = elem_off[cc]
stop = elem_off[cc + 1]
elem_targets = elem['targets'][start:stop]
elem_xy = elem['xy'][start:stop]
elem_invert_y = elem['invert_y'][start:stop]
elem_angle_rad = elem['angle_rad'][start:stop]
elem_scale = elem['scale'][start:stop]
raw_mode = global_args['raw_mode']
_append_plain_refs_sorted(
pat=pat,
cell_names=cell_names,
elem_targets=elem_targets,
elem_xy=elem_xy,
elem_invert_y=elem_invert_y,
elem_angle_rad=elem_angle_rad,
elem_scale=elem_scale,
raw_mode=raw_mode,
)
def _append_plain_refs_sorted(
*,
pat: Pattern,
cell_names: list[str],
elem_targets: NDArray[numpy.integer[Any]],
elem_xy: NDArray[numpy.integer[Any]],
elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
elem_angle_rad: NDArray[numpy.floating[Any]],
elem_scale: NDArray[numpy.floating[Any]],
raw_mode: bool,
) -> None:
elem_count = len(elem_targets)
if elem_count == 0:
return
make_ref = Ref._from_raw if raw_mode else Ref
target_start = 0
while target_start < elem_count:
target_id = elem_targets[target_start]
target_stop = target_start + 1
while target_stop < elem_count and elem_targets[target_stop] == target_id:
target_stop += 1
append_refs = pat.refs[cell_names[target_id]].extend
append_refs(
make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=None,
annotations=None,
)
for ee in range(target_start, target_stop)
)
target_start = target_stop
def _arefs_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
start = elem_off[cc]
stop = elem_off[cc + 1]
elem_targets = elem['targets'][start:stop]
elem_xy = elem['xy'][start:stop]
elem_invert_y = elem['invert_y'][start:stop]
elem_angle_rad = elem['angle_rad'][start:stop]
elem_scale = elem['scale'][start:stop]
elem_xy0 = elem['xy0'][start:stop]
elem_xy1 = elem['xy1'][start:stop]
elem_counts = elem['counts'][start:stop]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
make_grid = Grid._from_raw if raw_mode else Grid
if len(elem_targets) == 0:
return
target = None
append_ref: Callable[[Ref], Any] | None = None
for ee in range(len(elem_targets)):
if target != elem_targets[ee]:
target = elem_targets[ee]
append_ref = pat.refs[cell_names[target]].append
assert append_ref is not None
a_count, b_count = elem_counts[ee]
append_ref(make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
annotations=None,
))
def _sref_props_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
prop_offs = elem['prop_off'][elem_slc]
elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
for ee in range(elem_count):
target = cell_names[elem_targets[ee]]
offset = elem_xy[ee]
mirr = elem_invert_y[ee]
rot = elem_angle_rad[ee]
mag = elem_scale[ee]
rep: None | Grid = None
if rep_valid[ee]:
a_vector = elem_rep_xy0[ee]
b_vector = elem_rep_xy1[ee]
a_count, b_count = elem_rep_counts[ee]
if raw_mode:
rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
else:
rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
if raw_mode:
ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
else:
ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
pat.refs[target].append(ref)
ref = make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=None,
annotations=annotations,
)
pat.refs[cell_names[elem_targets[ee]]].append(ref)
def _aref_props_to_mrefs(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
cell_names = global_args['cell_names']
elem_off = elem['offsets']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
if elem_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
prop_offs = elem['prop_off'][elem_slc]
elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]]
elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]]
elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]]
raw_mode = global_args['raw_mode']
make_ref = Ref._from_raw if raw_mode else Ref
make_grid = Grid._from_raw if raw_mode else Grid
for ee in range(elem_count):
a_count, b_count = elem_counts[ee]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
ref = make_ref(
offset=elem_xy[ee],
mirrored=elem_invert_y[ee],
rotation=elem_angle_rad[ee],
scale=elem_scale[ee],
repetition=make_grid(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
annotations=annotations,
)
pat.refs[cell_names[elem_targets[ee]]].append(ref)
def _texts_to_labels(
@ -520,16 +712,53 @@ def _boundary_batches_to_polygons(
for bb in range(batch_count):
layer = layer_tups[elem_layer_inds[bb]]
vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp)
vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]]
if vertex_offsets.size == 1:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
if raw_mode:
poly = Polygon._from_raw(vertices=vertices, annotations=None)
else:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(poly)
else:
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
if raw_mode:
polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
else:
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(polys)
def _rect_batches_to_rectcollections(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets']
rect_arr = elem['rect_arr']
rect_off = elem['rect_off']
layer_inds = elem['layer_inds']
layer_tups = global_args['layer_tups']
batch_count = elem_off[cc + 1] - elem_off[cc]
if batch_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1)
elem_rect_off = rect_off[elem_slc]
elem_layer_inds = layer_inds[elem_slc][:batch_count]
raw_mode = global_args['raw_mode']
for bb in range(batch_count):
layer = layer_tups[elem_layer_inds[bb]]
rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]]
if raw_mode:
rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
else:
rect_collection = RectCollection(rects=rects, offset=ZERO_OFFSET, annotations=None, raw=False)
pat.shapes[layer].append(rect_collection)
def _boundary_props_to_polygons(
pat: Pattern,
global_args: dict[str, Any],
@ -558,7 +787,10 @@ def _boundary_props_to_polygons(
layer = layer_tups[elem_layer_inds[ee]]
vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode)
if raw_mode:
poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
else:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=False)
pat.shapes[layer].append(poly)