jan/masque
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

[gdsii_arrow] further improvements to speed

Browse source
This commit is contained in:
Jan Petykiewicz 2026-04-02 15:47:41 -07:00
commit 4aca8acd3d
3 changed files with 235 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

144
masque/file/gdsii_arrow.py
View file

@ -57,6 +57,8 @@ ffi.cdef('void read_path(char* path, struct ArrowArray* array, struct ArrowSchem
clib: Any | None = None
ZERO_OFFSET = numpy.zeros(2)
path_cap_map = {
0: Path.Cap.Flush,
@ -263,6 +265,29 @@ def read_arrow(
)
return elem
def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]:
batches = libarr['cells'].values.field('boundary_batches')
return dict(
offsets = batches.offsets.to_numpy(),
layer_inds = batches.values.field('layer').to_numpy(),
vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)),
vert_off = batches.values.field('vertices').offsets.to_numpy() // 2,
poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(),
poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
)
def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]:
boundaries = libarr['cells'].values.field('boundary_props')
return dict(
offsets = boundaries.offsets.to_numpy(),
layer_inds = boundaries.values.field('layer').to_numpy(),
vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)),
vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2,
prop_off = boundaries.values.field('properties').offsets.to_numpy(),
prop_key = boundaries.values.field('properties').values.field('key').to_numpy(),
prop_val = boundaries.values.field('properties').values.field('value').to_pylist(),
)
rf = libarr['cells'].values.field('refs')
refs = dict(
offsets = rf.offsets.to_numpy(),
@ -292,10 +317,9 @@ def read_arrow(
)
elements = dict(
boundaries = get_geom(libarr, 'boundaries'),
boundary_batches = get_boundary_batches(libarr),
boundary_props = get_boundary_props(libarr),
paths = get_geom(libarr, 'paths'),
boxes = get_geom(libarr, 'boxes'),
nodes = get_geom(libarr, 'nodes'),
texts = texts,
refs = refs,
)
@ -320,7 +344,8 @@ def read_arrow(
for cc in range(len(libarr['cells'])):
name = cell_names[cell_ids[cc]]
pat = Pattern()
_boundaries_to_polygons(pat, global_args, elements['boundaries'], cc)
_boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc)
_boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc)
_gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
_grefs_to_mrefs(pat, global_args, elements['refs'], cc)
_texts_to_labels(pat, global_args, elements['texts'], cc)
@ -366,6 +391,7 @@ def _grefs_to_mrefs(
elem_rep_xy1 = elem['rep_xy1'][elem_slc][:elem_count]
elem_rep_counts = elem['rep_counts'][elem_slc][:elem_count]
rep_valid = elem['rep_valid'][elem_slc][:elem_count]
raw_mode = global_args['raw_mode']
for ee in range(elem_count):
@ -380,10 +406,16 @@ def _grefs_to_mrefs(
a_vector = elem_rep_xy0[ee]
b_vector = elem_rep_xy1[ee]
a_count, b_count = elem_rep_counts[ee]
rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
if raw_mode:
rep = Grid._from_raw(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
else:
rep = Grid(a_vector=a_vector, b_vector=b_vector, a_count=a_count, b_count=b_count)
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
if raw_mode:
ref = Ref._from_raw(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
else:
ref = Ref(offset=offset, mirrored=mirr, rotation=rot, scale=mag, repetition=rep, annotations=annotations)
pat.refs[target].append(ref)
@ -406,6 +438,7 @@ def _texts_to_labels(
elem_xy = xy[elem_slc][:elem_count]
elem_layer_inds = layer_inds[elem_slc][:elem_count]
elem_strings = elem['string'][elem_slc][:elem_count]
raw_mode = global_args['raw_mode']
for ee in range(elem_count):
layer = layer_tups[elem_layer_inds[ee]]
@ -413,7 +446,10 @@ def _texts_to_labels(
string = elem_strings[ee]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
mlabel = Label(string=string, offset=offset, annotations=annotations)
if raw_mode:
mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations)
else:
mlabel = Label(string=string, offset=offset, annotations=annotations)
pat.labels[layer].append(mlabel)
@ -439,7 +475,6 @@ def _gpaths_to_mpaths(
elem_path_types = elem['path_type'][elem_slc][:elem_count]
elem_extensions = elem['extensions'][elem_slc][:elem_count]
zeros = numpy.zeros((elem_count, 2))
raw_mode = global_args['raw_mode']
for ee in range(elem_count):
layer = layer_tups[elem_layer_inds[ee]]
@ -453,65 +488,78 @@ def _gpaths_to_mpaths(
cap_extensions = None
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
path = Path(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode,
path = Path(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode,
width=width, cap=cap,cap_extensions=cap_extensions)
pat.shapes[layer].append(path)
def _boundaries_to_polygons(
def _boundary_batches_to_polygons(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets'] # which elements belong to each cell
xy_val = elem['xy_arr']
vert_arr = elem['vert_arr']
vert_off = elem['vert_off']
layer_inds = elem['layer_inds']
layer_tups = global_args['layer_tups']
poly_off = elem['poly_off']
poly_offsets = elem['poly_offsets']
batch_count = elem_off[cc + 1] - elem_off[cc]
if batch_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1) # +1 to capture ending location for last elem
elem_vert_off = vert_off[elem_slc]
elem_poly_off = poly_off[elem_slc]
elem_layer_inds = layer_inds[elem_slc][:batch_count]
raw_mode = global_args['raw_mode']
for bb in range(batch_count):
layer = layer_tups[elem_layer_inds[bb]]
vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
vertex_offsets = numpy.asarray(poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]], dtype=numpy.intp)
if vertex_offsets.size == 1:
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
pat.shapes[layer].append(poly)
else:
polys = PolyCollection(vertex_lists=vertices, vertex_offsets=vertex_offsets, offset=ZERO_OFFSET, annotations=None, raw=raw_mode)
pat.shapes[layer].append(polys)
def _boundary_props_to_polygons(
pat: Pattern,
global_args: dict[str, Any],
elem: dict[str, Any],
cc: int,
) -> None:
elem_off = elem['offsets']
vert_arr = elem['vert_arr']
vert_off = elem['vert_off']
layer_inds = elem['layer_inds']
layer_tups = global_args['layer_tups']
prop_key = elem['prop_key']
prop_val = elem['prop_val']
elem_count = elem_off[cc + 1] - elem_off[cc]
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1) # +1 to capture ending location for last elem
xy_offs = elem['xy_off'][elem_slc] # which xy coords belong to each element
xy_counts = xy_offs[1:] - xy_offs[:-1]
prop_offs = elem['prop_off'][elem_slc] # which props belong to each element
prop_counts = prop_offs[1:] - prop_offs[:-1]
if elem_count == 0:
return
elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
elem_vert_off = vert_off[elem_slc]
prop_offs = elem['prop_off'][elem_slc]
elem_layer_inds = layer_inds[elem_slc][:elem_count]
order = numpy.argsort(elem_layer_inds, stable=True)
unilayer_inds, unilayer_first, unilayer_count = numpy.unique(elem_layer_inds, return_index=True, return_counts=True)
zeros = numpy.zeros((elem_count, 2))
raw_mode = global_args['raw_mode']
for layer_ind, ff, nn in zip(unilayer_inds, unilayer_first, unilayer_count, strict=True):
ee_inds = order[ff:ff + nn]
layer = layer_tups[layer_ind]
propless_mask = prop_counts[ee_inds] == 0
poly_count_on_layer = propless_mask.sum()
if poly_count_on_layer == 1:
propless_mask[:] = 0 # Never make a 1-element collection
elif poly_count_on_layer > 1:
propless_vert_counts = xy_counts[ee_inds[propless_mask]] - 1 # -1 to drop closing point
vertex_lists = numpy.empty((propless_vert_counts.sum(), 2), dtype=numpy.float64)
vertex_offsets = numpy.cumsum(numpy.concatenate([[0], propless_vert_counts]))
for ii, ee in enumerate(ee_inds[propless_mask]):
vo = vertex_offsets[ii]
vertex_lists[vo:vo + propless_vert_counts[ii]] = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1]
polys = PolyCollection(vertex_lists=vertex_lists, vertex_offsets=vertex_offsets, offset=zeros[ee])
pat.shapes[layer].append(polys)
# Handle single polygons
for ee in ee_inds[~propless_mask]:
layer = layer_tups[elem_layer_inds[ee]]
vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1] - 1] # -1 to drop closing point
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
poly = Polygon(vertices=vertices, offset=zeros[ee], annotations=annotations, raw=raw_mode)
pat.shapes[layer].append(poly)
for ee in range(elem_count):
layer = layer_tups[elem_layer_inds[ee]]
vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
poly = Polygon(vertices=vertices, offset=ZERO_OFFSET, annotations=annotations, raw=raw_mode)
pat.shapes[layer].append(poly)
#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t: