# ruff: noqa: ARG001, F401
"""
GDSII file format readers and writers using the `TODO` library.

Note that GDSII references follow the same convention as `masque`,
  with this order of operations:
   1. Mirroring
   2. Rotation
   3. Scaling
   4. Offset and array expansion (no mirroring/rotation/scaling applied to offsets)

  Scaling, rotation, and mirroring apply to individual instances, not grid
   vectors or offsets.

Notes:
 * absolute positioning is not supported
 * PLEX is not supported
 * ELFLAGS are not supported
 * GDS does not support library- or structure-level annotations
 * GDS creation/modification/access times are set to 1900-01-01 for reproducibility.
 * Gzip modification time is set to 0 (start of current epoch, usually 1970-01-01)

 TODO writing
 TODO warn on boxes, nodes
"""
from typing import IO, cast, Any
from collections.abc import Iterable, Mapping, Callable
from importlib.machinery import EXTENSION_SUFFIXES
import importlib.util
import mmap
import logging
import os
import pathlib
import gzip
import string
import sys
import tempfile
from pprint import pformat

from klamath.basic import KlamathError
import numpy
from numpy.typing import ArrayLike, NDArray
import pyarrow
from pyarrow.cffi import ffi

from ..utils import is_gzipped, tmpfile
from ... import Pattern, Ref, PatternError, LibraryError, Label, Shape
from ...shapes import Polygon, Path, PolyCollection, RectCollection
from ...repetition import Grid
from ...utils import layer_t, annotations_t
from ...library import LazyLibrary, Library, ILibrary, ILibraryView


logger = logging.getLogger(__name__)

ffi.cdef(
    """
    const char* last_error_message(void);
    int read_path(const char* path, struct ArrowArray* array, struct ArrowSchema* schema);
    int scan_bytes(uint8_t* data, size_t size, struct ArrowArray* array, struct ArrowSchema* schema);
    int read_cells_bytes(
        uint8_t* data,
        size_t size,
        uint64_t* ranges,
        size_t range_count,
        struct ArrowArray* array,
        struct ArrowSchema* schema
    );
    """
)

clib: Any | None = None


path_cap_map = {
    0: Path.Cap.Flush,
    1: Path.Cap.Circle,
    2: Path.Cap.Square,
    4: Path.Cap.SquareCustom,
    }


def rint_cast(val: ArrayLike) -> NDArray[numpy.int32]:
    return numpy.rint(val).astype(numpy.int32)


def _packed_layer_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int16]:
    layer = (values >> numpy.uint32(16)).astype(numpy.uint16).view(numpy.int16)
    dtype = (values & numpy.uint32(0xffff)).astype(numpy.uint16).view(numpy.int16)
    return numpy.stack((layer, dtype), axis=-1)


def _packed_counts_u32_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int64]:
    a_count = (values >> numpy.uint32(16)).astype(numpy.uint16).astype(numpy.int64)
    b_count = (values & numpy.uint32(0xffff)).astype(numpy.uint16).astype(numpy.int64)
    return numpy.stack((a_count, b_count), axis=-1)


def _packed_xy_u64_to_pairs(values: NDArray[numpy.unsignedinteger[Any]]) -> NDArray[numpy.int32]:
    xx = (values >> numpy.uint64(32)).astype(numpy.uint32).view(numpy.int32)
    yy = (values & numpy.uint64(0xffff_ffff)).astype(numpy.uint32).view(numpy.int32)
    return numpy.stack((xx, yy), axis=-1)


def _local_library_filename() -> str:
    if sys.platform.startswith('linux'):
        return 'libklamath_rs_ext.so'
    if sys.platform == 'darwin':
        return 'libklamath_rs_ext.dylib'
    if sys.platform == 'win32':
        return 'klamath_rs_ext.dll'
    raise OSError(f'Unsupported platform for klamath_rs_ext: {sys.platform!r}')


def _installed_library_candidates() -> list[pathlib.Path]:
    candidates: list[pathlib.Path] = []

    try:
        spec = importlib.util.find_spec('klamath_rs_ext.klamath_rs_ext')
    except ModuleNotFoundError:
        spec = None
    if spec is not None and spec.origin is not None:
        candidates.append(pathlib.Path(spec.origin))

    try:
        pkg_spec = importlib.util.find_spec('klamath_rs_ext')
    except ModuleNotFoundError:
        pkg_spec = None
    if pkg_spec is not None and pkg_spec.submodule_search_locations is not None:
        for location in pkg_spec.submodule_search_locations:
            pkg_dir = pathlib.Path(location)
            for suffix in EXTENSION_SUFFIXES:
                candidates.extend(sorted(pkg_dir.glob(f'klamath_rs_ext*{suffix}')))

    return candidates


def _repo_library_candidates() -> list[pathlib.Path]:
    repo_root = pathlib.Path(__file__).resolve().parents[3]
    library_name = _local_library_filename()
    return [
        repo_root / 'klamath-rs' / 'target' / 'release' / library_name,
        repo_root / 'klamath-rs' / 'target' / 'debug' / library_name,
        ]


def find_klamath_rs_library() -> pathlib.Path | None:
    env_path = os.environ.get('KLAMATH_RS_EXT_LIB')
    if env_path:
        candidate = pathlib.Path(env_path).expanduser()
        if candidate.exists():
            return candidate.resolve()

    seen: set[pathlib.Path] = set()
    for candidate in _installed_library_candidates() + _repo_library_candidates():
        resolved = candidate.expanduser()
        if resolved in seen:
            continue
        seen.add(resolved)
        if resolved.exists():
            return resolved.resolve()
    return None


def is_available() -> bool:
    return find_klamath_rs_library() is not None


def _get_clib() -> Any:
    global clib  # noqa: PLW0603
    if clib is None:
        lib_path = find_klamath_rs_library()
        if lib_path is None:
            raise ImportError(
                'Could not locate klamath_rs_ext shared library. '
                'Build klamath-rs with `cargo build --release --manifest-path klamath-rs/Cargo.toml` '
                'or set KLAMATH_RS_EXT_LIB to the built library path.'
                )
        clib = ffi.dlopen(str(lib_path))
    return clib


def _read_annotations(
        prop_offs: NDArray[numpy.integer[Any]],
        prop_key: NDArray[numpy.integer[Any]],
        prop_val: list[str],
        ee: int,
        ) -> annotations_t:
    prop_ii, prop_ff = prop_offs[ee], prop_offs[ee + 1]
    if prop_ii >= prop_ff:
        return None
    return {str(prop_key[off]): [prop_val[off]] for off in range(prop_ii, prop_ff)}


def _read_to_arrow(
        filename: str | pathlib.Path,
        ) -> pyarrow.Array:
    path = pathlib.Path(filename).expanduser().resolve()
    ptr_array = ffi.new('struct ArrowArray[]', 1)
    ptr_schema = ffi.new('struct ArrowSchema[]', 1)
    if is_gzipped(path):
        with gzip.open(path, mode='rb') as src:
            data = src.read()
        with tempfile.NamedTemporaryFile(suffix='.gds', delete=False) as tmp_stream:
            tmp_stream.write(data)
            tmp_name = tmp_stream.name
        try:
            _call_native(_get_clib().read_path(tmp_name.encode(), ptr_array, ptr_schema), 'read_path')
        finally:
            pathlib.Path(tmp_name).unlink(missing_ok=True)
    else:
        _call_native(_get_clib().read_path(str(path).encode(), ptr_array, ptr_schema), 'read_path')
    return _import_arrow_array(ptr_array, ptr_schema)


def _import_arrow_array(ptr_array: Any, ptr_schema: Any) -> pyarrow.Array:
    iptr_schema = int(ffi.cast('uintptr_t', ptr_schema))
    iptr_array = int(ffi.cast('uintptr_t', ptr_array))
    return pyarrow.Array._import_from_c(iptr_array, iptr_schema)


def _call_native(status: int, action: str) -> None:
    if status == 0:
        return

    err_ptr = _get_clib().last_error_message()
    if err_ptr == ffi.NULL:
        raise KlamathError(f'{action} failed')

    message = ffi.string(err_ptr).decode(errors='replace')
    raise KlamathError(message)


def _scan_buffer_to_arrow(buffer: bytes | mmap.mmap | memoryview) -> pyarrow.Array:
    ptr_array = ffi.new('struct ArrowArray[]', 1)
    ptr_schema = ffi.new('struct ArrowSchema[]', 1)
    buf_view = memoryview(buffer)
    cbuf = ffi.from_buffer('uint8_t[]', buf_view)
    _call_native(_get_clib().scan_bytes(cbuf, len(buf_view), ptr_array, ptr_schema), 'scan_bytes')
    return _import_arrow_array(ptr_array, ptr_schema)


def _read_selected_cells_to_arrow(
        buffer: bytes | mmap.mmap | memoryview,
        ranges: NDArray[numpy.uint64],
        ) -> pyarrow.Array:
    ptr_array = ffi.new('struct ArrowArray[]', 1)
    ptr_schema = ffi.new('struct ArrowSchema[]', 1)
    buf_view = memoryview(buffer)
    cbuf = ffi.from_buffer('uint8_t[]', buf_view)
    flat_ranges = numpy.require(ranges, dtype=numpy.uint64, requirements=('C_CONTIGUOUS', 'ALIGNED'))
    cranges = ffi.from_buffer('uint64_t[]', flat_ranges)
    _call_native(
        _get_clib().read_cells_bytes(cbuf, len(buf_view), cranges, int(flat_ranges.shape[0]), ptr_array, ptr_schema),
        'read_cells_bytes',
    )
    return _import_arrow_array(ptr_array, ptr_schema)


def readfile(
        filename: str | pathlib.Path,
        ) -> tuple[Library, dict[str, Any]]:
    """
    Read a GDSII file from a path into `masque.Library` / `Pattern` objects.

    Will automatically decompress gzipped files.

    Args:
        filename: Filename to read.

    For callers that can consume Arrow directly, prefer `readfile_arrow()`
    to skip Python `Pattern` construction entirely.
    """
    arrow_arr = _read_to_arrow(filename)
    assert len(arrow_arr) == 1

    results = read_arrow(arrow_arr[0])

    return results


def readfile_arrow(
        filename: str | pathlib.Path,
        ) -> tuple[pyarrow.StructScalar, dict[str, Any]]:
    """
    Read a GDSII file into the native Arrow representation without converting
    it into `masque.Library` / `Pattern` objects.

    This is the lowest-overhead public read path exposed by this module.

    Args:
        filename: Filename to read.

    Returns:
        - Arrow struct scalar for the library payload
        - dict of GDSII library info
    """
    arrow_arr = _read_to_arrow(filename)
    assert len(arrow_arr) == 1
    libarr = arrow_arr[0]
    return libarr, _read_header(libarr)


def read_arrow(
        libarr: pyarrow.Array,
        ) -> tuple[Library, dict[str, Any]]:
    """
    # TODO check GDSII file for cycles!
    Read a gdsii file and translate it into a dict of Pattern objects. GDSII structures are
     translated into Pattern objects; boundaries are translated into polygons, and srefs and arefs
     are translated into Ref objects.

    Additional library info is returned in a dict, containing:
      'name': name of the library
      'meters_per_unit': number of meters per database unit (all values are in database units)
      'logical_units_per_unit': number of "logical" units displayed by layout tools (typically microns)
                                per database unit

    Args:
        libarr: Arrow library payload as returned by `readfile_arrow()`.

    Returns:
        - dict of pattern_name:Patterns generated from GDSII structures
        - dict of GDSII library info
    """
    library_info = _read_header(libarr)

    layer_names_np = _packed_layer_u32_to_pairs(libarr['layers'].values.to_numpy())
    layer_tups = [(int(pair[0]), int(pair[1])) for pair in layer_names_np]

    cell_ids = libarr['cells'].values.field('id').to_numpy()
    cell_names = libarr['cell_names'].as_py()

    def get_geom(libarr: pyarrow.Array, geom_type: str) -> dict[str, Any]:
        el = libarr['cells'].values.field(geom_type)
        elem = dict(
            offsets = el.offsets.to_numpy(),
            xy_arr = el.values.field('xy').values.to_numpy().reshape((-1, 2)),
            xy_off = el.values.field('xy').offsets.to_numpy() // 2,
            layer_inds = el.values.field('layer').to_numpy(),
            prop_off = el.values.field('properties').offsets.to_numpy(),
            prop_key = el.values.field('properties').values.field('key').to_numpy(),
            prop_val = el.values.field('properties').values.field('value').to_pylist(),
            )
        return elem

    def get_boundary_batches(libarr: pyarrow.Array) -> dict[str, Any]:
        batches = libarr['cells'].values.field('boundary_batches')
        return dict(
            offsets = batches.offsets.to_numpy(),
            layer_inds = batches.values.field('layer').to_numpy(),
            vert_arr = batches.values.field('vertices').values.to_numpy().reshape((-1, 2)),
            vert_off = batches.values.field('vertices').offsets.to_numpy() // 2,
            poly_off = batches.values.field('vertex_offsets').offsets.to_numpy(),
            poly_offsets = batches.values.field('vertex_offsets').values.to_numpy(),
            )

    def get_rect_batches(libarr: pyarrow.Array) -> dict[str, Any]:
        batches = libarr['cells'].values.field('rect_batches')
        return dict(
            offsets = batches.offsets.to_numpy(),
            layer_inds = batches.values.field('layer').to_numpy(),
            rect_arr = batches.values.field('rects').values.to_numpy().reshape((-1, 4)),
            rect_off = batches.values.field('rects').offsets.to_numpy() // 4,
            )

    def get_boundary_props(libarr: pyarrow.Array) -> dict[str, Any]:
        boundaries = libarr['cells'].values.field('boundary_props')
        return dict(
            offsets = boundaries.offsets.to_numpy(),
            layer_inds = boundaries.values.field('layer').to_numpy(),
            vert_arr = boundaries.values.field('vertices').values.to_numpy().reshape((-1, 2)),
            vert_off = boundaries.values.field('vertices').offsets.to_numpy() // 2,
            prop_off = boundaries.values.field('properties').offsets.to_numpy(),
            prop_key = boundaries.values.field('properties').values.field('key').to_numpy(),
            prop_val = boundaries.values.field('properties').values.field('value').to_pylist(),
            )

    def get_refs(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
        refs = libarr['cells'].values.field(geom_type)
        values = refs.values
        elem = dict(
            offsets = refs.offsets.to_numpy(),
            targets = values.field('target').to_numpy(),
            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
            invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
            angle_rad = values.field('angle_rad').to_numpy(),
            scale = values.field('scale').to_numpy(),
            )
        if has_repetition:
            elem.update(dict(
                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
                counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
                ))
        return elem

    def get_ref_props(libarr: pyarrow.Array, geom_type: str, has_repetition: bool) -> dict[str, Any]:
        refs = libarr['cells'].values.field(geom_type)
        values = refs.values
        elem = dict(
            offsets = refs.offsets.to_numpy(),
            targets = values.field('target').to_numpy(),
            xy = _packed_xy_u64_to_pairs(values.field('xy').to_numpy()),
            invert_y = values.field('invert_y').to_numpy(zero_copy_only=False),
            angle_rad = values.field('angle_rad').to_numpy(),
            scale = values.field('scale').to_numpy(),
            prop_off = values.field('properties').offsets.to_numpy(),
            prop_key = values.field('properties').values.field('key').to_numpy(),
            prop_val = values.field('properties').values.field('value').to_pylist(),
            )
        if has_repetition:
            elem.update(dict(
                xy0 = _packed_xy_u64_to_pairs(values.field('xy0').to_numpy()),
                xy1 = _packed_xy_u64_to_pairs(values.field('xy1').to_numpy()),
                counts = _packed_counts_u32_to_pairs(values.field('counts').to_numpy()),
                ))
        return elem

    txt = libarr['cells'].values.field('texts')
    texts = dict(
        offsets = txt.offsets.to_numpy(),
        layer_inds = txt.values.field('layer').to_numpy(),
        xy = _packed_xy_u64_to_pairs(txt.values.field('xy').to_numpy()),
        string = txt.values.field('string').to_pylist(),
        prop_off = txt.values.field('properties').offsets.to_numpy(),
        prop_key = txt.values.field('properties').values.field('key').to_numpy(),
        prop_val = txt.values.field('properties').values.field('value').to_pylist(),
        )

    elements = dict(
        srefs = get_refs(libarr, 'srefs', has_repetition=False),
        arefs = get_refs(libarr, 'arefs', has_repetition=True),
        sref_props = get_ref_props(libarr, 'sref_props', has_repetition=False),
        aref_props = get_ref_props(libarr, 'aref_props', has_repetition=True),
        rect_batches = get_rect_batches(libarr),
        boundary_batches = get_boundary_batches(libarr),
        boundary_props = get_boundary_props(libarr),
        paths = get_geom(libarr, 'paths'),
        texts = texts,
        )

    paths = libarr['cells'].values.field('paths')
    elements['paths'].update(dict(
        width = paths.values.field('width').fill_null(0).to_numpy(),
        path_type = paths.values.field('path_type').fill_null(0).to_numpy(),
        extensions = numpy.stack((
            paths.values.field('extension_start').fill_null(0).to_numpy(),
            paths.values.field('extension_end').fill_null(0).to_numpy(),
            ), axis=-1),
        ))

    global_args = dict(
        cell_names = cell_names,
        layer_tups = layer_tups,
        )

    mlib = Library()
    for cc in range(len(libarr['cells'])):
        name = cell_names[int(cell_ids[cc])]
        pat = Pattern()
        _rect_batches_to_rectcollections(pat, global_args, elements['rect_batches'], cc)
        _boundary_batches_to_polygons(pat, global_args, elements['boundary_batches'], cc)
        _boundary_props_to_polygons(pat, global_args, elements['boundary_props'], cc)
        _gpaths_to_mpaths(pat, global_args, elements['paths'], cc)
        _srefs_to_mrefs(pat, global_args, elements['srefs'], cc)
        _arefs_to_mrefs(pat, global_args, elements['arefs'], cc)
        _sref_props_to_mrefs(pat, global_args, elements['sref_props'], cc)
        _aref_props_to_mrefs(pat, global_args, elements['aref_props'], cc)
        _texts_to_labels(pat, global_args, elements['texts'], cc)
        mlib[name] = pat

    return mlib, library_info


def _read_header(libarr: pyarrow.Array) -> dict[str, Any]:
    """
    Read the file header and create the library_info dict.
    """
    library_info = dict(
        name = libarr['lib_name'].as_py(),
        meters_per_unit = libarr['meters_per_db_unit'].as_py(),
        logical_units_per_unit = libarr['user_units_per_db_unit'].as_py(),
        )
    return library_info


def _srefs_to_mrefs(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    cell_names = global_args['cell_names']
    elem_off = elem['offsets']
    elem_count = elem_off[cc + 1] - elem_off[cc]
    if elem_count == 0:
        return

    start = elem_off[cc]
    stop = elem_off[cc + 1]
    elem_targets = elem['targets'][start:stop]
    elem_xy = elem['xy'][start:stop]
    elem_invert_y = elem['invert_y'][start:stop]
    elem_angle_rad = elem['angle_rad'][start:stop]
    elem_scale = elem['scale'][start:stop]

    _append_plain_refs_sorted(
        pat=pat,
        cell_names=cell_names,
        elem_targets=elem_targets,
        elem_xy=elem_xy,
        elem_invert_y=elem_invert_y,
        elem_angle_rad=elem_angle_rad,
        elem_scale=elem_scale,
        )


def _append_plain_refs_sorted(
        *,
        pat: Pattern,
        cell_names: list[str],
        elem_targets: NDArray[numpy.integer[Any]],
        elem_xy: NDArray[numpy.integer[Any]],
        elem_invert_y: NDArray[numpy.bool_ | numpy.bool],
        elem_angle_rad: NDArray[numpy.floating[Any]],
        elem_scale: NDArray[numpy.floating[Any]],
        ) -> None:
    elem_count = len(elem_targets)
    if elem_count == 0:
        return

    target_start = 0
    while target_start < elem_count:
        target_id = int(elem_targets[target_start])
        target_stop = target_start + 1
        while target_stop < elem_count and elem_targets[target_stop] == target_id:
            target_stop += 1

        append_refs = pat.refs[cell_names[target_id]].extend
        append_refs(
            Ref._from_raw(
                offset=elem_xy[ee],
                mirrored=elem_invert_y[ee],
                rotation=elem_angle_rad[ee],
                scale=elem_scale[ee],
                repetition=None,
                annotations=None,
                )
            for ee in range(target_start, target_stop)
            )

        target_start = target_stop


def _arefs_to_mrefs(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    cell_names = global_args['cell_names']
    elem_off = elem['offsets']
    elem_count = elem_off[cc + 1] - elem_off[cc]
    if elem_count == 0:
        return

    start = elem_off[cc]
    stop = elem_off[cc + 1]
    elem_targets = elem['targets'][start:stop]
    elem_xy = elem['xy'][start:stop]
    elem_invert_y = elem['invert_y'][start:stop]
    elem_angle_rad = elem['angle_rad'][start:stop]
    elem_scale = elem['scale'][start:stop]
    elem_xy0 = elem['xy0'][start:stop]
    elem_xy1 = elem['xy1'][start:stop]
    elem_counts = elem['counts'][start:stop]

    if len(elem_targets) == 0:
        return

    target = None
    append_ref: Callable[[Ref], Any] | None = None
    for ee in range(len(elem_targets)):
        target_id = int(elem_targets[ee])
        if target != target_id:
            target = target_id
            append_ref = pat.refs[cell_names[target_id]].append
        assert append_ref is not None
        a_count, b_count = elem_counts[ee]
        append_ref(Ref._from_raw(
            offset=elem_xy[ee],
            mirrored=elem_invert_y[ee],
            rotation=elem_angle_rad[ee],
            scale=elem_scale[ee],
            repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
            annotations=None,
            ))


def _sref_props_to_mrefs(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    cell_names = global_args['cell_names']
    elem_off = elem['offsets']
    prop_key = elem['prop_key']
    prop_val = elem['prop_val']

    elem_count = elem_off[cc + 1] - elem_off[cc]
    if elem_count == 0:
        return

    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
    prop_offs = elem['prop_off'][elem_slc]
    elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
    elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
    elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
    elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
    elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]

    for ee in range(elem_count):
        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
        ref = Ref._from_raw(
            offset=elem_xy[ee],
            mirrored=elem_invert_y[ee],
            rotation=elem_angle_rad[ee],
            scale=elem_scale[ee],
            repetition=None,
            annotations=annotations,
            )
        pat.refs[cell_names[int(elem_targets[ee])]].append(ref)


def _aref_props_to_mrefs(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    cell_names = global_args['cell_names']
    elem_off = elem['offsets']
    prop_key = elem['prop_key']
    prop_val = elem['prop_val']

    elem_count = elem_off[cc + 1] - elem_off[cc]
    if elem_count == 0:
        return

    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
    prop_offs = elem['prop_off'][elem_slc]
    elem_targets = elem['targets'][elem_off[cc]:elem_off[cc + 1]]
    elem_xy = elem['xy'][elem_off[cc]:elem_off[cc + 1]]
    elem_invert_y = elem['invert_y'][elem_off[cc]:elem_off[cc + 1]]
    elem_angle_rad = elem['angle_rad'][elem_off[cc]:elem_off[cc + 1]]
    elem_scale = elem['scale'][elem_off[cc]:elem_off[cc + 1]]
    elem_xy0 = elem['xy0'][elem_off[cc]:elem_off[cc + 1]]
    elem_xy1 = elem['xy1'][elem_off[cc]:elem_off[cc + 1]]
    elem_counts = elem['counts'][elem_off[cc]:elem_off[cc + 1]]

    for ee in range(elem_count):
        a_count, b_count = elem_counts[ee]
        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
        ref = Ref._from_raw(
            offset=elem_xy[ee],
            mirrored=elem_invert_y[ee],
            rotation=elem_angle_rad[ee],
            scale=elem_scale[ee],
            repetition=Grid._from_raw(a_vector=elem_xy0[ee], b_vector=elem_xy1[ee], a_count=a_count, b_count=b_count),
            annotations=annotations,
            )
        pat.refs[cell_names[int(elem_targets[ee])]].append(ref)


def _texts_to_labels(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    elem_off = elem['offsets']      # which elements belong to each cell
    xy = elem['xy']
    layer_tups = global_args['layer_tups']
    layer_inds = elem['layer_inds']
    prop_key = elem['prop_key']
    prop_val = elem['prop_val']

    elem_count = elem_off[cc + 1] - elem_off[cc]
    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)   # +1 to capture ending location for last elem
    prop_offs = elem['prop_off'][elem_slc]  # which props belong to each element
    elem_xy = xy[elem_slc][:elem_count]
    elem_layer_inds = layer_inds[elem_slc][:elem_count]
    elem_strings = elem['string'][elem_slc][:elem_count]

    for ee in range(elem_count):
        layer = layer_tups[int(elem_layer_inds[ee])]
        offset = elem_xy[ee]
        string = elem_strings[ee]

        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
        mlabel = Label._from_raw(string=string, offset=offset, annotations=annotations)
        pat.labels[layer].append(mlabel)


def _gpaths_to_mpaths(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    elem_off = elem['offsets']      # which elements belong to each cell
    xy_val = elem['xy_arr']
    layer_tups = global_args['layer_tups']
    layer_inds = elem['layer_inds']
    prop_key = elem['prop_key']
    prop_val = elem['prop_val']

    elem_count = elem_off[cc + 1] - elem_off[cc]
    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)   # +1 to capture ending location for last elem
    xy_offs = elem['xy_off'][elem_slc]      # which xy coords belong to each element
    prop_offs = elem['prop_off'][elem_slc]  # which props belong to each element
    elem_layer_inds = layer_inds[elem_slc][:elem_count]
    elem_widths = elem['width'][elem_slc][:elem_count]
    elem_path_types = elem['path_type'][elem_slc][:elem_count]
    elem_extensions = elem['extensions'][elem_slc][:elem_count]

    for ee in range(elem_count):
        layer = layer_tups[int(elem_layer_inds[ee])]
        vertices = xy_val[xy_offs[ee]:xy_offs[ee + 1]]
        width = elem_widths[ee]
        cap_int = int(elem_path_types[ee])
        if cap_int not in path_cap_map:
            raise PatternError(f'Unrecognized path type: {cap_int}')
        cap = path_cap_map[cap_int]
        if cap_int == 4:
            cap_extensions = elem_extensions[ee]
        else:
            cap_extensions = None

        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
        path = Path._from_raw(
            vertices=vertices,
            width=width,
            cap=cap,
            cap_extensions=cap_extensions,
            annotations=annotations,
            )
        pat.shapes[layer].append(path)


def _boundary_batches_to_polygons(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    elem_off = elem['offsets']      # which elements belong to each cell
    vert_arr = elem['vert_arr']
    vert_off = elem['vert_off']
    layer_inds = elem['layer_inds']
    layer_tups = global_args['layer_tups']
    poly_off = elem['poly_off']
    poly_offsets = elem['poly_offsets']

    batch_count = elem_off[cc + 1] - elem_off[cc]
    if batch_count == 0:
        return

    elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1)   # +1 to capture ending location for last elem
    elem_vert_off = vert_off[elem_slc]
    elem_poly_off = poly_off[elem_slc]
    elem_layer_inds = layer_inds[elem_slc][:batch_count]

    for bb in range(batch_count):
        layer = layer_tups[int(elem_layer_inds[bb])]
        vertices = vert_arr[elem_vert_off[bb]:elem_vert_off[bb + 1]]
        vertex_offsets = poly_offsets[elem_poly_off[bb]:elem_poly_off[bb + 1]]

        if vertex_offsets.size == 1:
            poly = Polygon._from_raw(vertices=vertices, annotations=None)
            pat.shapes[layer].append(poly)
        else:
            polys = PolyCollection._from_raw(vertex_lists=vertices, vertex_offsets=vertex_offsets, annotations=None)
            pat.shapes[layer].append(polys)


def _rect_batches_to_rectcollections(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    elem_off = elem['offsets']
    rect_arr = elem['rect_arr']
    rect_off = elem['rect_off']
    layer_inds = elem['layer_inds']
    layer_tups = global_args['layer_tups']

    batch_count = elem_off[cc + 1] - elem_off[cc]
    if batch_count == 0:
        return

    elem_slc = slice(elem_off[cc], elem_off[cc] + batch_count + 1)
    elem_rect_off = rect_off[elem_slc]
    elem_layer_inds = layer_inds[elem_slc][:batch_count]

    for bb in range(batch_count):
        layer = layer_tups[int(elem_layer_inds[bb])]
        rects = rect_arr[elem_rect_off[bb]:elem_rect_off[bb + 1]]
        rect_collection = RectCollection._from_raw(rects=rects, annotations=None)
        pat.shapes[layer].append(rect_collection)


def _boundary_props_to_polygons(
        pat: Pattern,
        global_args: dict[str, Any],
        elem: dict[str, Any],
        cc: int,
        ) -> None:
    elem_off = elem['offsets']
    vert_arr = elem['vert_arr']
    vert_off = elem['vert_off']
    layer_inds = elem['layer_inds']
    layer_tups = global_args['layer_tups']
    prop_key = elem['prop_key']
    prop_val = elem['prop_val']

    elem_count = elem_off[cc + 1] - elem_off[cc]
    if elem_count == 0:
        return

    elem_slc = slice(elem_off[cc], elem_off[cc] + elem_count + 1)
    elem_vert_off = vert_off[elem_slc]
    prop_offs = elem['prop_off'][elem_slc]
    elem_layer_inds = layer_inds[elem_slc][:elem_count]

    for ee in range(elem_count):
        layer = layer_tups[int(elem_layer_inds[ee])]
        vertices = vert_arr[elem_vert_off[ee]:elem_vert_off[ee + 1]]
        annotations = _read_annotations(prop_offs, prop_key, prop_val, ee)
        poly = Polygon._from_raw(vertices=vertices, annotations=annotations)
        pat.shapes[layer].append(poly)


#def _properties_to_annotations(properties: pyarrow.Array) -> annotations_t:
#    return {prop['key'].as_py(): prop['value'].as_py() for prop in properties}


def check_valid_names(
        names: Iterable[str],
        max_length: int = 32,
        ) -> None:
    """
    Check all provided names to see if they're valid GDSII cell names.

    Args:
        names: Collection of names to check
        max_length: Max allowed length

    """
    allowed_chars = set(string.ascii_letters + string.digits + '_?$')

    bad_chars = [
        name for name in names
        if not set(name).issubset(allowed_chars)
        ]

    bad_lengths = [
        name for name in names
        if len(name) > max_length
        ]

    if bad_chars:
        logger.error('Names contain invalid characters:\n' + pformat(bad_chars))

    if bad_lengths:
        logger.error(f'Names too long (>{max_length}:\n' + pformat(bad_chars))

    if bad_chars or bad_lengths:
        raise LibraryError('Library contains invalid names, see log above')