Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy
scans and selective cell loading. Validate numeric packing and array pitches,
and preserve normalized REAL8 encodings exactly within the supported range.
Add standalone dump, structural diff, hierarchy, layer-name annotation and Git
diff commands. Provide scoped text and JSON reports, deterministic comparison,
and record-preserving metadata edits.
Package the shared library and optional Python Arrow readers as pheon using
CFFI ABI mode and Python-independent platform wheels. Include locked source
builds, native API checks, cross-platform artifact workflows, documentation
and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer.
Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14,
and 71 installed companion integration tests pass. Full, scanned and selected
Arrow layouts match the previous schemas.
2026-11-16 16:21:21 -08:00
|
|
|
"""Public Arrow readers, lifetime management and input validation."""
|
|
|
|
|
|
|
|
|
|
import gc
|
|
|
|
|
import gzip
|
2026-11-16 20:44:56 -08:00
|
|
|
import io
|
Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy
scans and selective cell loading. Validate numeric packing and array pitches,
and preserve normalized REAL8 encodings exactly within the supported range.
Add standalone dump, structural diff, hierarchy, layer-name annotation and Git
diff commands. Provide scoped text and JSON reports, deterministic comparison,
and record-preserving metadata edits.
Package the shared library and optional Python Arrow readers as pheon using
CFFI ABI mode and Python-independent platform wheels. Include locked source
builds, native API checks, cross-platform artifact workflows, documentation
and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer.
Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14,
and 71 installed companion integration tests pass. Full, scanned and selected
Arrow layouts match the previous schemas.
2026-11-16 16:21:21 -08:00
|
|
|
import mmap
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
import tempfile
|
|
|
|
|
from types import SimpleNamespace
|
|
|
|
|
import unittest
|
|
|
|
|
from unittest.mock import patch
|
|
|
|
|
|
|
|
|
|
import numpy
|
|
|
|
|
|
|
|
|
|
from pheon import PheonError, arrow
|
|
|
|
|
from pheon._native import ffi
|
|
|
|
|
from test_ffi import layout
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ArrowTests(unittest.TestCase):
|
|
|
|
|
def test_full_scan_and_selected_reads_own_their_data(self):
|
|
|
|
|
data = bytearray(layout())
|
|
|
|
|
full = arrow.read_bytes(data)
|
|
|
|
|
scan = arrow.scan_bytes(data)
|
|
|
|
|
cells = scan.field("cells")[0].values
|
|
|
|
|
ranges = list(
|
|
|
|
|
zip(
|
|
|
|
|
cells.field("struct_start_offset").to_pylist(),
|
|
|
|
|
cells.field("struct_end_offset").to_pylist(),
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
selected = arrow.read_cells_bytes(data, ranges)
|
|
|
|
|
data[:] = b"\0" * len(data)
|
|
|
|
|
del data
|
|
|
|
|
gc.collect()
|
|
|
|
|
self.assertEqual(full.to_pylist(), selected.to_pylist())
|
|
|
|
|
self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"])
|
|
|
|
|
self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"])
|
|
|
|
|
|
|
|
|
|
def test_paths_gzip_magic_and_closed_mmap(self):
|
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
|
|
|
path = Path(directory) / "layout with spaces.gds"
|
|
|
|
|
path.write_bytes(layout())
|
|
|
|
|
expected = arrow.read_path(path).to_pylist()
|
|
|
|
|
with path.open("rb") as stream:
|
|
|
|
|
with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
|
|
|
|
|
result = arrow.read_bytes(buffer)
|
|
|
|
|
self.assertEqual(result.to_pylist(), expected)
|
|
|
|
|
path.write_bytes(gzip.compress(layout()))
|
|
|
|
|
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
|
|
|
|
|
2026-11-16 20:44:56 -08:00
|
|
|
def test_path_reader_keeps_using_the_opened_stream(self):
|
|
|
|
|
for payload in (layout(), gzip.compress(layout())):
|
|
|
|
|
with self.subTest(compressed=payload.startswith(b"\x1f\x8b")):
|
|
|
|
|
source = io.BytesIO(payload)
|
|
|
|
|
with patch("builtins.open", return_value=source) as opened:
|
|
|
|
|
result = arrow.read_path("opened-stream-only.gds")
|
|
|
|
|
opened.assert_called_once_with("opened-stream-only.gds", "rb")
|
|
|
|
|
self.assertTrue(source.closed)
|
|
|
|
|
self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"])
|
|
|
|
|
|
Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy
scans and selective cell loading. Validate numeric packing and array pitches,
and preserve normalized REAL8 encodings exactly within the supported range.
Add standalone dump, structural diff, hierarchy, layer-name annotation and Git
diff commands. Provide scoped text and JSON reports, deterministic comparison,
and record-preserving metadata edits.
Package the shared library and optional Python Arrow readers as pheon using
CFFI ABI mode and Python-independent platform wheels. Include locked source
builds, native API checks, cross-platform artifact workflows, documentation
and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer.
Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14,
and 71 installed companion integration tests pass. Full, scanned and selected
Arrow layouts match the previous schemas.
2026-11-16 16:21:21 -08:00
|
|
|
def test_empty_and_noncontiguous_cell_selections(self):
|
|
|
|
|
data = layout()
|
|
|
|
|
empty = arrow.read_cells_bytes(data, [])
|
|
|
|
|
self.assertEqual(empty.field("cells")[0].as_py(), [])
|
|
|
|
|
values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2]
|
|
|
|
|
self.assertFalse(values.flags.c_contiguous)
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
arrow.read_cells_bytes(data, values).to_pylist(),
|
|
|
|
|
arrow.read_bytes(data).to_pylist(),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def test_bad_inputs_fail_before_importing_arrow(self):
|
|
|
|
|
with self.assertRaises(PheonError):
|
|
|
|
|
arrow.read_bytes(b"invalid")
|
|
|
|
|
with self.assertRaises(PheonError):
|
|
|
|
|
arrow.scan_bytes(b"")
|
|
|
|
|
with self.assertRaises(PheonError):
|
|
|
|
|
arrow.read_cells_bytes(layout(), [[0, 10]])
|
|
|
|
|
with self.assertRaises(ValueError):
|
|
|
|
|
arrow.read_bytes(memoryview(layout())[::2])
|
|
|
|
|
for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]:
|
|
|
|
|
with self.assertRaises(ValueError):
|
|
|
|
|
arrow.read_cells_bytes(layout(), ranges)
|
|
|
|
|
with self.assertRaises(TypeError):
|
|
|
|
|
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
|
|
|
|
|
|
2026-11-16 21:28:54 -08:00
|
|
|
def test_malformed_strings_and_years_raise_located_parse_errors(self):
|
|
|
|
|
data = layout()
|
|
|
|
|
cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0]
|
|
|
|
|
start = cell["struct_start_offset"]
|
|
|
|
|
ranges = [[start, cell["struct_end_offset"]]]
|
|
|
|
|
cases = [
|
|
|
|
|
(38, b"\xff", "Invalid UTF-8"),
|
|
|
|
|
(start + 32, b"\xff", "Invalid UTF-8"),
|
|
|
|
|
]
|
|
|
|
|
cases += [
|
|
|
|
|
(offset, b"\x7f\xff", "Timestamp year")
|
|
|
|
|
for offset in (10, 22, start + 4, start + 16)
|
|
|
|
|
]
|
|
|
|
|
for offset, value, message in cases:
|
|
|
|
|
malformed = bytearray(data)
|
|
|
|
|
malformed[offset:offset + len(value)] = value
|
|
|
|
|
for read in (
|
|
|
|
|
arrow.read_bytes,
|
|
|
|
|
arrow.scan_bytes,
|
|
|
|
|
lambda payload: arrow.read_cells_bytes(payload, ranges),
|
|
|
|
|
):
|
|
|
|
|
with self.subTest(offset=offset, reader=read):
|
|
|
|
|
with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"):
|
|
|
|
|
read(malformed)
|
|
|
|
|
|
Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy
scans and selective cell loading. Validate numeric packing and array pitches,
and preserve normalized REAL8 encodings exactly within the supported range.
Add standalone dump, structural diff, hierarchy, layer-name annotation and Git
diff commands. Provide scoped text and JSON reports, deterministic comparison,
and record-preserving metadata edits.
Package the shared library and optional Python Arrow readers as pheon using
CFFI ABI mode and Python-independent platform wheels. Include locked source
builds, native API checks, cross-platform artifact workflows, documentation
and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer.
Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14,
and 71 installed companion integration tests pass. Full, scanned and selected
Arrow layouts match the previous schemas.
2026-11-16 16:21:21 -08:00
|
|
|
def test_import_failure_releases_both_native_outputs(self):
|
|
|
|
|
releases = []
|
|
|
|
|
|
|
|
|
|
@ffi.callback("void(struct ArrowArray *)")
|
|
|
|
|
def release_array(value):
|
|
|
|
|
releases.append("array")
|
|
|
|
|
value.release = ffi.NULL
|
|
|
|
|
|
|
|
|
|
@ffi.callback("void(struct ArrowSchema *)")
|
|
|
|
|
def release_schema(value):
|
|
|
|
|
releases.append("schema")
|
|
|
|
|
value.release = ffi.NULL
|
|
|
|
|
|
|
|
|
|
def export(array, schema):
|
|
|
|
|
array.release = release_array
|
|
|
|
|
schema.release = release_schema
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
def fail_import(*args):
|
|
|
|
|
raise ValueError("import failed")
|
|
|
|
|
|
|
|
|
|
with patch.object(
|
|
|
|
|
arrow,
|
|
|
|
|
"pyarrow",
|
|
|
|
|
SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)),
|
|
|
|
|
):
|
|
|
|
|
with self.assertRaisesRegex(ValueError, "import failed"):
|
|
|
|
|
arrow._export(export)
|
|
|
|
|
self.assertEqual(releases, ["array", "schema"])
|