Introduce Pheon GDSII parsing and layout review tools
Implement native GDSII reading and writing with Arrow interchange, hierarchy scans and selective cell loading. Validate numeric packing and array pitches, and preserve normalized REAL8 encodings exactly within the supported range. Add standalone dump, structural diff, hierarchy, layer-name annotation and Git diff commands. Provide scoped text and JSON reports, deterministic comparison, and record-preserving metadata edits. Package the shared library and optional Python Arrow readers as pheon using CFFI ABI mode and Python-independent platform wheels. Include locked source builds, native API checks, cross-platform artifact workflows, documentation and regression coverage. Require Rust 1.88 or newer and Python 3.11 or newer. Validation: 87 native tests, 14 binding tests on Python 3.11/3.13/3.14, and 71 installed companion integration tests pass. Full, scanned and selected Arrow layouts match the previous schemas.
This commit is contained in:
parent
e0a634f476
commit
b3346b40a4
4 changed files with 12956 additions and 1 deletions
105
tests/python/test_arrow.py
Normal file
105
tests/python/test_arrow.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
"""Public Arrow readers, lifetime management and input validation."""
|
||||
|
||||
import gc
|
||||
import gzip
|
||||
import mmap
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy
|
||||
|
||||
from pheon import PheonError, arrow
|
||||
from pheon._native import ffi
|
||||
from test_ffi import layout
|
||||
|
||||
|
||||
class ArrowTests(unittest.TestCase):
|
||||
def test_full_scan_and_selected_reads_own_their_data(self):
|
||||
data = bytearray(layout())
|
||||
full = arrow.read_bytes(data)
|
||||
scan = arrow.scan_bytes(data)
|
||||
cells = scan.field("cells")[0].values
|
||||
ranges = list(
|
||||
zip(
|
||||
cells.field("struct_start_offset").to_pylist(),
|
||||
cells.field("struct_end_offset").to_pylist(),
|
||||
)
|
||||
)
|
||||
selected = arrow.read_cells_bytes(data, ranges)
|
||||
data[:] = b"\0" * len(data)
|
||||
del data
|
||||
gc.collect()
|
||||
self.assertEqual(full.to_pylist(), selected.to_pylist())
|
||||
self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"])
|
||||
self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"])
|
||||
|
||||
def test_paths_gzip_magic_and_closed_mmap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "layout with spaces.gds"
|
||||
path.write_bytes(layout())
|
||||
expected = arrow.read_path(path).to_pylist()
|
||||
with path.open("rb") as stream:
|
||||
with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
|
||||
result = arrow.read_bytes(buffer)
|
||||
self.assertEqual(result.to_pylist(), expected)
|
||||
path.write_bytes(gzip.compress(layout()))
|
||||
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
||||
|
||||
def test_empty_and_noncontiguous_cell_selections(self):
|
||||
data = layout()
|
||||
empty = arrow.read_cells_bytes(data, [])
|
||||
self.assertEqual(empty.field("cells")[0].as_py(), [])
|
||||
values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2]
|
||||
self.assertFalse(values.flags.c_contiguous)
|
||||
self.assertEqual(
|
||||
arrow.read_cells_bytes(data, values).to_pylist(),
|
||||
arrow.read_bytes(data).to_pylist(),
|
||||
)
|
||||
|
||||
def test_bad_inputs_fail_before_importing_arrow(self):
|
||||
with self.assertRaises(PheonError):
|
||||
arrow.read_bytes(b"invalid")
|
||||
with self.assertRaises(PheonError):
|
||||
arrow.scan_bytes(b"")
|
||||
with self.assertRaises(PheonError):
|
||||
arrow.read_cells_bytes(layout(), [[0, 10]])
|
||||
with self.assertRaises(ValueError):
|
||||
arrow.read_bytes(memoryview(layout())[::2])
|
||||
for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]:
|
||||
with self.assertRaises(ValueError):
|
||||
arrow.read_cells_bytes(layout(), ranges)
|
||||
with self.assertRaises(TypeError):
|
||||
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
|
||||
|
||||
def test_import_failure_releases_both_native_outputs(self):
|
||||
releases = []
|
||||
|
||||
@ffi.callback("void(struct ArrowArray *)")
|
||||
def release_array(value):
|
||||
releases.append("array")
|
||||
value.release = ffi.NULL
|
||||
|
||||
@ffi.callback("void(struct ArrowSchema *)")
|
||||
def release_schema(value):
|
||||
releases.append("schema")
|
||||
value.release = ffi.NULL
|
||||
|
||||
def export(array, schema):
|
||||
array.release = release_array
|
||||
schema.release = release_schema
|
||||
return 0
|
||||
|
||||
def fail_import(*args):
|
||||
raise ValueError("import failed")
|
||||
|
||||
with patch.object(
|
||||
arrow,
|
||||
"pyarrow",
|
||||
SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)),
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "import failed"):
|
||||
arrow._export(export)
|
||||
self.assertEqual(releases, ["array", "schema"])
|
||||
70
tests/python/test_ffi.py
Normal file
70
tests/python/test_ffi.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Exercise the explicit C ABI declarations against the actual shared library."""
|
||||
import struct
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from pheon._native import ffi, lib
|
||||
|
||||
|
||||
def record(tag, data=b""):
|
||||
return struct.pack(">HH", len(data) + 4, tag) + data
|
||||
|
||||
|
||||
def layout():
|
||||
return b"".join([
|
||||
record(0x0002, struct.pack(">h", 600)),
|
||||
record(0x0102, b"\0" * 24),
|
||||
record(0x0206, b"test"),
|
||||
record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")),
|
||||
record(0x0502, b"\0" * 24),
|
||||
record(0x0606, b"cell"),
|
||||
record(0x0700),
|
||||
record(0x0400),
|
||||
])
|
||||
|
||||
|
||||
class FfiTests(unittest.TestCase):
|
||||
def check_export(self, call):
|
||||
array = ffi.new("struct ArrowArray *")
|
||||
schema = ffi.new("struct ArrowSchema *")
|
||||
try:
|
||||
self.assertEqual(call(array, schema), 0)
|
||||
self.assertEqual(lib.last_error_message(), ffi.NULL)
|
||||
self.assertEqual(array.length, 1)
|
||||
self.assertEqual(ffi.string(schema.format), b"+s")
|
||||
self.assertNotEqual(array.release, ffi.NULL)
|
||||
self.assertNotEqual(schema.release, ffi.NULL)
|
||||
self.assertGreaterEqual(array.n_children, 7)
|
||||
self.assertEqual(array.n_children, schema.n_children)
|
||||
self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit")
|
||||
finally:
|
||||
if array.release != ffi.NULL:
|
||||
array.release(array)
|
||||
if schema.release != ffi.NULL:
|
||||
schema.release(schema)
|
||||
|
||||
def test_scan_selected_read_and_path_exports(self):
|
||||
data = layout()
|
||||
self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s))
|
||||
self.check_export(lambda a, s: lib.read_bytes(data, len(data), a, s))
|
||||
# The library header is 62 bytes; the one cell ends just before ENDLIB.
|
||||
ranges = ffi.new("uint64_t[]", [62, len(data) - 4])
|
||||
self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "file with spaces.gds"
|
||||
path.write_bytes(data)
|
||||
self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s))
|
||||
|
||||
def test_failures_leave_outputs_empty_and_expose_error_message(self):
|
||||
array = ffi.new("struct ArrowArray *")
|
||||
schema = ffi.new("struct ArrowSchema *")
|
||||
self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1)
|
||||
self.assertNotEqual(lib.last_error_message(), ffi.NULL)
|
||||
self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message()))
|
||||
self.assertEqual(array.release, ffi.NULL)
|
||||
self.assertEqual(schema.release, ffi.NULL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
36
tests/python/test_loader.py
Normal file
36
tests/python/test_loader.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""Reject incompatible native libraries before exposing callable functions."""
|
||||
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pheon._native
|
||||
|
||||
|
||||
class LoaderTests(unittest.TestCase):
|
||||
def load(self, library):
|
||||
with patch("cffi.FFI.dlopen", return_value=library):
|
||||
return runpy.run_path(str(Path(pheon._native.__file__)))
|
||||
|
||||
def test_matching_versions_are_accepted(self):
|
||||
self.load(
|
||||
SimpleNamespace(
|
||||
pheon_abi_version=lambda: 1, pheon_arrow_schema_version=lambda: 1
|
||||
)
|
||||
)
|
||||
|
||||
def test_incompatible_abi_or_schema_is_rejected(self):
|
||||
for abi, schema in [(2, 1), (1, 2)]:
|
||||
with self.assertRaisesRegex(ImportError, "Incompatible Pheon library"):
|
||||
self.load(
|
||||
SimpleNamespace(
|
||||
pheon_abi_version=lambda: abi,
|
||||
pheon_arrow_schema_version=lambda: schema,
|
||||
)
|
||||
)
|
||||
|
||||
def test_unversioned_library_is_rejected(self):
|
||||
with self.assertRaisesRegex(ImportError, "version API"):
|
||||
self.load(SimpleNamespace())
|
||||
66
tests/python/test_packing.py
Normal file
66
tests/python/test_packing.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from pheon import pack_int2, pack_int4
|
||||
|
||||
|
||||
class PackingTests(unittest.TestCase):
|
||||
def test_scalars_lists_and_dimensions(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]:
|
||||
for shape in [(), (1,), (2, 2), (2, 2, 2)]:
|
||||
count = int(np.prod(shape)) if shape else 1
|
||||
data = np.arange(count, dtype=dtype).reshape(shape)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, dtype=dtype, shape=shape):
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
|
||||
np.testing.assert_array_equal(data, before)
|
||||
for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]:
|
||||
with self.subTest(width=width, data=data):
|
||||
self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes())
|
||||
|
||||
def test_noncontiguous_readonly_and_endian_inputs(self):
|
||||
original = np.arange(48, dtype=np.int64).reshape(6, 8)
|
||||
readonly = original.copy()
|
||||
readonly.setflags(write=False)
|
||||
for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly,
|
||||
original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]:
|
||||
before = data.copy()
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
with self.subTest(width=width, strides=data.strides, dtype=data.dtype):
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C"))
|
||||
np.testing.assert_array_equal(data, before)
|
||||
|
||||
def test_signed_limits_and_native_integer_widths(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1
|
||||
data = np.array([[low, -1], [0, high]], dtype=np.int64)
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
|
||||
for dtype in [np.int8, np.uint8, np.int16, np.uint16]:
|
||||
data = np.array([[1, 2], [3, 4]], dtype=dtype)
|
||||
self.assertEqual(pack_int2(data), data.astype(">i2").tobytes())
|
||||
self.assertEqual(pack_int4(data), data.astype(">i4").tobytes())
|
||||
|
||||
def test_failed_conversions_preserve_caller_data(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
limit = 1 << (8 * width - 1)
|
||||
for invalid in [limit, -limit - 1]:
|
||||
data = np.array([[1, 2], [3, invalid]], dtype=np.int64)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, invalid=invalid):
|
||||
with self.assertRaises(ValueError):
|
||||
pack(data)
|
||||
np.testing.assert_array_equal(data, before)
|
||||
for dtype in [np.float32, np.float64]:
|
||||
for invalid in [np.nan, np.inf, -np.inf]:
|
||||
data = np.array([[1, 2], [3, invalid]], dtype=dtype)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, invalid=invalid, dtype=dtype):
|
||||
with self.assertRaises(ValueError):
|
||||
pack(data)
|
||||
np.testing.assert_array_equal(data, before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in a new issue