pheon/tests/python/test_arrow.py
Jan Petykiewicz 31afef0529 Reject invalid record sizes and return malformed-input errors
Validate payload sizes before record/header writes and make header packing
return io::Result when the total size overflows the format limit.

Return located parse errors for invalid UTF-8 and timestamp year overflow.
Validate cell timestamps consistently in full reads, selected reads, and
hierarchy scans while retaining scan behavior for skipped text geometry.

Add regression coverage for preserved writer outputs, padded metadata sizes,
malformed strings, timestamp boundaries, bounded errors, and Python bindings.
Document the fallible header packing API.

Validation: all 97 Rust tests pass in debug and release; all 16 Python tests
pass against the rebuilt native library.
2026-11-16 21:28:54 -08:00

141 lines
5.4 KiB
Python

"""Public Arrow readers, lifetime management and input validation."""
import gc
import gzip
import io
import mmap
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy
from pheon import PheonError, arrow
from pheon._native import ffi
from test_ffi import layout
class ArrowTests(unittest.TestCase):
def test_full_scan_and_selected_reads_own_their_data(self):
data = bytearray(layout())
full = arrow.read_bytes(data)
scan = arrow.scan_bytes(data)
cells = scan.field("cells")[0].values
ranges = list(
zip(
cells.field("struct_start_offset").to_pylist(),
cells.field("struct_end_offset").to_pylist(),
)
)
selected = arrow.read_cells_bytes(data, ranges)
data[:] = b"\0" * len(data)
del data
gc.collect()
self.assertEqual(full.to_pylist(), selected.to_pylist())
self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"])
self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"])
def test_paths_gzip_magic_and_closed_mmap(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "layout with spaces.gds"
path.write_bytes(layout())
expected = arrow.read_path(path).to_pylist()
with path.open("rb") as stream:
with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
result = arrow.read_bytes(buffer)
self.assertEqual(result.to_pylist(), expected)
path.write_bytes(gzip.compress(layout()))
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
def test_path_reader_keeps_using_the_opened_stream(self):
for payload in (layout(), gzip.compress(layout())):
with self.subTest(compressed=payload.startswith(b"\x1f\x8b")):
source = io.BytesIO(payload)
with patch("builtins.open", return_value=source) as opened:
result = arrow.read_path("opened-stream-only.gds")
opened.assert_called_once_with("opened-stream-only.gds", "rb")
self.assertTrue(source.closed)
self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"])
def test_empty_and_noncontiguous_cell_selections(self):
data = layout()
empty = arrow.read_cells_bytes(data, [])
self.assertEqual(empty.field("cells")[0].as_py(), [])
values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2]
self.assertFalse(values.flags.c_contiguous)
self.assertEqual(
arrow.read_cells_bytes(data, values).to_pylist(),
arrow.read_bytes(data).to_pylist(),
)
def test_bad_inputs_fail_before_importing_arrow(self):
with self.assertRaises(PheonError):
arrow.read_bytes(b"invalid")
with self.assertRaises(PheonError):
arrow.scan_bytes(b"")
with self.assertRaises(PheonError):
arrow.read_cells_bytes(layout(), [[0, 10]])
with self.assertRaises(ValueError):
arrow.read_bytes(memoryview(layout())[::2])
for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]:
with self.assertRaises(ValueError):
arrow.read_cells_bytes(layout(), ranges)
with self.assertRaises(TypeError):
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
def test_malformed_strings_and_years_raise_located_parse_errors(self):
data = layout()
cell = arrow.scan_bytes(data).field("cells")[0].as_py()[0]
start = cell["struct_start_offset"]
ranges = [[start, cell["struct_end_offset"]]]
cases = [
(38, b"\xff", "Invalid UTF-8"),
(start + 32, b"\xff", "Invalid UTF-8"),
]
cases += [
(offset, b"\x7f\xff", "Timestamp year")
for offset in (10, 22, start + 4, start + 16)
]
for offset, value, message in cases:
malformed = bytearray(data)
malformed[offset:offset + len(value)] = value
for read in (
arrow.read_bytes,
arrow.scan_bytes,
lambda payload: arrow.read_cells_bytes(payload, ranges),
):
with self.subTest(offset=offset, reader=read):
with self.assertRaisesRegex(PheonError, rf"at byte \d+: {message}"):
read(malformed)
def test_import_failure_releases_both_native_outputs(self):
releases = []
@ffi.callback("void(struct ArrowArray *)")
def release_array(value):
releases.append("array")
value.release = ffi.NULL
@ffi.callback("void(struct ArrowSchema *)")
def release_schema(value):
releases.append("schema")
value.release = ffi.NULL
def export(array, schema):
array.release = release_array
schema.release = release_schema
return 0
def fail_import(*args):
raise ValueError("import failed")
with patch.object(
arrow,
"pyarrow",
SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)),
):
with self.assertRaisesRegex(ValueError, "import failed"):
arrow._export(export)
self.assertEqual(releases, ["array", "schema"])