Read each input path through one opened stream, including gzip detection, and use direct numeric converter lookup while preserving byte-order handling and caller data. Read the package version from installed metadata and report range errors consistently as ValueError. Require the supplied Cargo.lock instead of generating it during builds, use the standard sdist command, and update package checks and development guides. Validation: 15 Python tests pass against source and the rebuilt installed wheel; the source archive and platform wheel build successfully.
116 lines
4.4 KiB
Python
116 lines
4.4 KiB
Python
"""Public Arrow readers, lifetime management and input validation."""
|
|
|
|
import gc
|
|
import gzip
|
|
import io
|
|
import mmap
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy
|
|
|
|
from pheon import PheonError, arrow
|
|
from pheon._native import ffi
|
|
from test_ffi import layout
|
|
|
|
|
|
class ArrowTests(unittest.TestCase):
|
|
def test_full_scan_and_selected_reads_own_their_data(self):
|
|
data = bytearray(layout())
|
|
full = arrow.read_bytes(data)
|
|
scan = arrow.scan_bytes(data)
|
|
cells = scan.field("cells")[0].values
|
|
ranges = list(
|
|
zip(
|
|
cells.field("struct_start_offset").to_pylist(),
|
|
cells.field("struct_end_offset").to_pylist(),
|
|
)
|
|
)
|
|
selected = arrow.read_cells_bytes(data, ranges)
|
|
data[:] = b"\0" * len(data)
|
|
del data
|
|
gc.collect()
|
|
self.assertEqual(full.to_pylist(), selected.to_pylist())
|
|
self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"])
|
|
self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"])
|
|
|
|
def test_paths_gzip_magic_and_closed_mmap(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / "layout with spaces.gds"
|
|
path.write_bytes(layout())
|
|
expected = arrow.read_path(path).to_pylist()
|
|
with path.open("rb") as stream:
|
|
with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
|
|
result = arrow.read_bytes(buffer)
|
|
self.assertEqual(result.to_pylist(), expected)
|
|
path.write_bytes(gzip.compress(layout()))
|
|
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
|
|
|
def test_path_reader_keeps_using_the_opened_stream(self):
|
|
for payload in (layout(), gzip.compress(layout())):
|
|
with self.subTest(compressed=payload.startswith(b"\x1f\x8b")):
|
|
source = io.BytesIO(payload)
|
|
with patch("builtins.open", return_value=source) as opened:
|
|
result = arrow.read_path("opened-stream-only.gds")
|
|
opened.assert_called_once_with("opened-stream-only.gds", "rb")
|
|
self.assertTrue(source.closed)
|
|
self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"])
|
|
|
|
def test_empty_and_noncontiguous_cell_selections(self):
|
|
data = layout()
|
|
empty = arrow.read_cells_bytes(data, [])
|
|
self.assertEqual(empty.field("cells")[0].as_py(), [])
|
|
values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2]
|
|
self.assertFalse(values.flags.c_contiguous)
|
|
self.assertEqual(
|
|
arrow.read_cells_bytes(data, values).to_pylist(),
|
|
arrow.read_bytes(data).to_pylist(),
|
|
)
|
|
|
|
def test_bad_inputs_fail_before_importing_arrow(self):
|
|
with self.assertRaises(PheonError):
|
|
arrow.read_bytes(b"invalid")
|
|
with self.assertRaises(PheonError):
|
|
arrow.scan_bytes(b"")
|
|
with self.assertRaises(PheonError):
|
|
arrow.read_cells_bytes(layout(), [[0, 10]])
|
|
with self.assertRaises(ValueError):
|
|
arrow.read_bytes(memoryview(layout())[::2])
|
|
for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]:
|
|
with self.assertRaises(ValueError):
|
|
arrow.read_cells_bytes(layout(), ranges)
|
|
with self.assertRaises(TypeError):
|
|
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
|
|
|
|
def test_import_failure_releases_both_native_outputs(self):
|
|
releases = []
|
|
|
|
@ffi.callback("void(struct ArrowArray *)")
|
|
def release_array(value):
|
|
releases.append("array")
|
|
value.release = ffi.NULL
|
|
|
|
@ffi.callback("void(struct ArrowSchema *)")
|
|
def release_schema(value):
|
|
releases.append("schema")
|
|
value.release = ffi.NULL
|
|
|
|
def export(array, schema):
|
|
array.release = release_array
|
|
schema.release = release_schema
|
|
return 0
|
|
|
|
def fail_import(*args):
|
|
raise ValueError("import failed")
|
|
|
|
with patch.object(
|
|
arrow,
|
|
"pyarrow",
|
|
SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)),
|
|
):
|
|
with self.assertRaisesRegex(ValueError, "import failed"):
|
|
arrow._export(export)
|
|
self.assertEqual(releases, ["array", "schema"])
|