105 lines
3.8 KiB
Python
105 lines
3.8 KiB
Python
|
|
"""Public Arrow readers, lifetime management and input validation."""
|
||
|
|
|
||
|
|
import gc
|
||
|
|
import gzip
|
||
|
|
import mmap
|
||
|
|
from pathlib import Path
|
||
|
|
import tempfile
|
||
|
|
from types import SimpleNamespace
|
||
|
|
import unittest
|
||
|
|
from unittest.mock import patch
|
||
|
|
|
||
|
|
import numpy
|
||
|
|
|
||
|
|
from pheon import PheonError, arrow
|
||
|
|
from pheon._native import ffi
|
||
|
|
from test_ffi import layout
|
||
|
|
|
||
|
|
|
||
|
|
class ArrowTests(unittest.TestCase):
|
||
|
|
def test_full_scan_and_selected_reads_own_their_data(self):
|
||
|
|
data = bytearray(layout())
|
||
|
|
full = arrow.read_bytes(data)
|
||
|
|
scan = arrow.scan_bytes(data)
|
||
|
|
cells = scan.field("cells")[0].values
|
||
|
|
ranges = list(
|
||
|
|
zip(
|
||
|
|
cells.field("struct_start_offset").to_pylist(),
|
||
|
|
cells.field("struct_end_offset").to_pylist(),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
selected = arrow.read_cells_bytes(data, ranges)
|
||
|
|
data[:] = b"\0" * len(data)
|
||
|
|
del data
|
||
|
|
gc.collect()
|
||
|
|
self.assertEqual(full.to_pylist(), selected.to_pylist())
|
||
|
|
self.assertEqual(full.field("cell_names")[0].as_py(), ["cell"])
|
||
|
|
self.assertEqual(scan.field("cell_names")[0].as_py(), ["cell"])
|
||
|
|
|
||
|
|
def test_paths_gzip_magic_and_closed_mmap(self):
|
||
|
|
with tempfile.TemporaryDirectory() as directory:
|
||
|
|
path = Path(directory) / "layout with spaces.gds"
|
||
|
|
path.write_bytes(layout())
|
||
|
|
expected = arrow.read_path(path).to_pylist()
|
||
|
|
with path.open("rb") as stream:
|
||
|
|
with mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
|
||
|
|
result = arrow.read_bytes(buffer)
|
||
|
|
self.assertEqual(result.to_pylist(), expected)
|
||
|
|
path.write_bytes(gzip.compress(layout()))
|
||
|
|
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
||
|
|
|
||
|
|
def test_empty_and_noncontiguous_cell_selections(self):
|
||
|
|
data = layout()
|
||
|
|
empty = arrow.read_cells_bytes(data, [])
|
||
|
|
self.assertEqual(empty.field("cells")[0].as_py(), [])
|
||
|
|
values = numpy.array([[62, 0, len(data) - 4, 0]], dtype=numpy.uint64)[:, ::2]
|
||
|
|
self.assertFalse(values.flags.c_contiguous)
|
||
|
|
self.assertEqual(
|
||
|
|
arrow.read_cells_bytes(data, values).to_pylist(),
|
||
|
|
arrow.read_bytes(data).to_pylist(),
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_bad_inputs_fail_before_importing_arrow(self):
|
||
|
|
with self.assertRaises(PheonError):
|
||
|
|
arrow.read_bytes(b"invalid")
|
||
|
|
with self.assertRaises(PheonError):
|
||
|
|
arrow.scan_bytes(b"")
|
||
|
|
with self.assertRaises(PheonError):
|
||
|
|
arrow.read_cells_bytes(layout(), [[0, 10]])
|
||
|
|
with self.assertRaises(ValueError):
|
||
|
|
arrow.read_bytes(memoryview(layout())[::2])
|
||
|
|
for ranges in [[62, 100], [[-1, 100]], [[1, 2, 3]]]:
|
||
|
|
with self.assertRaises(ValueError):
|
||
|
|
arrow.read_cells_bytes(layout(), ranges)
|
||
|
|
with self.assertRaises(TypeError):
|
||
|
|
arrow.read_cells_bytes(layout(), [[1.5, 2.5]])
|
||
|
|
|
||
|
|
def test_import_failure_releases_both_native_outputs(self):
|
||
|
|
releases = []
|
||
|
|
|
||
|
|
@ffi.callback("void(struct ArrowArray *)")
|
||
|
|
def release_array(value):
|
||
|
|
releases.append("array")
|
||
|
|
value.release = ffi.NULL
|
||
|
|
|
||
|
|
@ffi.callback("void(struct ArrowSchema *)")
|
||
|
|
def release_schema(value):
|
||
|
|
releases.append("schema")
|
||
|
|
value.release = ffi.NULL
|
||
|
|
|
||
|
|
def export(array, schema):
|
||
|
|
array.release = release_array
|
||
|
|
schema.release = release_schema
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def fail_import(*args):
|
||
|
|
raise ValueError("import failed")
|
||
|
|
|
||
|
|
with patch.object(
|
||
|
|
arrow,
|
||
|
|
"pyarrow",
|
||
|
|
SimpleNamespace(Array=SimpleNamespace(_import_from_c=fail_import)),
|
||
|
|
):
|
||
|
|
with self.assertRaisesRegex(ValueError, "import failed"):
|
||
|
|
arrow._export(export)
|
||
|
|
self.assertEqual(releases, ["array", "schema"])
|