fix(python): flatten packing buffers and package Cargo libraries
This commit is contained in:
parent
f081a808ea
commit
912a267028
2 changed files with 355 additions and 9 deletions
69
tests/python/test_ffi.py
Normal file
69
tests/python/test_ffi.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""Exercise the explicit C ABI declarations against the actual shared library."""
|
||||
import struct
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from klamath_rs_ext.klamath_rs_ext import ffi, lib
|
||||
|
||||
|
||||
def record(tag, data=b""):
|
||||
return struct.pack(">HH", len(data) + 4, tag) + data
|
||||
|
||||
|
||||
def layout():
|
||||
return b"".join([
|
||||
record(0x0002, struct.pack(">h", 600)),
|
||||
record(0x0102, b"\0" * 24),
|
||||
record(0x0206, b"test"),
|
||||
record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")),
|
||||
record(0x0502, b"\0" * 24),
|
||||
record(0x0606, b"cell"),
|
||||
record(0x0700),
|
||||
record(0x0400),
|
||||
])
|
||||
|
||||
|
||||
class FfiTests(unittest.TestCase):
|
||||
def check_export(self, call):
|
||||
array = ffi.new("struct ArrowArray *")
|
||||
schema = ffi.new("struct ArrowSchema *")
|
||||
try:
|
||||
self.assertEqual(call(array, schema), 0)
|
||||
self.assertEqual(lib.last_error_message(), ffi.NULL)
|
||||
self.assertEqual(array.length, 1)
|
||||
self.assertEqual(ffi.string(schema.format), b"+s")
|
||||
self.assertNotEqual(array.release, ffi.NULL)
|
||||
self.assertNotEqual(schema.release, ffi.NULL)
|
||||
self.assertGreaterEqual(array.n_children, 7)
|
||||
self.assertEqual(array.n_children, schema.n_children)
|
||||
self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit")
|
||||
finally:
|
||||
if array.release != ffi.NULL:
|
||||
array.release(array)
|
||||
if schema.release != ffi.NULL:
|
||||
schema.release(schema)
|
||||
|
||||
def test_scan_selected_read_and_path_exports(self):
|
||||
data = layout()
|
||||
self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s))
|
||||
# The library header is 62 bytes; the one cell ends just before ENDLIB.
|
||||
ranges = ffi.new("uint64_t[]", [62, len(data) - 4])
|
||||
self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "file with spaces.gds"
|
||||
path.write_bytes(data)
|
||||
self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s))
|
||||
|
||||
def test_failures_leave_outputs_empty_and_expose_error_message(self):
|
||||
array = ffi.new("struct ArrowArray *")
|
||||
schema = ffi.new("struct ArrowSchema *")
|
||||
self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1)
|
||||
self.assertNotEqual(lib.last_error_message(), ffi.NULL)
|
||||
self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message()))
|
||||
self.assertEqual(array.release, ffi.NULL)
|
||||
self.assertEqual(schema.release, ffi.NULL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
66
tests/python/test_packing.py
Normal file
66
tests/python/test_packing.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from klamath_rs_ext import pack_int2, pack_int4
|
||||
|
||||
|
||||
class PackingTests(unittest.TestCase):
|
||||
def test_scalars_lists_and_dimensions(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]:
|
||||
for shape in [(), (1,), (2, 2), (2, 2, 2)]:
|
||||
count = int(np.prod(shape)) if shape else 1
|
||||
data = np.arange(count, dtype=dtype).reshape(shape)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, dtype=dtype, shape=shape):
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
|
||||
np.testing.assert_array_equal(data, before)
|
||||
for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]:
|
||||
with self.subTest(width=width, data=data):
|
||||
self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes())
|
||||
|
||||
def test_noncontiguous_readonly_and_endian_inputs(self):
|
||||
original = np.arange(48, dtype=np.int64).reshape(6, 8)
|
||||
readonly = original.copy()
|
||||
readonly.setflags(write=False)
|
||||
for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly,
|
||||
original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]:
|
||||
before = data.copy()
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
with self.subTest(width=width, strides=data.strides, dtype=data.dtype):
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C"))
|
||||
np.testing.assert_array_equal(data, before)
|
||||
|
||||
def test_signed_limits_and_native_integer_widths(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1
|
||||
data = np.array([[low, -1], [0, high]], dtype=np.int64)
|
||||
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
|
||||
for dtype in [np.int8, np.uint8, np.int16, np.uint16]:
|
||||
data = np.array([[1, 2], [3, 4]], dtype=dtype)
|
||||
self.assertEqual(pack_int2(data), data.astype(">i2").tobytes())
|
||||
self.assertEqual(pack_int4(data), data.astype(">i4").tobytes())
|
||||
|
||||
def test_failed_conversions_preserve_caller_data(self):
|
||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||
limit = 1 << (8 * width - 1)
|
||||
for invalid in [limit, -limit - 1]:
|
||||
data = np.array([[1, 2], [3, invalid]], dtype=np.int64)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, invalid=invalid):
|
||||
with self.assertRaises(ValueError):
|
||||
pack(data)
|
||||
np.testing.assert_array_equal(data, before)
|
||||
for dtype in [np.float32, np.float64]:
|
||||
for invalid in [np.nan, np.inf, -np.inf]:
|
||||
data = np.array([[1, 2], [3, invalid]], dtype=dtype)
|
||||
before = data.copy()
|
||||
with self.subTest(width=width, invalid=invalid, dtype=dtype):
|
||||
with self.assertRaises(ValueError):
|
||||
pack(data)
|
||||
np.testing.assert_array_equal(data, before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in a new issue