fix(python): flatten packing buffers and package Cargo libraries

This commit is contained in:
Jan Petykiewicz 2026-11-15 20:01:26 -08:00
commit 912a267028
2 changed files with 355 additions and 9 deletions

69
tests/python/test_ffi.py Normal file
View file

@ -0,0 +1,69 @@
"""Exercise the explicit C ABI declarations against the actual shared library."""
import struct
import tempfile
from pathlib import Path
import unittest
from klamath_rs_ext.klamath_rs_ext import ffi, lib
def record(tag, data=b""):
return struct.pack(">HH", len(data) + 4, tag) + data
def layout():
return b"".join([
record(0x0002, struct.pack(">h", 600)),
record(0x0102, b"\0" * 24),
record(0x0206, b"test"),
record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")),
record(0x0502, b"\0" * 24),
record(0x0606, b"cell"),
record(0x0700),
record(0x0400),
])
class FfiTests(unittest.TestCase):
def check_export(self, call):
array = ffi.new("struct ArrowArray *")
schema = ffi.new("struct ArrowSchema *")
try:
self.assertEqual(call(array, schema), 0)
self.assertEqual(lib.last_error_message(), ffi.NULL)
self.assertEqual(array.length, 1)
self.assertEqual(ffi.string(schema.format), b"+s")
self.assertNotEqual(array.release, ffi.NULL)
self.assertNotEqual(schema.release, ffi.NULL)
self.assertGreaterEqual(array.n_children, 7)
self.assertEqual(array.n_children, schema.n_children)
self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit")
finally:
if array.release != ffi.NULL:
array.release(array)
if schema.release != ffi.NULL:
schema.release(schema)
def test_scan_selected_read_and_path_exports(self):
data = layout()
self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s))
# The library header is 62 bytes; the one cell ends just before ENDLIB.
ranges = ffi.new("uint64_t[]", [62, len(data) - 4])
self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s))
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "file with spaces.gds"
path.write_bytes(data)
self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s))
def test_failures_leave_outputs_empty_and_expose_error_message(self):
array = ffi.new("struct ArrowArray *")
schema = ffi.new("struct ArrowSchema *")
self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1)
self.assertNotEqual(lib.last_error_message(), ffi.NULL)
self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message()))
self.assertEqual(array.release, ffi.NULL)
self.assertEqual(schema.release, ffi.NULL)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,66 @@
import unittest
import numpy as np
from klamath_rs_ext import pack_int2, pack_int4
class PackingTests(unittest.TestCase):
def test_scalars_lists_and_dimensions(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]:
for shape in [(), (1,), (2, 2), (2, 2, 2)]:
count = int(np.prod(shape)) if shape else 1
data = np.arange(count, dtype=dtype).reshape(shape)
before = data.copy()
with self.subTest(width=width, dtype=dtype, shape=shape):
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
np.testing.assert_array_equal(data, before)
for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]:
with self.subTest(width=width, data=data):
self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes())
def test_noncontiguous_readonly_and_endian_inputs(self):
original = np.arange(48, dtype=np.int64).reshape(6, 8)
readonly = original.copy()
readonly.setflags(write=False)
for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly,
original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]:
before = data.copy()
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
with self.subTest(width=width, strides=data.strides, dtype=data.dtype):
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C"))
np.testing.assert_array_equal(data, before)
def test_signed_limits_and_native_integer_widths(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1
data = np.array([[low, -1], [0, high]], dtype=np.int64)
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
for dtype in [np.int8, np.uint8, np.int16, np.uint16]:
data = np.array([[1, 2], [3, 4]], dtype=dtype)
self.assertEqual(pack_int2(data), data.astype(">i2").tobytes())
self.assertEqual(pack_int4(data), data.astype(">i4").tobytes())
def test_failed_conversions_preserve_caller_data(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
limit = 1 << (8 * width - 1)
for invalid in [limit, -limit - 1]:
data = np.array([[1, 2], [3, invalid]], dtype=np.int64)
before = data.copy()
with self.subTest(width=width, invalid=invalid):
with self.assertRaises(ValueError):
pack(data)
np.testing.assert_array_equal(data, before)
for dtype in [np.float32, np.float64]:
for invalid in [np.nan, np.inf, -np.inf]:
data = np.array([[1, 2], [3, invalid]], dtype=dtype)
before = data.copy()
with self.subTest(width=width, invalid=invalid, dtype=dtype):
with self.assertRaises(ValueError):
pack(data)
np.testing.assert_array_equal(data, before)
if __name__ == "__main__":
unittest.main()