fix(python): flatten packing buffers and package Cargo libraries

This commit is contained in:
Jan Petykiewicz 2026-11-15 20:01:26 -08:00
commit 912a267028
11 changed files with 355 additions and 9 deletions

5
.gitignore vendored
View file

@ -4,3 +4,8 @@ Cargo.lock
*.swp
*.swo
__pycache__/
/build/
/dist/
/*.egg-info/

6
MANIFEST.in Normal file
View file

@ -0,0 +1,6 @@
include Cargo.toml Cargo.lock README.md ARROW_SCHEMA.md
recursive-include src *.rs
recursive-include klamath_rs_ext *.py *.h py.typed
recursive-include tests *.rs *.py *.json *.txt *.mmd
recursive-include docs *.md *.json
recursive-include contrib *

View file

@ -58,6 +58,51 @@ range values must be aligned. Exported Arrow data owns its storage independently
of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan
schema, offset definitions, and ID assignment.
## Python package and validation
The Python packing helpers require NumPy and CFFI, both declared as runtime
dependencies. Setuptools invokes `cargo build --release --lib` and bundles the
C ABI shared library in a platform-specific wheel. Its version comes from
`Cargo.toml`. The C declarations are in `klamath_rs_ext/_ffi.h`; the loader uses
CFFI ABI mode, so no Python extension compilation or Maturin is required.
```sh
python3 -m pip install .
```
To build a source archive and a wheel with the `build` frontend and installed
Setuptools/Wheel tooling:
```sh
python3 -m build --no-isolation
```
Source archives include a Cargo lockfile, and wheel builds use `--locked` so
building from an archive retains the resolved Rust dependency versions. Wheels
are tagged for the build platform and use the Python-independent `py3-none` ABI.
The CLI continues to build directly with Cargo and has no Python dependency.
For source-tree development, point the CFFI loader at a Cargo-built library:
```sh
cargo build --release --lib
KLAMATH_RS_LIBRARY="$PWD/target/release/libklamath_rs_ext.so" \
python3 -m unittest discover -s tests/python -v
```
Use `libklamath_rs_ext.dylib` on macOS or `klamath_rs_ext.dll` on Windows. Without
the explicit override, the loader uses the library bundled inside the package.
After installing a wheel, run its tests outside the source checkout to exercise
the installed package:
```sh
python3 -I -m unittest discover -s /absolute/path/to/klamath-rs/tests/python -v
```
The Python tests cover multidimensional and noncontiguous packing, input
preservation, range errors, and the C declarations for Arrow exports and release
callbacks. Run `cargo test` and `cargo test --release` for native validation.
## Layout review CLI
The review CLI is a standalone Rust executable:

View file

@ -2,5 +2,5 @@ from .basic import pack_int2 as pack_int2
from .basic import pack_int4 as pack_int4
__version__ = 0.2
__version__ = "0.2.0"

50
klamath_rs_ext/_ffi.h Normal file
View file

@ -0,0 +1,50 @@
#ifndef KLAMATH_RS_EXT_H
#define KLAMATH_RS_EXT_H
#include <stdint.h>
#include <stddef.h>
struct ArrowArray {
int64_t length;
int64_t null_count;
int64_t offset;
int64_t n_buffers;
int64_t n_children;
const void **buffers;
struct ArrowArray **children;
struct ArrowArray *dictionary;
void (*release)(struct ArrowArray *);
void *private_data;
};
struct ArrowSchema {
const char *format;
const char *name;
const char *metadata;
int64_t flags;
int64_t n_children;
struct ArrowSchema **children;
struct ArrowSchema *dictionary;
void (*release)(struct ArrowSchema *);
void *private_data;
};
int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema);
int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema);
int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema);
const char *last_error_message(void);
double f64_to_i16(double *data, size_t size);
double f64_to_i32(double *data, size_t size);
float f32_to_i16(float *data, size_t size);
float f32_to_i32(float *data, size_t size);
int64_t i64_to_i16(int64_t *data, size_t size);
int64_t i64_to_i32(int64_t *data, size_t size);
int32_t i32_to_i16(int32_t *data, size_t size);
int32_t i32_to_i32(int32_t *data, size_t size);
int16_t i16_to_i16(int16_t *data, size_t size);
uint64_t u64_to_i16(uint64_t *data, size_t size);
uint64_t u64_to_i32(uint64_t *data, size_t size);
uint32_t u32_to_i16(uint32_t *data, size_t size);
uint32_t u32_to_i32(uint32_t *data, size_t size);
uint16_t u16_to_i16(uint16_t *data, size_t size);
#endif

View file

@ -49,7 +49,7 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes:
if result != 0:
raise ValueError(f'Invalid value for conversion to Int2: {result}')
i2arr = arr.view('>i2')[::arr.itemsize // 2]
i2arr = arr.reshape(-1).view('>i2')[::arr.itemsize // 2]
return i2arr.tobytes()
if arr.dtype == numpy.dtype('>i2'):
@ -79,7 +79,7 @@ def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes:
if result != 0:
raise ValueError(f'Invalid value for conversion to Int4: {result}')
i4arr = arr.view('>i4')[::arr.itemsize // 4]
i4arr = arr.reshape(-1).view('>i4')[::arr.itemsize // 4]
return i4arr.tobytes()
if arr.dtype == numpy.dtype('>i4'):

View file

@ -0,0 +1,23 @@
"""CFFI ABI access to the Cargo-built shared library."""
import os
from pathlib import Path
import sys
from cffi import FFI
ffi = FFI()
header = Path(__file__).with_name("_ffi.h").read_text()
ffi.cdef("\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")))
if sys.platform == "win32":
filename = "klamath_rs_ext.dll"
elif sys.platform == "darwin":
filename = "libklamath_rs_ext.dylib"
else:
filename = "libklamath_rs_ext.so"
# An explicit override supports source-tree development without copying binaries
# into the package. Installed wheels load only their bundled native library.
library = Path(os.environ["KLAMATH_RS_LIBRARY"]) if "KLAMATH_RS_LIBRARY" in os.environ else Path(__file__).with_name(filename)
lib = ffi.dlopen(str(library.resolve()))

View file

@ -1,6 +1,6 @@
[build-system]
requires = ["maturin>1.0,<2.0"]
build-backend = "maturin"
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "klamath_rs_ext"
@ -10,8 +10,6 @@ description = "Compiled extensions for klamath GDS library"
authors = [
{ name="Jan Petykiewicz", email="jan@mpxd.net" },
]
homepage = "https://mpxd.net/code/jan/klamath-rs"
repository = "https://mpxd.net/code/jan/klamath-rs"
classifiers = [
"Programming Language :: Python :: 3",
"Development Status :: 4 - Beta",
@ -30,8 +28,16 @@ requires-python = ">=3.11"
dynamic = ["version"]
dependencies = [
"cffi",
"numpy",
]
[tool.maturin]
bindings = "cffi"
[project.urls]
Homepage = "https://mpxd.net/code/jan/klamath-rs"
Repository = "https://mpxd.net/code/jan/klamath-rs"
[tool.setuptools]
packages = ["klamath_rs_ext"]
[tool.setuptools.package-data]
klamath_rs_ext = ["py.typed", "_ffi.h"]

76
setup.py Normal file
View file

@ -0,0 +1,76 @@
"""Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie."""
import json
from pathlib import Path
import shutil
import subprocess
import sys
import tomllib
from setuptools import Distribution, setup
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
from wheel.bdist_wheel import bdist_wheel
ROOT = Path(__file__).resolve().parent
VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"]
def ensure_lockfile():
if not (ROOT / "Cargo.lock").is_file():
subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True)
class CargoBuild(build_py):
def run(self):
ensure_lockfile()
super().run()
result = subprocess.run(
["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"],
cwd=ROOT, capture_output=True, text=True,
)
sys.stderr.write(result.stderr)
library = None
for line in result.stdout.splitlines():
try:
message = json.loads(line)
except json.JSONDecodeError:
print(line, file=sys.stderr)
continue
if message.get("reason") == "compiler-message":
sys.stderr.write(message["message"].get("rendered") or "")
if (message.get("reason") == "compiler-artifact"
and message["target"]["name"] == "klamath_rs_ext"
and "cdylib" in message["target"]["crate_types"]):
for filename in message["filenames"]:
if Path(filename).suffix in (".so", ".dylib", ".dll"):
library = Path(filename)
result.check_returncode()
if library is None:
raise RuntimeError("Cargo did not produce the klamath_rs_ext shared library")
shutil.copy2(library, Path(self.build_lib) / "klamath_rs_ext" / library.name)
class CargoSdist(sdist):
def run(self):
ensure_lockfile()
super().run()
class NativeDistribution(Distribution):
def has_ext_modules(self):
return True
class NativeWheel(bdist_wheel):
def get_tag(self):
# CFFI ABI mode needs the host platform, but no CPython extension ABI.
_, _, platform = super().get_tag()
return "py3", "none", platform
setup(
version=VERSION,
distclass=NativeDistribution,
cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel},
)

69
tests/python/test_ffi.py Normal file
View file

@ -0,0 +1,69 @@
"""Exercise the explicit C ABI declarations against the actual shared library."""
import struct
import tempfile
from pathlib import Path
import unittest
from klamath_rs_ext.klamath_rs_ext import ffi, lib
def record(tag, data=b""):
return struct.pack(">HH", len(data) + 4, tag) + data
def layout():
return b"".join([
record(0x0002, struct.pack(">h", 600)),
record(0x0102, b"\0" * 24),
record(0x0206, b"test"),
record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")),
record(0x0502, b"\0" * 24),
record(0x0606, b"cell"),
record(0x0700),
record(0x0400),
])
class FfiTests(unittest.TestCase):
def check_export(self, call):
array = ffi.new("struct ArrowArray *")
schema = ffi.new("struct ArrowSchema *")
try:
self.assertEqual(call(array, schema), 0)
self.assertEqual(lib.last_error_message(), ffi.NULL)
self.assertEqual(array.length, 1)
self.assertEqual(ffi.string(schema.format), b"+s")
self.assertNotEqual(array.release, ffi.NULL)
self.assertNotEqual(schema.release, ffi.NULL)
self.assertGreaterEqual(array.n_children, 7)
self.assertEqual(array.n_children, schema.n_children)
self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit")
finally:
if array.release != ffi.NULL:
array.release(array)
if schema.release != ffi.NULL:
schema.release(schema)
def test_scan_selected_read_and_path_exports(self):
data = layout()
self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s))
# The library header is 62 bytes; the one cell ends just before ENDLIB.
ranges = ffi.new("uint64_t[]", [62, len(data) - 4])
self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s))
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "file with spaces.gds"
path.write_bytes(data)
self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s))
def test_failures_leave_outputs_empty_and_expose_error_message(self):
array = ffi.new("struct ArrowArray *")
schema = ffi.new("struct ArrowSchema *")
self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1)
self.assertNotEqual(lib.last_error_message(), ffi.NULL)
self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message()))
self.assertEqual(array.release, ffi.NULL)
self.assertEqual(schema.release, ffi.NULL)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,66 @@
import unittest
import numpy as np
from klamath_rs_ext import pack_int2, pack_int4
class PackingTests(unittest.TestCase):
def test_scalars_lists_and_dimensions(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]:
for shape in [(), (1,), (2, 2), (2, 2, 2)]:
count = int(np.prod(shape)) if shape else 1
data = np.arange(count, dtype=dtype).reshape(shape)
before = data.copy()
with self.subTest(width=width, dtype=dtype, shape=shape):
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
np.testing.assert_array_equal(data, before)
for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]:
with self.subTest(width=width, data=data):
self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes())
def test_noncontiguous_readonly_and_endian_inputs(self):
original = np.arange(48, dtype=np.int64).reshape(6, 8)
readonly = original.copy()
readonly.setflags(write=False)
for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly,
original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]:
before = data.copy()
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
with self.subTest(width=width, strides=data.strides, dtype=data.dtype):
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C"))
np.testing.assert_array_equal(data, before)
def test_signed_limits_and_native_integer_widths(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1
data = np.array([[low, -1], [0, high]], dtype=np.int64)
self.assertEqual(pack(data), data.astype(f">i{width}").tobytes())
for dtype in [np.int8, np.uint8, np.int16, np.uint16]:
data = np.array([[1, 2], [3, 4]], dtype=dtype)
self.assertEqual(pack_int2(data), data.astype(">i2").tobytes())
self.assertEqual(pack_int4(data), data.astype(">i4").tobytes())
def test_failed_conversions_preserve_caller_data(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
limit = 1 << (8 * width - 1)
for invalid in [limit, -limit - 1]:
data = np.array([[1, 2], [3, invalid]], dtype=np.int64)
before = data.copy()
with self.subTest(width=width, invalid=invalid):
with self.assertRaises(ValueError):
pack(data)
np.testing.assert_array_equal(data, before)
for dtype in [np.float32, np.float64]:
for invalid in [np.nan, np.inf, -np.inf]:
data = np.array([[1, 2], [3, invalid]], dtype=dtype)
before = data.copy()
with self.subTest(width=width, invalid=invalid, dtype=dtype):
with self.assertRaises(ValueError):
pack(data)
np.testing.assert_array_equal(data, before)
if __name__ == "__main__":
unittest.main()