From 912a267028f305b6e2210a2d3e32e2087be8283f Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sun, 15 Nov 2026 20:01:26 -0800 Subject: [PATCH] fix(python): flatten packing buffers and package Cargo libraries --- .gitignore | 5 +++ MANIFEST.in | 6 +++ README.md | 45 +++++++++++++++++++ klamath_rs_ext/__init__.py | 2 +- klamath_rs_ext/_ffi.h | 50 +++++++++++++++++++++ klamath_rs_ext/basic.py | 4 +- klamath_rs_ext/klamath_rs_ext.py | 23 ++++++++++ pyproject.toml | 18 +++++--- setup.py | 76 ++++++++++++++++++++++++++++++++ tests/python/test_ffi.py | 69 +++++++++++++++++++++++++++++ tests/python/test_packing.py | 66 +++++++++++++++++++++++++++ 11 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 MANIFEST.in create mode 100644 klamath_rs_ext/_ffi.h create mode 100644 klamath_rs_ext/klamath_rs_ext.py create mode 100644 setup.py create mode 100644 tests/python/test_ffi.py create mode 100644 tests/python/test_packing.py diff --git a/.gitignore b/.gitignore index 5d80173..ed58073 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,8 @@ Cargo.lock *.swp *.swo + +__pycache__/ +/build/ +/dist/ +/*.egg-info/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9ba61f7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include Cargo.toml Cargo.lock README.md ARROW_SCHEMA.md +recursive-include src *.rs +recursive-include klamath_rs_ext *.py *.h py.typed +recursive-include tests *.rs *.py *.json *.txt *.mmd +recursive-include docs *.md *.json +recursive-include contrib * diff --git a/README.md b/README.md index 96b63e8..2e51acf 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,51 @@ range values must be aligned. Exported Arrow data owns its storage independently of the input. See [Arrow Schema Documentation](ARROW_SCHEMA.md) for the scan schema, offset definitions, and ID assignment. +## Python package and validation + +The Python packing helpers require NumPy and CFFI, both declared as runtime +dependencies. Setuptools invokes `cargo build --release --lib` and bundles the +C ABI shared library in a platform-specific wheel. Its version comes from +`Cargo.toml`. The C declarations are in `klamath_rs_ext/_ffi.h`; the loader uses +CFFI ABI mode, so no Python extension compilation or Maturin is required. + +```sh +python3 -m pip install . +``` + +To build a source archive and a wheel with the `build` frontend and installed +Setuptools/Wheel tooling: + +```sh +python3 -m build --no-isolation +``` + +Source archives include a Cargo lockfile, and wheel builds use `--locked` so +building from an archive retains the resolved Rust dependency versions. Wheels +are tagged for the build platform and use the Python-independent `py3-none` ABI. +The CLI continues to build directly with Cargo and has no Python dependency. + +For source-tree development, point the CFFI loader at a Cargo-built library: + +```sh +cargo build --release --lib +KLAMATH_RS_LIBRARY="$PWD/target/release/libklamath_rs_ext.so" \ + python3 -m unittest discover -s tests/python -v +``` + +Use `libklamath_rs_ext.dylib` on macOS or `klamath_rs_ext.dll` on Windows. Without +the explicit override, the loader uses the library bundled inside the package. +After installing a wheel, run its tests outside the source checkout to exercise +the installed package: + +```sh +python3 -I -m unittest discover -s /absolute/path/to/klamath-rs/tests/python -v +``` + +The Python tests cover multidimensional and noncontiguous packing, input +preservation, range errors, and the C declarations for Arrow exports and release +callbacks. Run `cargo test` and `cargo test --release` for native validation. + ## Layout review CLI The review CLI is a standalone Rust executable: diff --git a/klamath_rs_ext/__init__.py b/klamath_rs_ext/__init__.py index d042d92..2916461 100644 --- a/klamath_rs_ext/__init__.py +++ b/klamath_rs_ext/__init__.py @@ -2,5 +2,5 @@ from .basic import pack_int2 as pack_int2 from .basic import pack_int4 as pack_int4 -__version__ = 0.2 +__version__ = "0.2.0" diff --git a/klamath_rs_ext/_ffi.h b/klamath_rs_ext/_ffi.h new file mode 100644 index 0000000..273067e --- /dev/null +++ b/klamath_rs_ext/_ffi.h @@ -0,0 +1,50 @@ +#ifndef KLAMATH_RS_EXT_H +#define KLAMATH_RS_EXT_H +#include +#include + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema); +int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema); +const char *last_error_message(void); + +double f64_to_i16(double *data, size_t size); +double f64_to_i32(double *data, size_t size); +float f32_to_i16(float *data, size_t size); +float f32_to_i32(float *data, size_t size); +int64_t i64_to_i16(int64_t *data, size_t size); +int64_t i64_to_i32(int64_t *data, size_t size); +int32_t i32_to_i16(int32_t *data, size_t size); +int32_t i32_to_i32(int32_t *data, size_t size); +int16_t i16_to_i16(int16_t *data, size_t size); +uint64_t u64_to_i16(uint64_t *data, size_t size); +uint64_t u64_to_i32(uint64_t *data, size_t size); +uint32_t u32_to_i16(uint32_t *data, size_t size); +uint32_t u32_to_i32(uint32_t *data, size_t size); +uint16_t u16_to_i16(uint16_t *data, size_t size); +#endif diff --git a/klamath_rs_ext/basic.py b/klamath_rs_ext/basic.py index aabc246..c7bcd8e 100644 --- a/klamath_rs_ext/basic.py +++ b/klamath_rs_ext/basic.py @@ -49,7 +49,7 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: if result != 0: raise ValueError(f'Invalid value for conversion to Int2: {result}') - i2arr = arr.view('>i2')[::arr.itemsize // 2] + i2arr = arr.reshape(-1).view('>i2')[::arr.itemsize // 2] return i2arr.tobytes() if arr.dtype == numpy.dtype('>i2'): @@ -79,7 +79,7 @@ def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes: if result != 0: raise ValueError(f'Invalid value for conversion to Int4: {result}') - i4arr = arr.view('>i4')[::arr.itemsize // 4] + i4arr = arr.reshape(-1).view('>i4')[::arr.itemsize // 4] return i4arr.tobytes() if arr.dtype == numpy.dtype('>i4'): diff --git a/klamath_rs_ext/klamath_rs_ext.py b/klamath_rs_ext/klamath_rs_ext.py new file mode 100644 index 0000000..0fc0ccd --- /dev/null +++ b/klamath_rs_ext/klamath_rs_ext.py @@ -0,0 +1,23 @@ +"""CFFI ABI access to the Cargo-built shared library.""" +import os +from pathlib import Path +import sys + +from cffi import FFI + + +ffi = FFI() +header = Path(__file__).with_name("_ffi.h").read_text() +ffi.cdef("\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#"))) + +if sys.platform == "win32": + filename = "klamath_rs_ext.dll" +elif sys.platform == "darwin": + filename = "libklamath_rs_ext.dylib" +else: + filename = "libklamath_rs_ext.so" + +# An explicit override supports source-tree development without copying binaries +# into the package. Installed wheels load only their bundled native library. +library = Path(os.environ["KLAMATH_RS_LIBRARY"]) if "KLAMATH_RS_LIBRARY" in os.environ else Path(__file__).with_name(filename) +lib = ffi.dlopen(str(library.resolve())) diff --git a/pyproject.toml b/pyproject.toml index 06aa1af..7a14183 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["maturin>1.0,<2.0"] -build-backend = "maturin" +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" [project] name = "klamath_rs_ext" @@ -10,8 +10,6 @@ description = "Compiled extensions for klamath GDS library" authors = [ { name="Jan Petykiewicz", email="jan@mpxd.net" }, ] -homepage = "https://mpxd.net/code/jan/klamath-rs" -repository = "https://mpxd.net/code/jan/klamath-rs" classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 4 - Beta", @@ -30,8 +28,16 @@ requires-python = ">=3.11" dynamic = ["version"] dependencies = [ "cffi", + "numpy", ] -[tool.maturin] -bindings = "cffi" +[project.urls] +Homepage = "https://mpxd.net/code/jan/klamath-rs" +Repository = "https://mpxd.net/code/jan/klamath-rs" + +[tool.setuptools] +packages = ["klamath_rs_ext"] + +[tool.setuptools.package-data] +klamath_rs_ext = ["py.typed", "_ffi.h"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..698a4dd --- /dev/null +++ b/setup.py @@ -0,0 +1,76 @@ +"""Bundle Cargo's C ABI library in a platform wheel, without a Python ABI tie.""" +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tomllib + +from setuptools import Distribution, setup +from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist +from wheel.bdist_wheel import bdist_wheel + + +ROOT = Path(__file__).resolve().parent +VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] + + +def ensure_lockfile(): + if not (ROOT / "Cargo.lock").is_file(): + subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True) + + +class CargoBuild(build_py): + def run(self): + ensure_lockfile() + super().run() + result = subprocess.run( + ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], + cwd=ROOT, capture_output=True, text=True, + ) + sys.stderr.write(result.stderr) + library = None + for line in result.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + print(line, file=sys.stderr) + continue + if message.get("reason") == "compiler-message": + sys.stderr.write(message["message"].get("rendered") or "") + if (message.get("reason") == "compiler-artifact" + and message["target"]["name"] == "klamath_rs_ext" + and "cdylib" in message["target"]["crate_types"]): + for filename in message["filenames"]: + if Path(filename).suffix in (".so", ".dylib", ".dll"): + library = Path(filename) + result.check_returncode() + if library is None: + raise RuntimeError("Cargo did not produce the klamath_rs_ext shared library") + shutil.copy2(library, Path(self.build_lib) / "klamath_rs_ext" / library.name) + + +class CargoSdist(sdist): + def run(self): + ensure_lockfile() + super().run() + + +class NativeDistribution(Distribution): + def has_ext_modules(self): + return True + + +class NativeWheel(bdist_wheel): + def get_tag(self): + # CFFI ABI mode needs the host platform, but no CPython extension ABI. + _, _, platform = super().get_tag() + return "py3", "none", platform + + +setup( + version=VERSION, + distclass=NativeDistribution, + cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, +) diff --git a/tests/python/test_ffi.py b/tests/python/test_ffi.py new file mode 100644 index 0000000..bf95534 --- /dev/null +++ b/tests/python/test_ffi.py @@ -0,0 +1,69 @@ +"""Exercise the explicit C ABI declarations against the actual shared library.""" +import struct +import tempfile +from pathlib import Path +import unittest + +from klamath_rs_ext.klamath_rs_ext import ffi, lib + + +def record(tag, data=b""): + return struct.pack(">HH", len(data) + 4, tag) + data + + +def layout(): + return b"".join([ + record(0x0002, struct.pack(">h", 600)), + record(0x0102, b"\0" * 24), + record(0x0206, b"test"), + record(0x0305, bytes.fromhex("3e4189374bc6a7f03944b82fa09b5a54")), + record(0x0502, b"\0" * 24), + record(0x0606, b"cell"), + record(0x0700), + record(0x0400), + ]) + + +class FfiTests(unittest.TestCase): + def check_export(self, call): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + try: + self.assertEqual(call(array, schema), 0) + self.assertEqual(lib.last_error_message(), ffi.NULL) + self.assertEqual(array.length, 1) + self.assertEqual(ffi.string(schema.format), b"+s") + self.assertNotEqual(array.release, ffi.NULL) + self.assertNotEqual(schema.release, ffi.NULL) + self.assertGreaterEqual(array.n_children, 7) + self.assertEqual(array.n_children, schema.n_children) + self.assertEqual(ffi.string(schema.children[0].name), b"meters_per_db_unit") + finally: + if array.release != ffi.NULL: + array.release(array) + if schema.release != ffi.NULL: + schema.release(schema) + + def test_scan_selected_read_and_path_exports(self): + data = layout() + self.check_export(lambda a, s: lib.scan_bytes(data, len(data), a, s)) + # The library header is 62 bytes; the one cell ends just before ENDLIB. + ranges = ffi.new("uint64_t[]", [62, len(data) - 4]) + self.check_export(lambda a, s: lib.read_cells_bytes(data, len(data), ranges, 1, a, s)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "file with spaces.gds" + path.write_bytes(data) + self.check_export(lambda a, s: lib.read_path(str(path).encode(), a, s)) + + def test_failures_leave_outputs_empty_and_expose_error_message(self): + array = ffi.new("struct ArrowArray *") + schema = ffi.new("struct ArrowSchema *") + self.assertEqual(lib.scan_bytes(ffi.NULL, 0, array, schema), -1) + self.assertNotEqual(lib.last_error_message(), ffi.NULL) + self.assertIn(b"scan_bytes", ffi.string(lib.last_error_message())) + self.assertEqual(array.release, ffi.NULL) + self.assertEqual(schema.release, ffi.NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_packing.py b/tests/python/test_packing.py new file mode 100644 index 0000000..43f6487 --- /dev/null +++ b/tests/python/test_packing.py @@ -0,0 +1,66 @@ +import unittest + +import numpy as np + +from klamath_rs_ext import pack_int2, pack_int4 + + +class PackingTests(unittest.TestCase): + def test_scalars_lists_and_dimensions(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + for dtype in [np.int64, np.int32, np.uint64, np.uint32, np.float64, np.float32]: + for shape in [(), (1,), (2, 2), (2, 2, 2)]: + count = int(np.prod(shape)) if shape else 1 + data = np.arange(count, dtype=dtype).reshape(shape) + before = data.copy() + with self.subTest(width=width, dtype=dtype, shape=shape): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + np.testing.assert_array_equal(data, before) + for data in [5, [1, 2, 3], [[1, 2], [3, 4]], [], [[], []]]: + with self.subTest(width=width, data=data): + self.assertEqual(pack(data), np.asarray(data).astype(f">i{width}").tobytes()) + + def test_noncontiguous_readonly_and_endian_inputs(self): + original = np.arange(48, dtype=np.int64).reshape(6, 8) + readonly = original.copy() + readonly.setflags(write=False) + for data in [original.T, original[::-1, ::2], np.asfortranarray(original), readonly, + original.astype(">i8"), original.astype(">i2"), original.astype(">i4")]: + before = data.copy() + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + with self.subTest(width=width, strides=data.strides, dtype=data.dtype): + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes(order="C")) + np.testing.assert_array_equal(data, before) + + def test_signed_limits_and_native_integer_widths(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + low, high = -(1 << (8 * width - 1)), (1 << (8 * width - 1)) - 1 + data = np.array([[low, -1], [0, high]], dtype=np.int64) + self.assertEqual(pack(data), data.astype(f">i{width}").tobytes()) + for dtype in [np.int8, np.uint8, np.int16, np.uint16]: + data = np.array([[1, 2], [3, 4]], dtype=dtype) + self.assertEqual(pack_int2(data), data.astype(">i2").tobytes()) + self.assertEqual(pack_int4(data), data.astype(">i4").tobytes()) + + def test_failed_conversions_preserve_caller_data(self): + for pack, width in [(pack_int2, 2), (pack_int4, 4)]: + limit = 1 << (8 * width - 1) + for invalid in [limit, -limit - 1]: + data = np.array([[1, 2], [3, invalid]], dtype=np.int64) + before = data.copy() + with self.subTest(width=width, invalid=invalid): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + for dtype in [np.float32, np.float64]: + for invalid in [np.nan, np.inf, -np.inf]: + data = np.array([[1, 2], [3, invalid]], dtype=dtype) + before = data.copy() + with self.subTest(width=width, invalid=invalid, dtype=dtype): + with self.assertRaises(ValueError): + pack(data) + np.testing.assert_array_equal(data, before) + + +if __name__ == "__main__": + unittest.main()