fix(python): flatten packing buffers and package Cargo libraries

This commit is contained in:
Jan Petykiewicz 2026-11-15 20:01:26 -08:00
commit 912a267028
4 changed files with 355 additions and 9 deletions

View file

@ -2,5 +2,5 @@ from .basic import pack_int2 as pack_int2
from .basic import pack_int4 as pack_int4
__version__ = 0.2
__version__ = "0.2.0"

50
klamath_rs_ext/_ffi.h Normal file
View file

@ -0,0 +1,50 @@
#ifndef KLAMATH_RS_EXT_H
#define KLAMATH_RS_EXT_H
#include <stdint.h>
#include <stddef.h>
struct ArrowArray {
int64_t length;
int64_t null_count;
int64_t offset;
int64_t n_buffers;
int64_t n_children;
const void **buffers;
struct ArrowArray **children;
struct ArrowArray *dictionary;
void (*release)(struct ArrowArray *);
void *private_data;
};
struct ArrowSchema {
const char *format;
const char *name;
const char *metadata;
int64_t flags;
int64_t n_children;
struct ArrowSchema **children;
struct ArrowSchema *dictionary;
void (*release)(struct ArrowSchema *);
void *private_data;
};
int32_t read_path(const char *path, struct ArrowArray *array, struct ArrowSchema *schema);
int32_t scan_bytes(const uint8_t *data, size_t size, struct ArrowArray *array, struct ArrowSchema *schema);
int32_t read_cells_bytes(const uint8_t *data, size_t size, const uint64_t *ranges, size_t range_count, struct ArrowArray *array, struct ArrowSchema *schema);
const char *last_error_message(void);
double f64_to_i16(double *data, size_t size);
double f64_to_i32(double *data, size_t size);
float f32_to_i16(float *data, size_t size);
float f32_to_i32(float *data, size_t size);
int64_t i64_to_i16(int64_t *data, size_t size);
int64_t i64_to_i32(int64_t *data, size_t size);
int32_t i32_to_i16(int32_t *data, size_t size);
int32_t i32_to_i32(int32_t *data, size_t size);
int16_t i16_to_i16(int16_t *data, size_t size);
uint64_t u64_to_i16(uint64_t *data, size_t size);
uint64_t u64_to_i32(uint64_t *data, size_t size);
uint32_t u32_to_i16(uint32_t *data, size_t size);
uint32_t u32_to_i32(uint32_t *data, size_t size);
uint16_t u16_to_i16(uint16_t *data, size_t size);
#endif

View file

@ -49,7 +49,7 @@ def pack_int2(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes:
if result != 0:
raise ValueError(f'Invalid value for conversion to Int2: {result}')
i2arr = arr.view('>i2')[::arr.itemsize // 2]
i2arr = arr.reshape(-1).view('>i2')[::arr.itemsize // 2]
return i2arr.tobytes()
if arr.dtype == numpy.dtype('>i2'):
@ -79,7 +79,7 @@ def pack_int4(data: NDArray[numpy.integer] | Sequence[int] | int) -> bytes:
if result != 0:
raise ValueError(f'Invalid value for conversion to Int4: {result}')
i4arr = arr.view('>i4')[::arr.itemsize // 4]
i4arr = arr.reshape(-1).view('>i4')[::arr.itemsize // 4]
return i4arr.tobytes()
if arr.dtype == numpy.dtype('>i4'):

View file

@ -0,0 +1,23 @@
"""CFFI ABI access to the Cargo-built shared library."""
import os
from pathlib import Path
import sys
from cffi import FFI
ffi = FFI()
header = Path(__file__).with_name("_ffi.h").read_text()
ffi.cdef("\n".join(line for line in header.splitlines() if not line.lstrip().startswith("#")))
if sys.platform == "win32":
filename = "klamath_rs_ext.dll"
elif sys.platform == "darwin":
filename = "libklamath_rs_ext.dylib"
else:
filename = "libklamath_rs_ext.so"
# An explicit override supports source-tree development without copying binaries
# into the package. Installed wheels load only their bundled native library.
library = Path(os.environ["KLAMATH_RS_LIBRARY"]) if "KLAMATH_RS_LIBRARY" in os.environ else Path(__file__).with_name(filename)
lib = ffi.dlopen(str(library.resolve()))