Simplify Python bindings and package builds

Read each input path through one opened stream, including gzip detection, and
use direct numeric converter lookup while preserving byte-order handling and
caller data. Read the package version from installed metadata and report range
errors consistently as ValueError.

Require the supplied Cargo.lock instead of generating it during builds, use
the standard sdist command, and update package checks and development guides.

Validation: 15 Python tests pass against source and the rebuilt installed
wheel; the source archive and platform wheel build successfully.
This commit is contained in:
Jan Petykiewicz 2026-11-16 20:44:56 -08:00
commit a723ba2265
9 changed files with 40 additions and 38 deletions

View file

@ -6,7 +6,8 @@ with the [CLI](cli.md) or [Python](python.md) guide.
## Build and test ## Build and test
The checkout selects the stable Rust toolchain; the minimum supported compiler The checkout selects the stable Rust toolchain; the minimum supported compiler
is 1.88. Python bindings require Python 3.11 or newer. is 1.88. Python bindings require Python 3.11 or newer. Source builds require
the tracked `Cargo.lock`; update dependencies explicitly with Cargo.
```sh ```sh
cargo test --locked cargo test --locked
@ -17,14 +18,15 @@ python -I tools/check_installed.py
``` ```
Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler. Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler.
The Python package bundles its native library during installation. To test a The Python package bundles its native library during installation and reads its
newly built library without reinstalling, set `PHEON_LIBRARY` to its absolute version from installed package metadata. After installation, to test a newly
built library without reinstalling, set `PHEON_LIBRARY` to its absolute
path and run `python -m unittest discover -s tests/python` from the checkout. path and run `python -m unittest discover -s tests/python` from the checkout.
Unset this override when checking installed artifacts with `check_installed.py`. Unset this override when checking installed artifacts with `check_installed.py`.
## Interfaces and reports ## Interfaces and reports
The CFFI declarations live in [pheon/_ffi.h](../pheon/_ffi.h). Keep them aligned The CFFI declarations live in [pheon/_ffi.h](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h). Keep them aligned
with the exported native functions. The C API version and with the exported native functions. The C API version and
[Arrow layout version](arrow-schema.md) are independent of package releases and [Arrow layout version](arrow-schema.md) are independent of package releases and
the [JSON report schema](diff-reports.md). the [JSON report schema](diff-reports.md).

View file

@ -20,7 +20,7 @@ provides the CLI. Neither installation discovers or depends on the other.
## Validate artifacts ## Validate artifacts
The [package workflow](../.github/workflows/packages.yml) builds one source archive The [package workflow](https://mpxd.net/code/jan/pheon/src/branch/master/.github/workflows/packages.yml) builds one source archive
and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64, and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64,
and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag. and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag.
Linux releases need an appropriate manylinux tag; unrepaired local `linux_*` Linux releases need an appropriate manylinux tag; unrepaired local `linux_*`

View file

@ -1,8 +1,10 @@
"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers.""" """Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers."""
from importlib.metadata import version as _version
from ._native import PheonError as PheonError from ._native import PheonError as PheonError
from .basic import pack_int2 as pack_int2 from .basic import pack_int2 as pack_int2
from .basic import pack_int4 as pack_int4 from .basic import pack_int4 as pack_int4
__version__ = "0.3.0" __version__ = _version("pheon")

View file

@ -43,16 +43,13 @@ def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray:
def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray: def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray:
"""Read a GDSII file, recognizing gzip by its magic bytes.""" """Read a GDSII file, recognizing gzip by its magic bytes."""
filename = os.path.abspath(os.path.expanduser(os.fspath(path))) with open(os.path.expanduser(path), "rb") as stream:
with open(filename, "rb") as stream:
compressed = stream.read(2) == b"\x1f\x8b" compressed = stream.read(2) == b"\x1f\x8b"
if compressed:
stream.seek(0) stream.seek(0)
if compressed:
with gzip.GzipFile(fileobj=stream) as source: with gzip.GzipFile(fileobj=stream) as source:
return read_bytes(source.read()) return read_bytes(source.read())
# The native Windows path API accepts UTF-8; Unix accepts filesystem bytes. return read_bytes(stream.read())
encoded = filename.encode("utf-8") if os.name == "nt" else os.fsencode(filename)
return _export(lambda array, schema: lib.read_path(encoded, array, schema))
def _read_buffer( def _read_buffer(

View file

@ -36,10 +36,8 @@ def _pack_int(
arr = numpy.atleast_1d(data) arr = numpy.atleast_1d(data)
target_dtype = numpy.dtype(f">i{width}") target_dtype = numpy.dtype(f">i{width}")
for dtype, convert in converters.items(): convert = converters.get(arr.dtype.type) if arr.dtype.isnative else None
if arr.dtype != dtype: if convert is not None:
continue
arr = numpy.require( arr = numpy.require(
arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA") arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA")
) )
@ -60,7 +58,7 @@ def _pack_int(
limit = 1 << (width * 8 - 1) limit = 1 << (width * 8 - 1)
if (arr > limit - 1).any() or (arr < -limit).any(): if (arr > limit - 1).any() or (arr < -limit).any():
raise Exception(f"int{width} data out of range: {arr}") raise ValueError(f"int{width} data out of range: {arr}")
return arr.astype(target_dtype).tobytes() return arr.astype(target_dtype).tobytes()

View file

@ -8,7 +8,6 @@ import tomllib
from setuptools import Distribution, setup from setuptools import Distribution, setup
from setuptools.command.build_py import build_py from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
from wheel.bdist_wheel import bdist_wheel from wheel.bdist_wheel import bdist_wheel
@ -16,14 +15,12 @@ ROOT = Path(__file__).resolve().parent
VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"] VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"]
def ensure_lockfile(): if not (ROOT / "Cargo.lock").is_file():
if not (ROOT / "Cargo.lock").is_file(): raise FileNotFoundError("Source builds require the supplied Cargo.lock")
subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True)
class CargoBuild(build_py): class CargoBuild(build_py):
def run(self): def run(self):
ensure_lockfile()
super().run() super().run()
result = subprocess.run( result = subprocess.run(
["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"], ["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"],
@ -51,12 +48,6 @@ class CargoBuild(build_py):
shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name) shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name)
class CargoSdist(sdist):
def run(self):
ensure_lockfile()
super().run()
class NativeDistribution(Distribution): class NativeDistribution(Distribution):
def has_ext_modules(self): def has_ext_modules(self):
return True return True
@ -72,5 +63,5 @@ class NativeWheel(bdist_wheel):
setup( setup(
version=VERSION, version=VERSION,
distclass=NativeDistribution, distclass=NativeDistribution,
cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel}, cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel},
) )

View file

@ -2,6 +2,7 @@
import gc import gc
import gzip import gzip
import io
import mmap import mmap
from pathlib import Path from pathlib import Path
import tempfile import tempfile
@ -48,6 +49,16 @@ class ArrowTests(unittest.TestCase):
path.write_bytes(gzip.compress(layout())) path.write_bytes(gzip.compress(layout()))
self.assertEqual(arrow.read_path(path).to_pylist(), expected) self.assertEqual(arrow.read_path(path).to_pylist(), expected)
def test_path_reader_keeps_using_the_opened_stream(self):
for payload in (layout(), gzip.compress(layout())):
with self.subTest(compressed=payload.startswith(b"\x1f\x8b")):
source = io.BytesIO(payload)
with patch("builtins.open", return_value=source) as opened:
result = arrow.read_path("opened-stream-only.gds")
opened.assert_called_once_with("opened-stream-only.gds", "rb")
self.assertTrue(source.closed)
self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"])
def test_empty_and_noncontiguous_cell_selections(self): def test_empty_and_noncontiguous_cell_selections(self):
data = layout() data = layout()
empty = arrow.read_cells_bytes(data, []) empty = arrow.read_cells_bytes(data, [])

View file

@ -45,10 +45,11 @@ class PackingTests(unittest.TestCase):
def test_failed_conversions_preserve_caller_data(self): def test_failed_conversions_preserve_caller_data(self):
for pack, width in [(pack_int2, 2), (pack_int4, 4)]: for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
limit = 1 << (8 * width - 1) limit = 1 << (8 * width - 1)
for dtype in [np.dtype("i8"), np.dtype("i8").newbyteorder("S")]:
for invalid in [limit, -limit - 1]: for invalid in [limit, -limit - 1]:
data = np.array([[1, 2], [3, invalid]], dtype=np.int64) data = np.array([[1, 2], [3, invalid]], dtype=dtype)
before = data.copy() before = data.copy()
with self.subTest(width=width, invalid=invalid): with self.subTest(width=width, dtype=dtype, invalid=invalid):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
pack(data) pack(data)
np.testing.assert_array_equal(data, before) np.testing.assert_array_equal(data, before)

View file

@ -10,12 +10,12 @@ ROOT = Path(__file__).resolve().parents[1]
# A separate interpreter verifies that base packing does not import PyArrow. # A separate interpreter verifies that base packing does not import PyArrow.
subprocess.run( subprocess.run(
[sys.executable, "-I", "-c", """ [sys.executable, "-I", "-c", """
import importlib.util
import sys import sys
from importlib.metadata import version
import pheon import pheon
from pheon._native import library from pheon._native import library
assert pheon.__version__ == version('pheon')
assert 'pyarrow' not in sys.modules assert 'pyarrow' not in sys.modules
assert importlib.util.find_spec('klamath_rs_ext') is None
assert library.parent == __import__('pathlib').Path(pheon.__file__).parent assert library.parent == __import__('pathlib').Path(pheon.__file__).parent
assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe') assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe')
"""], """],