Simplify Python bindings and package builds
Read each input path through one opened stream, including gzip detection, and use direct numeric converter lookup while preserving byte-order handling and caller data. Read the package version from installed metadata and report range errors consistently as ValueError. Require the supplied Cargo.lock instead of generating it during builds, use the standard sdist command, and update package checks and development guides. Validation: 15 Python tests pass against source and the rebuilt installed wheel; the source archive and platform wheel build successfully.
This commit is contained in:
parent
54aa757678
commit
a723ba2265
9 changed files with 40 additions and 38 deletions
|
|
@ -6,7 +6,8 @@ with the [CLI](cli.md) or [Python](python.md) guide.
|
||||||
## Build and test
|
## Build and test
|
||||||
|
|
||||||
The checkout selects the stable Rust toolchain; the minimum supported compiler
|
The checkout selects the stable Rust toolchain; the minimum supported compiler
|
||||||
is 1.88. Python bindings require Python 3.11 or newer.
|
is 1.88. Python bindings require Python 3.11 or newer. Source builds require
|
||||||
|
the tracked `Cargo.lock`; update dependencies explicitly with Cargo.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cargo test --locked
|
cargo test --locked
|
||||||
|
|
@ -17,14 +18,15 @@ python -I tools/check_installed.py
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler.
|
Use `RUSTUP_TOOLCHAIN=1.88.0` to check source builds against the minimum compiler.
|
||||||
The Python package bundles its native library during installation. To test a
|
The Python package bundles its native library during installation and reads its
|
||||||
newly built library without reinstalling, set `PHEON_LIBRARY` to its absolute
|
version from installed package metadata. After installation, to test a newly
|
||||||
|
built library without reinstalling, set `PHEON_LIBRARY` to its absolute
|
||||||
path and run `python -m unittest discover -s tests/python` from the checkout.
|
path and run `python -m unittest discover -s tests/python` from the checkout.
|
||||||
Unset this override when checking installed artifacts with `check_installed.py`.
|
Unset this override when checking installed artifacts with `check_installed.py`.
|
||||||
|
|
||||||
## Interfaces and reports
|
## Interfaces and reports
|
||||||
|
|
||||||
The CFFI declarations live in [pheon/_ffi.h](../pheon/_ffi.h). Keep them aligned
|
The CFFI declarations live in [pheon/_ffi.h](https://mpxd.net/code/jan/pheon/src/branch/master/pheon/_ffi.h). Keep them aligned
|
||||||
with the exported native functions. The C API version and
|
with the exported native functions. The C API version and
|
||||||
[Arrow layout version](arrow-schema.md) are independent of package releases and
|
[Arrow layout version](arrow-schema.md) are independent of package releases and
|
||||||
the [JSON report schema](diff-reports.md).
|
the [JSON report schema](diff-reports.md).
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ provides the CLI. Neither installation discovers or depends on the other.
|
||||||
|
|
||||||
## Validate artifacts
|
## Validate artifacts
|
||||||
|
|
||||||
The [package workflow](../.github/workflows/packages.yml) builds one source archive
|
The [package workflow](https://mpxd.net/code/jan/pheon/src/branch/master/.github/workflows/packages.yml) builds one source archive
|
||||||
and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64,
|
and uses cibuildwheel to build and repair wheels for Linux and macOS x86-64/ARM64,
|
||||||
and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag.
|
and Windows x86-64. Wheels use `py3-none` interpreter/ABI tags plus a platform tag.
|
||||||
Linux releases need an appropriate manylinux tag; unrepaired local `linux_*`
|
Linux releases need an appropriate manylinux tag; unrepaired local `linux_*`
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers."""
|
"""Native GDSII helpers. Install pheon[arrow] for the pheon.arrow readers."""
|
||||||
|
|
||||||
|
from importlib.metadata import version as _version
|
||||||
|
|
||||||
from ._native import PheonError as PheonError
|
from ._native import PheonError as PheonError
|
||||||
from .basic import pack_int2 as pack_int2
|
from .basic import pack_int2 as pack_int2
|
||||||
from .basic import pack_int4 as pack_int4
|
from .basic import pack_int4 as pack_int4
|
||||||
|
|
||||||
|
|
||||||
__version__ = "0.3.0"
|
__version__ = _version("pheon")
|
||||||
|
|
|
||||||
|
|
@ -43,16 +43,13 @@ def _export(call: Callable[[Any, Any], int]) -> pyarrow.StructArray:
|
||||||
|
|
||||||
def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray:
|
def read_path(path: str | os.PathLike[str]) -> pyarrow.StructArray:
|
||||||
"""Read a GDSII file, recognizing gzip by its magic bytes."""
|
"""Read a GDSII file, recognizing gzip by its magic bytes."""
|
||||||
filename = os.path.abspath(os.path.expanduser(os.fspath(path)))
|
with open(os.path.expanduser(path), "rb") as stream:
|
||||||
with open(filename, "rb") as stream:
|
|
||||||
compressed = stream.read(2) == b"\x1f\x8b"
|
compressed = stream.read(2) == b"\x1f\x8b"
|
||||||
if compressed:
|
|
||||||
stream.seek(0)
|
stream.seek(0)
|
||||||
|
if compressed:
|
||||||
with gzip.GzipFile(fileobj=stream) as source:
|
with gzip.GzipFile(fileobj=stream) as source:
|
||||||
return read_bytes(source.read())
|
return read_bytes(source.read())
|
||||||
# The native Windows path API accepts UTF-8; Unix accepts filesystem bytes.
|
return read_bytes(stream.read())
|
||||||
encoded = filename.encode("utf-8") if os.name == "nt" else os.fsencode(filename)
|
|
||||||
return _export(lambda array, schema: lib.read_path(encoded, array, schema))
|
|
||||||
|
|
||||||
|
|
||||||
def _read_buffer(
|
def _read_buffer(
|
||||||
|
|
|
||||||
|
|
@ -36,10 +36,8 @@ def _pack_int(
|
||||||
arr = numpy.atleast_1d(data)
|
arr = numpy.atleast_1d(data)
|
||||||
target_dtype = numpy.dtype(f">i{width}")
|
target_dtype = numpy.dtype(f">i{width}")
|
||||||
|
|
||||||
for dtype, convert in converters.items():
|
convert = converters.get(arr.dtype.type) if arr.dtype.isnative else None
|
||||||
if arr.dtype != dtype:
|
if convert is not None:
|
||||||
continue
|
|
||||||
|
|
||||||
arr = numpy.require(
|
arr = numpy.require(
|
||||||
arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA")
|
arr, requirements=("C_CONTIGUOUS", "ALIGNED", "WRITEABLE", "OWNDATA")
|
||||||
)
|
)
|
||||||
|
|
@ -60,7 +58,7 @@ def _pack_int(
|
||||||
|
|
||||||
limit = 1 << (width * 8 - 1)
|
limit = 1 << (width * 8 - 1)
|
||||||
if (arr > limit - 1).any() or (arr < -limit).any():
|
if (arr > limit - 1).any() or (arr < -limit).any():
|
||||||
raise Exception(f"int{width} data out of range: {arr}")
|
raise ValueError(f"int{width} data out of range: {arr}")
|
||||||
return arr.astype(target_dtype).tobytes()
|
return arr.astype(target_dtype).tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
13
setup.py
13
setup.py
|
|
@ -8,7 +8,6 @@ import tomllib
|
||||||
|
|
||||||
from setuptools import Distribution, setup
|
from setuptools import Distribution, setup
|
||||||
from setuptools.command.build_py import build_py
|
from setuptools.command.build_py import build_py
|
||||||
from setuptools.command.sdist import sdist
|
|
||||||
from wheel.bdist_wheel import bdist_wheel
|
from wheel.bdist_wheel import bdist_wheel
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -16,14 +15,12 @@ ROOT = Path(__file__).resolve().parent
|
||||||
VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"]
|
VERSION = tomllib.loads((ROOT / "Cargo.toml").read_text())["package"]["version"]
|
||||||
|
|
||||||
|
|
||||||
def ensure_lockfile():
|
|
||||||
if not (ROOT / "Cargo.lock").is_file():
|
if not (ROOT / "Cargo.lock").is_file():
|
||||||
subprocess.run(["cargo", "generate-lockfile"], cwd=ROOT, check=True)
|
raise FileNotFoundError("Source builds require the supplied Cargo.lock")
|
||||||
|
|
||||||
|
|
||||||
class CargoBuild(build_py):
|
class CargoBuild(build_py):
|
||||||
def run(self):
|
def run(self):
|
||||||
ensure_lockfile()
|
|
||||||
super().run()
|
super().run()
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"],
|
["cargo", "build", "--release", "--lib", "--locked", "--message-format=json"],
|
||||||
|
|
@ -51,12 +48,6 @@ class CargoBuild(build_py):
|
||||||
shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name)
|
shutil.copy2(library, Path(self.build_lib) / "pheon" / library.name)
|
||||||
|
|
||||||
|
|
||||||
class CargoSdist(sdist):
|
|
||||||
def run(self):
|
|
||||||
ensure_lockfile()
|
|
||||||
super().run()
|
|
||||||
|
|
||||||
|
|
||||||
class NativeDistribution(Distribution):
|
class NativeDistribution(Distribution):
|
||||||
def has_ext_modules(self):
|
def has_ext_modules(self):
|
||||||
return True
|
return True
|
||||||
|
|
@ -72,5 +63,5 @@ class NativeWheel(bdist_wheel):
|
||||||
setup(
|
setup(
|
||||||
version=VERSION,
|
version=VERSION,
|
||||||
distclass=NativeDistribution,
|
distclass=NativeDistribution,
|
||||||
cmdclass={"build_py": CargoBuild, "sdist": CargoSdist, "bdist_wheel": NativeWheel},
|
cmdclass={"build_py": CargoBuild, "bdist_wheel": NativeWheel},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
import gc
|
import gc
|
||||||
import gzip
|
import gzip
|
||||||
|
import io
|
||||||
import mmap
|
import mmap
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
@ -48,6 +49,16 @@ class ArrowTests(unittest.TestCase):
|
||||||
path.write_bytes(gzip.compress(layout()))
|
path.write_bytes(gzip.compress(layout()))
|
||||||
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
self.assertEqual(arrow.read_path(path).to_pylist(), expected)
|
||||||
|
|
||||||
|
def test_path_reader_keeps_using_the_opened_stream(self):
|
||||||
|
for payload in (layout(), gzip.compress(layout())):
|
||||||
|
with self.subTest(compressed=payload.startswith(b"\x1f\x8b")):
|
||||||
|
source = io.BytesIO(payload)
|
||||||
|
with patch("builtins.open", return_value=source) as opened:
|
||||||
|
result = arrow.read_path("opened-stream-only.gds")
|
||||||
|
opened.assert_called_once_with("opened-stream-only.gds", "rb")
|
||||||
|
self.assertTrue(source.closed)
|
||||||
|
self.assertEqual(result.field("cell_names")[0].as_py(), ["cell"])
|
||||||
|
|
||||||
def test_empty_and_noncontiguous_cell_selections(self):
|
def test_empty_and_noncontiguous_cell_selections(self):
|
||||||
data = layout()
|
data = layout()
|
||||||
empty = arrow.read_cells_bytes(data, [])
|
empty = arrow.read_cells_bytes(data, [])
|
||||||
|
|
|
||||||
|
|
@ -45,10 +45,11 @@ class PackingTests(unittest.TestCase):
|
||||||
def test_failed_conversions_preserve_caller_data(self):
|
def test_failed_conversions_preserve_caller_data(self):
|
||||||
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
for pack, width in [(pack_int2, 2), (pack_int4, 4)]:
|
||||||
limit = 1 << (8 * width - 1)
|
limit = 1 << (8 * width - 1)
|
||||||
|
for dtype in [np.dtype("i8"), np.dtype("i8").newbyteorder("S")]:
|
||||||
for invalid in [limit, -limit - 1]:
|
for invalid in [limit, -limit - 1]:
|
||||||
data = np.array([[1, 2], [3, invalid]], dtype=np.int64)
|
data = np.array([[1, 2], [3, invalid]], dtype=dtype)
|
||||||
before = data.copy()
|
before = data.copy()
|
||||||
with self.subTest(width=width, invalid=invalid):
|
with self.subTest(width=width, dtype=dtype, invalid=invalid):
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
pack(data)
|
pack(data)
|
||||||
np.testing.assert_array_equal(data, before)
|
np.testing.assert_array_equal(data, before)
|
||||||
|
|
|
||||||
|
|
@ -10,12 +10,12 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||||
# A separate interpreter verifies that base packing does not import PyArrow.
|
# A separate interpreter verifies that base packing does not import PyArrow.
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
[sys.executable, "-I", "-c", """
|
[sys.executable, "-I", "-c", """
|
||||||
import importlib.util
|
|
||||||
import sys
|
import sys
|
||||||
|
from importlib.metadata import version
|
||||||
import pheon
|
import pheon
|
||||||
from pheon._native import library
|
from pheon._native import library
|
||||||
|
assert pheon.__version__ == version('pheon')
|
||||||
assert 'pyarrow' not in sys.modules
|
assert 'pyarrow' not in sys.modules
|
||||||
assert importlib.util.find_spec('klamath_rs_ext') is None
|
|
||||||
assert library.parent == __import__('pathlib').Path(pheon.__file__).parent
|
assert library.parent == __import__('pathlib').Path(pheon.__file__).parent
|
||||||
assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe')
|
assert pheon.pack_int2([1, -2]) == bytes.fromhex('0001fffe')
|
||||||
"""],
|
"""],
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue