jan/klamath
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixup real coversion for large exponents

Browse Source 
also implement subnormals, though they don't have real8 equivalents so
it's wasted effort
This commit is contained in:
jan 2021-03-26 10:14:02 -07:00
parent 329a86c94f
commit 38c22f081b
1 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
klamath/basic.py
View File

@ -98,7 +98,7 @@ def pack_int4(data: Sequence[int]) -> bytes:
def encode_real8(fnums: numpy.ndarray) -> numpy.ndarray: def encode_real8(fnums: numpy.ndarray) -> numpy.ndarray:
""" Convert from float64 to GDS REAL8 representation. """ """ Convert from float64 to GDS REAL8 representation. """
# Split the bitfields # Split the ieee float bitfields
ieee = numpy.atleast_1d(fnums.astype(numpy.float64).view(numpy.uint64)) ieee = numpy.atleast_1d(fnums.astype(numpy.float64).view(numpy.uint64))
sign = ieee & numpy.uint64(0x8000_0000_0000_0000) sign = ieee & numpy.uint64(0x8000_0000_0000_0000)
ieee_exp = (ieee >> numpy.uint64(52)).astype(numpy.int32) & numpy.int32(0x7ff) ieee_exp = (ieee >> numpy.uint64(52)).astype(numpy.int32) & numpy.int32(0x7ff)
@ -107,11 +107,17 @@ def encode_real8(fnums: numpy.ndarray) -> numpy.ndarray:
subnorm = (ieee_exp == 0) & (ieee_mant != 0) subnorm = (ieee_exp == 0) & (ieee_mant != 0)
zero = (ieee_exp == 0) & (ieee_mant == 0) zero = (ieee_exp == 0) & (ieee_mant == 0)
# IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023)
# IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022)
# GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64)
# = (gds_mant / 2^56) * 2^(4 * gds_exp - 256)
# Convert exponent. # Convert exponent.
# * 16-based exp2 = ieee_exp + 1 - 1023 # +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII)
# * +1 due to mantissa differences (1.xxxx in IEEE vs 0.1xxxxx in GDSII) exp2[subnorm] = -1022
exp16, rest = numpy.divmod(ieee_exp + 1 - 1023, 4) exp16, rest = numpy.divmod(exp2, 4)
# Compensate exponent conversion
# Compensate for exponent coarseness
comp = (rest != 0) comp = (rest != 0)
exp16[comp] += 1 exp16[comp] += 1
@ -120,30 +126,30 @@ def encode_real8(fnums: numpy.ndarray) -> numpy.ndarray:
shift -= 3 # account for gds bit position shift -= 3 # account for gds bit position
# add leading one # add leading one
gds_mant_unshifted = ieee_mant + 0x10_0000_0000_0000 gds_mant_unshifted = ieee_mant.copy()
gds_mant_unshifted[~subnorm] += 0x10_0000_0000_0000
rshift = (shift > 0) rshift = (shift > 0)
gds_mant = numpy.empty_like(ieee_mant) gds_mant = numpy.empty_like(ieee_mant)
gds_mant[~rshift] = gds_mant_unshifted[~rshift] << (-shift[~rshift]).astype(numpy.uint8) gds_mant[~rshift] = gds_mant_unshifted[~rshift] << (-shift[~rshift]).astype(numpy.uint16)
gds_mant[ rshift] = gds_mant_unshifted[ rshift] >> ( shift[ rshift]).astype(numpy.uint8) gds_mant[ rshift] = gds_mant_unshifted[ rshift] >> ( shift[ rshift]).astype(numpy.uint16)
# add gds exponent bias # add gds exponent bias
exp16_biased = exp16 + 64 gds_exp = exp16 + 64
neg_biased = (exp16_biased < 0) neg_biased = (gds_exp < 0)
gds_mant[neg_biased] >>= (exp16_biased[neg_biased] * 4).astype(numpy.uint8) gds_mant[neg_biased] >>= (gds_exp[neg_biased] * 4).astype(numpy.uint16)
exp16_biased[neg_biased] = 0 gds_exp[neg_biased] = 0
too_big = (exp16_biased > 0x7f) & ~(zero | subnorm) too_big = (gds_exp > 0x7f) & ~(zero | subnorm)
if too_big.any(): if too_big.any():
raise KlamathError(f'Number(s) too big for real8 format: {fnums[too_big]}') raise KlamathError(f'Number(s) too big for real8 format: {fnums[too_big]}')
gds_exp = exp16_biased.astype(numpy.uint64) << 56 gds_exp_bits = gds_exp.astype(numpy.uint64) << 56
real8 = sign | gds_exp | gds_mant real8 = sign | gds_exp_bits | gds_mant
real8[zero] = 0 real8[zero] = 0
real8[subnorm] = 0 # TODO handle subnormals real8[gds_exp < -14] = 0 # number is too small
real8[exp16_biased < -14] = 0 # number is too small
return real8 return real8