From f081a808eaccdf12b102d00acf7b4626ce38516a Mon Sep 17 00:00:00 2001 From: Jan Petykiewicz Date: Sun, 15 Nov 2026 19:51:13 -0800 Subject: [PATCH] fix(real8): normalize finite values and handle underflow consistently --- README.md | 4 ++ src/basic.rs | 91 ++++++++--------------------- tests/golden/report-equal.json | 8 +-- tests/golden/report-geometry.json | 8 +-- tests/golden/report-incomplete.json | 8 +-- tests/golden/report-metadata.json | 8 +-- tests/golden/review-canonical.txt | 2 +- tests/real8.rs | 75 ++++++++++++++++++++++++ 8 files changed, 120 insertions(+), 84 deletions(-) create mode 100644 tests/real8.rs diff --git a/README.md b/README.md index df3ea04..96b63e8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,10 @@ A fast GDSII parser and writer in Rust, with Apache Arrow integration for effici - Optimized "batch" representations for rectangles and boundaries sharing the same layer. - C-compatible FFI for integration with other languages (e.g., Python). +REAL8 writing uses normalized GDS encodings and represents every in-range `f64` +exactly. Finite magnitudes below `2^-260` underflow to canonical zero; nonfinite +values and magnitudes at or above `2^252` are rejected by the encoder. + ## Usage `read_path` exports a library through the Arrow C Data Interface. Allocate diff --git a/src/basic.rs b/src/basic.rs index 828c282..b0b5057 100644 --- a/src/basic.rs +++ b/src/basic.rs @@ -166,79 +166,36 @@ impl fmt::Display for FloatTooBigError { } } -/// Convert from float64 to GDS REAL8 representation. +/// Convert float64 to a normalized GDS REAL8 representation. +/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0. +/// Nonfinite values and magnitudes at or above 2^252 return an error. pub fn encode_real8(fnum: f64) -> Result { - // Split the ieee float bitfields let ieee = fnum.to_bits(); + let ieee_exp = ((ieee >> 52) & 0x7ff) as i32; + if ieee_exp == 0x7ff { + return Err(FloatTooBigError { float_value: fnum }); + } + + let exp2 = ieee_exp - 1023; + if exp2 < -260 { + // This also handles both signed zeros and every IEEE subnormal. + return Ok(0); + } + + let gds_exp = exp2.div_euclid(4) + 65; + if gds_exp > 0x7f { + return Err(FloatTooBigError { float_value: fnum }); + } + + // The 53-bit IEEE significand needs only 0..3 left shifts to become + // a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8. + let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52); + let mantissa = significand << exp2.rem_euclid(4); let sign = ieee & 0x8000_0000_0000_0000; - let ieee_exp = (ieee >> 52) as i32 & 0x7ff; - let ieee_mant = ieee & 0x000f_ffff_ffff_ffff; - - let subnorm = (ieee_exp == 0) & (ieee_mant != 0); - if (ieee_exp == 0) & (ieee_mant == 0) { - return Ok(0) - } - - // IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023) - // IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022) - // GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64) - // = (gds_mant / 2^56) * 2^(4 * gds_exp - 256) - - // Convert exponent. - let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII) - let mut exp16 = exp2 / 4; - let rest = exp2 % 4; - - // Compensate for exponent coarseness - let comp = rest != 0; - let mut shift; - if comp { - exp16 += 1; - shift = 4 - rest; - } else { - shift = rest; - } - shift -= 3; // account for gds bit position - - // add leading one - let mut gds_mant_unshifted = ieee_mant; - if !subnorm { - gds_mant_unshifted += 0x10_0000_0000_0000; - } - - let mut gds_mant = if shift > 0 { - gds_mant_unshifted >> shift - } else { - gds_mant_unshifted << -shift - }; - - // add gds exponent bias - let mut gds_exp = exp16 + 64; - - if gds_exp < -14 { - // number is too small - return Ok(0) - } - - let neg_biased = gds_exp < 0; - if neg_biased { - gds_mant >>= gds_exp * 4; - gds_exp = 0; - } - - let too_big = (gds_exp > 0x7f) & !subnorm; - if too_big { - return Err(FloatTooBigError{float_value: fnum}); - } - - let gds_exp_bits = (gds_exp as u64) << 56; - - let real8 = sign | gds_exp_bits | gds_mant; - Ok(real8) + Ok(sign | ((gds_exp as u64) << 56) | mantissa) } - #[cfg(test)] mod tests { #[test] diff --git a/tests/golden/report-equal.json b/tests/golden/report-equal.json index cc02059..239f183 100644 --- a/tests/golden/report-equal.json +++ b/tests/golden/report-equal.json @@ -17,11 +17,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } }, @@ -32,11 +32,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } } diff --git a/tests/golden/report-geometry.json b/tests/golden/report-geometry.json index 9e03987..58ae71f 100644 --- a/tests/golden/report-geometry.json +++ b/tests/golden/report-geometry.json @@ -299,11 +299,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } }, @@ -314,11 +314,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } } diff --git a/tests/golden/report-incomplete.json b/tests/golden/report-incomplete.json index 8f0346b..09d4ad3 100644 --- a/tests/golden/report-incomplete.json +++ b/tests/golden/report-incomplete.json @@ -44,11 +44,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } }, @@ -59,11 +59,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } } diff --git a/tests/golden/report-metadata.json b/tests/golden/report-metadata.json index 2c51187..3a60532 100644 --- a/tests/golden/report-metadata.json +++ b/tests/golden/report-metadata.json @@ -17,11 +17,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } }, @@ -32,11 +32,11 @@ "units": { "encoding": "real8", "name": "UNITS", - "payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5", + "payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54", "tag": 773, "value": [ 0.001, - 9.999999999999999e-10 + 1e-9 ] } } diff --git a/tests/golden/review-canonical.txt b/tests/golden/review-canonical.txt index 3cdc7ea..8811a25 100644 --- a/tests/golden/review-canonical.txt +++ b/tests/golden/review-canonical.txt @@ -3,7 +3,7 @@ scope top=all cell=all layer=all coordinates: integer database units; bounds: coordinate envelopes HEADER=[600] LIBNAME="test" -UNITS=[0.001, 9.999999999999999e-10] raw=0x3f04189374bc6a7f3a044b82fa09b5a5 +UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54 layer 1/0 name="WG" cell "CHIP" BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0] diff --git a/tests/real8.rs b/tests/real8.rs new file mode 100644 index 0000000..63c4821 --- /dev/null +++ b/tests/real8.rs @@ -0,0 +1,75 @@ +use klamath_rs_ext::basic::{decode_real8, encode_real8, pack_real8}; + +#[test] +fn normalized_real8_has_known_encodings_and_exact_values() { + for (value, expected) in [ + (0.0, 0), + (-0.0, 0), + (1.0, 0x4110_0000_0000_0000), + (-2.0, 0xc120_0000_0000_0000), + (0.5, 0x4080_0000_0000_0000), + (0.001, 0x3e41_8937_4bc6_a7f0), + (1e-9, 0x3944_b82f_a09b_5a54), + (2_f64.powi(-260), 0x0010_0000_0000_0000), + ] { + let bits = encode_real8(value).unwrap(); + assert_eq!(bits, expected, "{value:e}"); + assert_eq!(decode_real8(bits), value); + } +} + +#[test] +fn real8_round_trips_each_supported_binary_exponent_and_mantissa_boundary() { + for exponent in -260..=251_i32 { + for mantissa in [ + 0, + 1, + 0x0008_0000_0000_0000, + 0x000f_ffff_ffff_fffe, + 0x000f_ffff_ffff_ffff, + ] { + for sign in [0, 1_u64 << 63] { + let ieee = sign | (((exponent + 1023) as u64) << 52) | mantissa; + let value = f64::from_bits(ieee); + let encoded = encode_real8(value).unwrap(); + assert_ne!(encoded & 0x00f0_0000_0000_0000, 0, "{value:e}"); + assert_eq!(decode_real8(encoded).to_bits(), ieee, "{value:e}"); + } + } + } +} + +#[test] +fn real8_underflow_is_canonical_and_overflow_is_an_error() { + let minimum = 2_f64.powi(-260); + let maximum_exclusive = 2_f64.powi(252); + for magnitude in [ + 0.0, + f64::from_bits(1), + f64::MIN_POSITIVE, + 1e-90, + 1e-80, + f64::from_bits(minimum.to_bits() - 1), + ] { + for value in [magnitude, -magnitude] { + assert_eq!(encode_real8(value).unwrap(), 0, "{value:e}"); + } + } + let largest = f64::from_bits(maximum_exclusive.to_bits() - 1); + assert_eq!(encode_real8(largest).unwrap(), 0x7fff_ffff_ffff_fff8); + assert_eq!(decode_real8(encode_real8(largest).unwrap()), largest); + for value in [ + maximum_exclusive, + -maximum_exclusive, + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] { + assert!(encode_real8(value).is_err(), "{value}"); + let mut buffer = [0x55; 8]; + assert!(pack_real8(&mut buffer, value).is_err()); + assert_eq!(buffer, [0x55; 8]); + } +}