fix(real8): normalize finite values and handle underflow consistently

This commit is contained in:
Jan Petykiewicz 2026-11-15 19:51:13 -08:00
commit f081a808ea
6 changed files with 115 additions and 79 deletions

View file

@ -17,11 +17,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
},
@ -32,11 +32,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
}

View file

@ -299,11 +299,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
},
@ -314,11 +314,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
}

View file

@ -44,11 +44,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
},
@ -59,11 +59,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
}

View file

@ -17,11 +17,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
},
@ -32,11 +32,11 @@
"units": {
"encoding": "real8",
"name": "UNITS",
"payload_hex": "0x3f04189374bc6a7f3a044b82fa09b5a5",
"payload_hex": "0x3e4189374bc6a7f03944b82fa09b5a54",
"tag": 773,
"value": [
0.001,
9.999999999999999e-10
1e-9
]
}
}

View file

@ -3,7 +3,7 @@ scope top=all cell=all layer=all
coordinates: integer database units; bounds: coordinate envelopes
HEADER=[600]
LIBNAME="test"
UNITS=[0.001, 9.999999999999999e-10] raw=0x3f04189374bc6a7f3a044b82fa09b5a5
UNITS=[0.001, 1e-9] raw=0x3e4189374bc6a7f03944b82fa09b5a54
layer 1/0 name="WG"
cell "CHIP"
BOUNDARY layer=1 datatype=0 xy=[0, 0, 0, 10, 10, 10, 10, 0, 0, 0]

75
tests/real8.rs Normal file
View file

@ -0,0 +1,75 @@
use klamath_rs_ext::basic::{decode_real8, encode_real8, pack_real8};
#[test]
fn normalized_real8_has_known_encodings_and_exact_values() {
for (value, expected) in [
(0.0, 0),
(-0.0, 0),
(1.0, 0x4110_0000_0000_0000),
(-2.0, 0xc120_0000_0000_0000),
(0.5, 0x4080_0000_0000_0000),
(0.001, 0x3e41_8937_4bc6_a7f0),
(1e-9, 0x3944_b82f_a09b_5a54),
(2_f64.powi(-260), 0x0010_0000_0000_0000),
] {
let bits = encode_real8(value).unwrap();
assert_eq!(bits, expected, "{value:e}");
assert_eq!(decode_real8(bits), value);
}
}
#[test]
fn real8_round_trips_each_supported_binary_exponent_and_mantissa_boundary() {
for exponent in -260..=251_i32 {
for mantissa in [
0,
1,
0x0008_0000_0000_0000,
0x000f_ffff_ffff_fffe,
0x000f_ffff_ffff_ffff,
] {
for sign in [0, 1_u64 << 63] {
let ieee = sign | (((exponent + 1023) as u64) << 52) | mantissa;
let value = f64::from_bits(ieee);
let encoded = encode_real8(value).unwrap();
assert_ne!(encoded & 0x00f0_0000_0000_0000, 0, "{value:e}");
assert_eq!(decode_real8(encoded).to_bits(), ieee, "{value:e}");
}
}
}
}
#[test]
fn real8_underflow_is_canonical_and_overflow_is_an_error() {
let minimum = 2_f64.powi(-260);
let maximum_exclusive = 2_f64.powi(252);
for magnitude in [
0.0,
f64::from_bits(1),
f64::MIN_POSITIVE,
1e-90,
1e-80,
f64::from_bits(minimum.to_bits() - 1),
] {
for value in [magnitude, -magnitude] {
assert_eq!(encode_real8(value).unwrap(), 0, "{value:e}");
}
}
let largest = f64::from_bits(maximum_exclusive.to_bits() - 1);
assert_eq!(encode_real8(largest).unwrap(), 0x7fff_ffff_ffff_fff8);
assert_eq!(decode_real8(encode_real8(largest).unwrap()), largest);
for value in [
maximum_exclusive,
-maximum_exclusive,
f64::MAX,
-f64::MAX,
f64::INFINITY,
f64::NEG_INFINITY,
f64::NAN,
] {
assert!(encode_real8(value).is_err(), "{value}");
let mut buffer = [0x55; 8];
assert!(pack_real8(&mut buffer, value).is_err());
assert_eq!(buffer, [0x55; 8]);
}
}