fix(real8): normalize finite values and handle underflow consistently

This commit is contained in:
Jan Petykiewicz 2026-11-15 19:51:13 -08:00
commit f081a808ea

View file

@ -166,79 +166,36 @@ impl fmt::Display for FloatTooBigError {
}
}
/// Convert from float64 to GDS REAL8 representation.
/// Convert float64 to a normalized GDS REAL8 representation.
/// Finite magnitudes below 2^-260 underflow to canonical zero, including -0.0.
/// Nonfinite values and magnitudes at or above 2^252 return an error.
pub fn encode_real8(fnum: f64) -> Result<u64, FloatTooBigError> {
// Split the ieee float bitfields
let ieee = fnum.to_bits();
let ieee_exp = ((ieee >> 52) & 0x7ff) as i32;
if ieee_exp == 0x7ff {
return Err(FloatTooBigError { float_value: fnum });
}
let exp2 = ieee_exp - 1023;
if exp2 < -260 {
// This also handles both signed zeros and every IEEE subnormal.
return Ok(0);
}
let gds_exp = exp2.div_euclid(4) + 65;
if gds_exp > 0x7f {
return Err(FloatTooBigError { float_value: fnum });
}
// The 53-bit IEEE significand needs only 0..3 left shifts to become
// a normalized 56-bit GDS mantissa. Every in-range f64 is exact in REAL8.
let significand = (ieee & 0x000f_ffff_ffff_ffff) | (1_u64 << 52);
let mantissa = significand << exp2.rem_euclid(4);
let sign = ieee & 0x8000_0000_0000_0000;
let ieee_exp = (ieee >> 52) as i32 & 0x7ff;
let ieee_mant = ieee & 0x000f_ffff_ffff_ffff;
let subnorm = (ieee_exp == 0) & (ieee_mant != 0);
if (ieee_exp == 0) & (ieee_mant == 0) {
return Ok(0)
}
// IEEE normal double is (1 + ieee_mant / 2^52) * 2^(ieee_exp - 1023)
// IEEE subnormal double is (ieee_mant / 2^52) * 2^(-1022)
// GDS real8 is (gds_mant / 2^(7*8)) * 16^(gds_exp - 64)
// = (gds_mant / 2^56) * 2^(4 * gds_exp - 256)
// Convert exponent.
let exp2 = if subnorm { -1022 } else {ieee_exp + 1 - 1023}; // +1 is due to mantissa (1.xxxx in IEEE vs 0.xxxxx in GDSII)
let mut exp16 = exp2 / 4;
let rest = exp2 % 4;
// Compensate for exponent coarseness
let comp = rest != 0;
let mut shift;
if comp {
exp16 += 1;
shift = 4 - rest;
} else {
shift = rest;
}
shift -= 3; // account for gds bit position
// add leading one
let mut gds_mant_unshifted = ieee_mant;
if !subnorm {
gds_mant_unshifted += 0x10_0000_0000_0000;
}
let mut gds_mant = if shift > 0 {
gds_mant_unshifted >> shift
} else {
gds_mant_unshifted << -shift
};
// add gds exponent bias
let mut gds_exp = exp16 + 64;
if gds_exp < -14 {
// number is too small
return Ok(0)
}
let neg_biased = gds_exp < 0;
if neg_biased {
gds_mant >>= gds_exp * 4;
gds_exp = 0;
}
let too_big = (gds_exp > 0x7f) & !subnorm;
if too_big {
return Err(FloatTooBigError{float_value: fnum});
}
let gds_exp_bits = (gds_exp as u64) << 56;
let real8 = sign | gds_exp_bits | gds_mant;
Ok(real8)
Ok(sign | ((gds_exp as u64) << 56) | mantissa)
}
#[cfg(test)]
mod tests {
#[test]