The externally calibrated SFF-8472 values were truncated when stored in their 16-bit fields. Coefficients that are not exact in binary floating point lose one unit this way: RX_PWR(1) = 0.7 is stored as 0.69999999, and a raw reading of 1000 gave 699 instead of 700.
Round the calibrated values to the nearest integer instead. Signed-off-by: Roman Khromenok <[email protected]> --- lib/ethdev/sff_8472.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/ethdev/sff_8472.c b/lib/ethdev/sff_8472.c index 8cd41161e3..88d1f4a0a5 100644 --- a/lib/ethdev/sff_8472.c +++ b/lib/ethdev/sff_8472.c @@ -3,6 +3,7 @@ * Implements SFF-8472 optics diagnostics. */ +#include <math.h> #include <stdio.h> #include <string.h> @@ -186,14 +187,17 @@ static float befloattoh(const uint8_t *source) return converter.dst; } -/* Calibrated values are stored in 16-bit fields, saturate the out of range ones */ +/* + * Calibrated values are stored in 16-bit fields: + * round to the nearest integer and saturate the out of range ones. + */ static uint16_t sff_8472_cal_to_u16(double value) { if (!(value > 0)) return 0; if (value >= UINT16_MAX) return UINT16_MAX; - return value; + return lround(value); } static int16_t sff_8472_cal_to_s16(double value) @@ -202,7 +206,7 @@ static int16_t sff_8472_cal_to_s16(double value) return INT16_MIN; if (value >= INT16_MAX) return INT16_MAX; - return value; + return lround(value); } static void sff_8472_calibration(const uint8_t *data, struct sff_diags *sd) -- 2.47.3
