From 3e5c3d5e01d80d575eee507882bd72614ea9297a Mon Sep 17 00:00:00 2001 From: ratchov Date: Mon, 4 May 2015 12:51:13 +0000 Subject: [PATCH] Fix clipping bug in float to fixed point conversion, causing audible glitches when samples smaller than the quantification step are converted. --- usr.bin/aucat/dsp.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/usr.bin/aucat/dsp.c b/usr.bin/aucat/dsp.c index 927e00f1971..4575b26e5b3 100644 --- a/usr.bin/aucat/dsp.c +++ b/usr.bin/aucat/dsp.c @@ -1,4 +1,4 @@ -/* $OpenBSD: dsp.c,v 1.1 2015/01/21 08:43:55 ratchov Exp $ */ +/* $OpenBSD: dsp.c,v 1.2 2015/05/04 12:51:13 ratchov Exp $ */ /* * Copyright (c) 2008-2012 Alexandre Ratchov * @@ -604,9 +604,19 @@ f32_to_adata(unsigned int x) s = (x >> 31); e = (x >> 23) & 0xff; m = (x << 8) | 0x80000000; - if (e < 127 - 24) + + /* + * f32 exponent is (e - 127) and the point is after the 31-th + * bit, thus the shift is: + * + * 31 - (BITS - 1) - (e - 127) + * + * to ensure output is in the 0..(2^BITS)-1 range, the minimum + * shift is 31 - (BITS - 1), and maximum shift is 31 + */ + if (e < 127 - (ADATA_BITS - 1)) y = 0; - else if (e > 127 - 1) + else if (e > 127) y = ADATA_UNIT - 1; else y = m >> (127 + (32 - ADATA_BITS) - e); -- 2.20.1