Fix a bunch of UB bugs (fixed_point mode) (#141)

After fixing some oss-fuzz discoveries new ones have arrived.

Most interesting: typo in (sbr) DST4_32 - it affects both fixed-point and float version.

Author: eustas

libfaad/common.c

@@ -358,7 +358,7 @@ real_t pow2_fix(real_t val)
     int32_t whole = (val >> REAL_BITS);
 
     /* rest = [0..1] */
-    int32_t rest = val - (whole << REAL_BITS);
+    int32_t rest = val & ((1 << REAL_BITS) - 1);
 
     /* index into pow2_tab */
     int32_t index = rest >> (REAL_BITS-TABLE_BITS);
@@ -401,7 +401,7 @@ int32_t pow2_int(real_t val)
     int32_t whole = (val >> REAL_BITS);
 
     /* rest = [0..1] */
-    int32_t rest = val - (whole << REAL_BITS);
+    int32_t rest = val & ((1 << REAL_BITS) - 1);
 
     /* index into pow2_tab */
     int32_t index = rest >> (REAL_BITS-TABLE_BITS);

libfaad/fixed.h

@@ -48,7 +48,9 @@ extern "C" {
 /* FRAC is the fractional only part of the fixed point number [0.0..1.0) */
 #define FRAC_SIZE 32 /* frac is a 32 bit integer */
 #define FRAC_BITS 31
-#define FRAC_PRECISION ((uint32_t)(1 << FRAC_BITS))
+/* Multiplication by power of 2 will be compiled to left-shift */
+#define FRAC_MUL (1u << (FRAC_SIZE - FRAC_BITS))
+#define FRAC_PRECISION ((uint32_t)(1u << FRAC_BITS))
 #define FRAC_MAX 0x7FFFFFFF
 
 typedef int32_t real_t;
@@ -262,7 +264,7 @@ static INLINE void ComplexMult(real_t *y1, real_t *y2,
          : "=d" (__xxo) : "d" (X), "d" (Y) : "A0","A1"); __xxo; })
 #else
   #define _MulHigh(A,B) (real_t)(((int64_t)(A)*(int64_t)(B)+(1u << (FRAC_SIZE-1))) >> FRAC_SIZE)
-  #define MUL_F(A,B) (real_t)(((int64_t)(A)*(int64_t)(B)+(1 << (FRAC_BITS-1))) >> FRAC_BITS)
+  #define MUL_F(A,B) (real_t)(((int64_t)(A)*(int64_t)(B)+(1u << (FRAC_BITS-1))) >> FRAC_BITS)
 #endif
 #endif
   #define MUL_Q2(A,B) (real_t)(((int64_t)(A)*(int64_t)(B)+(1 << (Q2_BITS-1))) >> Q2_BITS)
@@ -273,8 +275,8 @@ static INLINE void ComplexMult(real_t *y1, real_t *y2,
 static INLINE void ComplexMult(real_t *y1, real_t *y2,
     real_t x1, real_t x2, real_t c1, real_t c2)
 {
-    *y1 = (_MulHigh(x1, c1) + _MulHigh(x2, c2))<<(FRAC_SIZE-FRAC_BITS);
-    *y2 = (_MulHigh(x2, c1) - _MulHigh(x1, c2))<<(FRAC_SIZE-FRAC_BITS);
+    *y1 = (_MulHigh(x1, c1) + _MulHigh(x2, c2)) * FRAC_MUL;
+    *y2 = (_MulHigh(x2, c1) - _MulHigh(x1, c2)) * FRAC_MUL;
 }
 
 #endif

libfaad/sbr_dct.c

@@ -675,7 +675,7 @@ void DST4_32(real_t *y, real_t *x)
     f148 = f145 + f146;
     f149 = f147 - f146;
     f150 = f4 + f26;
-    f151 = MUL_F(FRAC_CONST(1.2130114330978077), f4);
+    f151 = MUL_C(COEF_CONST(1.2130114330978077), f4);
     f152 = MUL_F(FRAC_CONST(-0.9700312531945440), f150);
     f153 = MUL_F(FRAC_CONST(-0.7270510732912803), f26);
     f154 = f151 + f152;

libfaad/sbr_hfadj.c

@@ -520,12 +520,16 @@ static void calculate_gain(sbr_info *sbr, sbr_hfadj_info *adj, uint8_t ch)
             else
                 acc1 = log2_int(acc1);
 
+            if (acc2 == 0)
+                acc2 = LOG2_MIN_INF;
+            else
+                acc2 = log2_int(acc2);
 
             /* calculate the maximum gain */
             /* ratio of the energy of the original signal and the energy
              * of the HF generated signal
              */
-            G_max = acc1 - log2_int(acc2) + limGain[sbr->bs_limiter_gains];
+            G_max = acc1 - acc2 + limGain[sbr->bs_limiter_gains];
             G_max = min(G_max, limGain[3]);
 
 
@@ -647,7 +651,7 @@ static void calculate_gain(sbr_info *sbr, sbr_hfadj_info *adj, uint8_t ch)
                         Q_M_size++;
                     }
                 } else {
-                    /* G > G_max */
+                    /* G >= G_max */
                     Q_M_lim[m] = Q_M + G_max - G;
                     G_lim[m] = G_max;
 
@@ -670,10 +674,14 @@ static void calculate_gain(sbr_info *sbr, sbr_hfadj_info *adj, uint8_t ch)
                 den += pow2_int(log2_int_tab[Q_M_size] + Q_M);
             }
 
+            if (den == 0)
+                den = LOG2_MIN_INF;
+            else
+                den = log2_int(den /*+ EPS*/);
 
             /* calculate the final gain */
             /* G_boost: [0..2.51188643] */
-            G_boost = acc1 - log2_int(den /*+ EPS*/);
+            G_boost = acc1 - den;
             G_boost = min(G_boost, REAL_CONST(1.328771237) /* log2(1.584893192 ^ 2) */);
 
 

libfaad/specrec.c

@@ -654,10 +654,10 @@ static uint8_t quant_to_spec(NeAACDecStruct *hDecoder,
                         spec_data[wb+2] = iq2 >> -exp;
                         spec_data[wb+3] = iq3 >> -exp;
                     } else {
-                        spec_data[wb+0] = iq0 << exp;
-                        spec_data[wb+1] = iq1 << exp;
-                        spec_data[wb+2] = iq2 << exp;
-                        spec_data[wb+3] = iq3 << exp;
+                        spec_data[wb+0] = (int32_t)((uint32_t)iq0 << exp);
+                        spec_data[wb+1] = (int32_t)((uint32_t)iq1 << exp);
+                        spec_data[wb+2] = (int32_t)((uint32_t)iq2 << exp);
+                        spec_data[wb+3] = (int32_t)((uint32_t)iq3 << exp);
                     }
                     if (frac != 0)
                     {