Merge remote-tracking branch 'qatar/release/0.6' into release/0.6

* qatar/release/0.6:
  matroskadec: Fix a bug where a pointer was cached to an array that might later move due to a realloc()
  vorbis: Avoid some out-of-bounds reads
  vp3: fix streams with non-zero last coefficient
  vp3: fix oob read for negative tokens and memleaks on error. (cherry picked from commit 8370e426e42f2e4b9d14a1fb8107ecfe5163ce7f)

Merged-by: Michael Niedermayer <michaelni@gmx.at>

2 files changed, 28 insertions, 9 deletions

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index 47388d8302..109737976c 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -156,7 +156,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     }
 }
 
-static inline void render_line_unrolled(intptr_t x, intptr_t y, int x1,
+static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {
@@ -179,7 +179,7 @@ static inline void render_line_unrolled(intptr_t x, intptr_t y, int x1,
     }
 }
 
-static void render_line(int x0, int y0, int x1, int y1, float *buf)
+static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
@@ -189,10 +189,10 @@ static void render_line(int x0, int y0, int x1, int y1, float *buf)
     if (ady*2 <= adx) { // optimized common case
         render_line_unrolled(x0, y0, x1, sy, ady, adx, buf);
     } else {
-        int base = dy / adx;
-        int x    = x0;
-        int y    = y0;
-        int err  = -adx;
+        int base  = dy / adx;
+        int x     = x0;
+        uint8_t y = y0;
+        int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {
             y += base;
@@ -210,7 +210,8 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
                                   uint_fast16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {
-    int lx, ly, i;
+    int lx, i;
+    uint8_t ly;
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {
diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index 99842e07c7..5e13bdcba1 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -884,7 +884,7 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
             /* decode a VLC into a token */
             token = get_vlc2(gb, vlc_table, 11, 3);
             /* use the token to get a zero run, a coefficient, and an eob run */
-            if (token <= 6) {
+            if ((unsigned) token <= 6U) {
                 eob_run = eob_run_base[token];
                 if (eob_run_get_bits[token])
                     eob_run += get_bits(gb, eob_run_get_bits[token]);
@@ -902,7 +902,7 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
                     coeff_i        += eob_run;
                     eob_run = 0;
                 }
-            } else {
+            } else if (token >= 0) {
                 bits_to_get = coeff_get_bits[token];
                 if (bits_to_get)
                     bits_to_get = get_bits(gb, bits_to_get);
@@ -936,6 +936,10 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
                 for (i = coeff_index+1; i <= coeff_index+zero_run; i++)
                     s->num_coded_frags[plane][i]--;
                 coeff_i++;
+            } else {
+                av_log(s->avctx, AV_LOG_ERROR,
+                       "Invalid token %d\n", token);
+                return -1;
             }
     }
 
@@ -985,6 +989,8 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     /* unpack the Y plane DC coefficients */
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_y_table], 0,
         0, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
 
     /* reverse prediction of the Y-plane DC coefficients */
     reverse_dc_prediction(s, 0, s->fragment_width[0], s->fragment_height[0]);
@@ -992,8 +998,12 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     /* unpack the C plane DC coefficients */
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_c_table], 0,
         1, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_c_table], 0,
         2, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
 
     /* reverse prediction of the C-plane DC coefficients */
     if (!(s->avctx->flags & CODEC_FLAG_GRAY))
@@ -1030,11 +1040,17 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     for (i = 1; i <= 63; i++) {
             residual_eob_run = unpack_vlcs(s, gb, y_tables[i], i,
                 0, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
 
             residual_eob_run = unpack_vlcs(s, gb, c_tables[i], i,
                 1, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
             residual_eob_run = unpack_vlcs(s, gb, c_tables[i], i,
                 2, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
     }
 
     return 0;
@@ -1300,6 +1316,8 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag,
             return i;
         }
     } while (i < 64);
+    // return value is expected to be a valid level
+    i--;
 end:
     // the actual DC+prediction is in the fragment structure
     block[0] = frag->dc * s->qmat[0][inter][plane][0];