Merge remote-tracking branch 'qatar/master'n0.11-dev

* qatar/master:
  smacker: Sanity check huffman tables found in the headers.
  smacker: remove dead store
  qdm2: Check data block size for bytes to bits overflow.
  mxfdec: Fix files with essence containers larger than 2 GiB.
  mxfdec: Employ correct printf conversion specifiers for POSIX int types.
  vc1: always read the bfraction element for interlaced fields
  fate: add XWD image regression test
  lavf: prevent infinite loops while flushing in avformat_find_stream_info
  matroskadec: Pad AAC extradata.
  ismindex: Fix build on mingw

Conflicts:
	libavformat/mxfdec.c
	libavformat/utils.c
	tests/lavf-regression.sh

Merged-by: Michael Niedermayer <michaelni@gmx.at>

6 files changed, 55 insertions, 36 deletions

diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index 2a8bae8a1b..a60a2b051b 100644
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -128,12 +128,12 @@ static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t pref
  */
 static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx)
 {
+    if (hc->current + 1 >= hc->length) {
+        av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");
+        return -1;
+    }
     if(!get_bits1(gb)){ //Leaf
         int val, i1, i2, b1, b2;
-        if(hc->current >= hc->length){
-            av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");
-            return -1;
-        }
         b1 = get_bits_count(gb);
         i1 = ctx->v1->table ? get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3) : 0;
         b1 = get_bits_count(gb) - b1;
@@ -157,7 +157,7 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx
         hc->values[hc->current++] = val;
         return 1;
     } else { //Node
-        int r = 0, t;
+        int r = 0, r_new, t;
 
         t = hc->current++;
         r = smacker_decode_bigtree(gb, hc, ctx);
@@ -165,8 +165,10 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx
             return r;
         hc->values[t] = SMK_NODE | r;
         r++;
-        r += smacker_decode_bigtree(gb, hc, ctx);
-        return r;
+        r_new = smacker_decode_bigtree(gb, hc, ctx);
+        if (r_new < 0)
+            return r_new;
+        return r + r_new;
     }
 }
 
@@ -181,6 +183,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
     VLC vlc[2];
     int escapes[3];
     DBCtx ctx;
+    int err = 0;
 
     if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow
         av_log(smk->avctx, AV_LOG_ERROR, "size too large\n");
@@ -254,7 +257,8 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
     huff.current = 0;
     huff.values = av_mallocz(huff.length * sizeof(int));
 
-    smacker_decode_bigtree(gb, &huff, &ctx);
+    if (smacker_decode_bigtree(gb, &huff, &ctx) < 0)
+        err = -1;
     skip_bits1(gb);
     if(ctx.last[0] == -1) ctx.last[0] = huff.current++;
     if(ctx.last[1] == -1) ctx.last[1] = huff.current++;
@@ -278,7 +282,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
     av_free(tmp2.lengths);
     av_free(tmp2.values);
 
-    return 0;
+    return err;
 }
 
 static int decode_header_trees(SmackVContext *smk) {
@@ -339,16 +343,14 @@ static av_always_inline void last_reset(int *recode, int *last) {
 /* get code and update history */
 static av_always_inline int smk_get_code(GetBitContext *gb, int *recode, int *last) {
     register int *table = recode;
-    int v, b;
+    int v;
 
-    b = get_bits_count(gb);
     while(*table & SMK_NODE) {
         if(get_bits1(gb))
             table += (*table) & (~SMK_NODE);
         table++;
     }
     v = *table;
-    b = get_bits_count(gb) - b;
 
     if(v != recode[last[0]]) {
         recode[last[2]] = recode[last[1]];
diff --git a/libavcodec/vc1.c b/libavcodec/vc1.c
index 64884fcc50..a012967e3e 100644
--- a/libavcodec/vc1.c
+++ b/libavcodec/vc1.c
@@ -901,20 +901,18 @@ int vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
     if (v->field_mode) {
         if (!v->refdist_flag)
             v->refdist = 0;
-        else {
-            if ((v->s.pict_type != AV_PICTURE_TYPE_B)
-                && (v->s.pict_type != AV_PICTURE_TYPE_BI)) {
-                v->refdist = get_bits(gb, 2);
-                if (v->refdist == 3)
-                    v->refdist += get_unary(gb, 0, 16);
-            } else {
-                v->bfraction_lut_index = get_vlc2(gb, ff_vc1_bfraction_vlc.table, VC1_BFRACTION_VLC_BITS, 1);
-                v->bfraction           = ff_vc1_bfraction_lut[v->bfraction_lut_index];
-                v->frfd = (v->bfraction * v->refdist) >> 8;
-                v->brfd = v->refdist - v->frfd - 1;
-                if (v->brfd < 0)
-                    v->brfd = 0;
-            }
+        else if ((v->s.pict_type != AV_PICTURE_TYPE_B) && (v->s.pict_type != AV_PICTURE_TYPE_BI)) {
+            v->refdist = get_bits(gb, 2);
+            if (v->refdist == 3)
+                v->refdist += get_unary(gb, 0, 16);
+        }
+        if ((v->s.pict_type == AV_PICTURE_TYPE_B) || (v->s.pict_type == AV_PICTURE_TYPE_BI)) {
+            v->bfraction_lut_index = get_vlc2(gb, ff_vc1_bfraction_vlc.table, VC1_BFRACTION_VLC_BITS, 1);
+            v->bfraction           = ff_vc1_bfraction_lut[v->bfraction_lut_index];
+            v->frfd = (v->bfraction * v->refdist) >> 8;
+            v->brfd = v->refdist - v->frfd - 1;
+            if (v->brfd < 0)
+                v->brfd = 0;
         }
         goto parse_common_info;
     }
diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c
index cb38e5caaf..c8baf189f8 100644
--- a/libavformat/mxfdec.c
+++ b/libavformat/mxfdec.c
@@ -471,15 +471,18 @@ static int mxf_read_partition_pack(void *arg, AVIOContext *pb, int tag, int size
     /* some files don'thave FooterPartition set in every partition */
     if (footer_partition) {
         if (mxf->footer_partition && mxf->footer_partition != footer_partition) {
-            av_log(mxf->fc, AV_LOG_ERROR, "inconsistent FooterPartition value: %" PRIi64 " != %" PRIi64 "\n",
+            av_log(mxf->fc, AV_LOG_ERROR,
+                   "inconsistent FooterPartition value: %"PRIu64" != %"PRIu64"\n",
                    mxf->footer_partition, footer_partition);
         } else {
             mxf->footer_partition = footer_partition;
         }
     }
 
-    av_dlog(mxf->fc, "PartitionPack: ThisPartition = 0x%" PRIx64 ", PreviousPartition = 0x%" PRIx64 ", "
-            "FooterPartition = 0x%" PRIx64 ", IndexSID = %i, BodySID = %i\n",
+    av_dlog(mxf->fc,
+            "PartitionPack: ThisPartition = 0x%"PRIX64
+            ", PreviousPartition = 0x%"PRIX64", "
+            "FooterPartition = 0x%"PRIX64", IndexSID = %i, BodySID = %i\n",
             partition->this_partition,
             partition->previous_partition, footer_partition,
             partition->index_sid, partition->body_sid);
@@ -959,7 +962,8 @@ static int mxf_absolute_bodysid_offset(MXFContext *mxf, int body_sid, int64_t of
         offset -= p->essence_length;
     }
 
-    av_log(mxf->fc, AV_LOG_ERROR, "failed to find absolute offset of %" PRIx64" in BodySID %i - partial file?\n",
+    av_log(mxf->fc, AV_LOG_ERROR,
+           "failed to find absolute offset of %"PRIX64" in BodySID %i - partial file?\n",
            offset_in, body_sid);
 
     return AVERROR_INVALIDDATA;
@@ -1602,7 +1606,8 @@ static void mxf_compute_essence_containers(MXFContext *mxf)
         if (p->essence_length < 0) {
             /* next ThisPartition < essence_offset */
             p->essence_length = 0;
-            av_log(mxf->fc, AV_LOG_ERROR, "partition %i: bad ThisPartition = %" PRIx64 "\n",
+            av_log(mxf->fc, AV_LOG_ERROR,
+                   "partition %i: bad ThisPartition = %"PRIX64"\n",
                    x+1, mxf->partitions[x+1].this_partition);
         }
     }
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 005704c7f0..c1de71d29e 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -2236,6 +2236,7 @@ static int has_decode_delay_been_guessed(AVStream *st)
         st->info->nb_decoded_frames >= 6;
 }
 
+/* returns 1 or 0 if or if not decoded data was returned, or a negative error */
 static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **options)
 {
     AVCodec *codec;
@@ -2283,6 +2284,7 @@ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **option
                 st->info->nb_decoded_frames++;
             pkt.data += ret;
             pkt.size -= ret;
+            ret       = got_picture;
         }
     }
     if(!pkt.data && !got_picture)
@@ -2589,16 +2591,21 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
             st = ic->streams[i];
 
             /* flush the decoders */
-            while ((err = try_decode_frame(st, &empty_pkt,
+            do {
+                err = try_decode_frame(st, &empty_pkt,
                                            (options && i < orig_nb_streams) ?
-                                            &options[i] : NULL)) >= 0)
-                if (has_codec_parameters(st->codec))
-                    break;
+                                           &options[i] : NULL);
+            } while (err > 0 && !has_codec_parameters(st->codec));
 
+            if (err < 0) {
+                av_log(ic, AV_LOG_INFO,
+                       "decoding for stream %d failed\n", st->index);
+            }
             if (!has_codec_parameters(st->codec)){
                 char buf[256];
                 avcodec_string(buf, sizeof(buf), st->codec, 0);
-                av_log(ic, AV_LOG_WARNING, "Could not find codec parameters (%s)\n", buf);
+                av_log(ic, AV_LOG_WARNING,
+                       "Could not find codec parameters (%s)\n", buf);
             } else {
                 ret = 0;
             }
diff --git a/tests/lavf-regression.sh b/tests/lavf-regression.sh
index af34e86893..884b10b989 100755
--- a/tests/lavf-regression.sh
+++ b/tests/lavf-regression.sh
@@ -187,6 +187,10 @@ if [ -n "$do_dpx" ] ; then
 do_image_formats dpx
 fi
 
+if [ -n "$do_xwd" ] ; then
+do_image_formats xwd
+fi
+
 # audio only
 
 if [ -n "$do_wav" ] ; then
diff --git a/tests/ref/lavf/xwd b/tests/ref/lavf/xwd
new file mode 100644
index 0000000000..6ba88cd2fd
--- /dev/null
+++ b/tests/ref/lavf/xwd
@@ -0,0 +1,3 @@
+0ab6cd4fe5fe85a3f3e87508c2eadfa0 *./tests/data/images/xwd/02.xwd
+./tests/data/images/xwd/%02d.xwd CRC=0x6da01946
+304239 ./tests/data/images/xwd/02.xwd