From 46e1af3b0f2c28936dfa88063cc5a35f466f5ac3 Mon Sep 17 00:00:00 2001
From: Kostya Shishkov <kostya.shishkov@gmail.com>
Date: Sat, 29 Oct 2011 19:35:20 +0200
Subject: utvideo: handle empty Huffman trees

If the frame is filled with the same colour, encoder may produce no data
and the fill value is indicated by zero code length (the rest of symbols
will have 0xFF for code length, meaning invalid).  So such Huffman trees
should be treated specially.

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>
---
 libavcodec/utvideo.c | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

(limited to 'libavcodec/utvideo.c')

diff --git a/libavcodec/utvideo.c b/libavcodec/utvideo.c
index 5faaa15a0c..4c3b2a1621 100644
--- a/libavcodec/utvideo.c
+++ b/libavcodec/utvideo.c
@@ -66,7 +66,7 @@ static int huff_cmp(const void *a, const void *b)
     return (aa->len - bb->len)*256 + aa->sym - bb->sym;
 }
 
-static int build_huff(const uint8_t *src, VLC *vlc)
+static int build_huff(const uint8_t *src, VLC *vlc, int *fsym)
 {
     int i;
     HuffEntry he[256];
@@ -76,13 +76,18 @@ static int build_huff(const uint8_t *src, VLC *vlc)
     uint8_t syms[256];
     uint32_t code;
 
+    *fsym = -1;
     for (i = 0; i < 256; i++) {
         he[i].sym = i;
         he[i].len = *src++;
     }
     qsort(he, 256, sizeof(*he), huff_cmp);
 
-    if (!he[0].len || he[0].len > 32)
+    if (!he[0].len) {
+        *fsym = he[0].sym;
+        return 0;
+    }
+    if (he[0].len > 32)
         return -1;
 
     last = 255;
@@ -112,13 +117,37 @@ static int decode_plane(UtvideoContext *c, int plane_no,
     int sstart, send;
     VLC vlc;
     GetBitContext gb;
-    int prev;
+    int prev, fsym;
     const int cmask = ~(!plane_no && c->avctx->pix_fmt == PIX_FMT_YUV420P);
 
-    if (build_huff(src, &vlc)) {
+    if (build_huff(src, &vlc, &fsym)) {
         av_log(c->avctx, AV_LOG_ERROR, "Cannot build Huffman codes\n");
         return AVERROR_INVALIDDATA;
     }
+    if (fsym >= 0) { // build_huff reported a symbol to fill slices with
+        send = 0;
+        for (slice = 0; slice < c->slices; slice++) {
+            uint8_t *dest;
+
+            sstart = send;
+            send   = (height * (slice + 1) / c->slices) & cmask;
+            dest   = dst + sstart * stride;
+
+            prev = 0x80;
+            for (j = sstart; j < send; j++) {
+                for (i = 0; i < width * step; i += step) {
+                    pix = fsym;
+                    if (use_pred) {
+                        prev += pix;
+                        pix   = prev;
+                    }
+                    dest[i] = pix;
+                }
+                dest += stride;
+            }
+        }
+        return 0;
+    }
 
     src      += 256;
     src_size -= 256;
-- 
cgit v1.2.3