Merge remote-tracking branch 'qatar/master'

* qatar/master: (35 commits)
  libopencore-amr: check output buffer size before decoding
  libopencore-amr: remove unneeded buf_size==0 check.
  libopencore-amr: remove unneeded frame_count field.
  aac_latm: remove unneeded check for zero-size packet.
  pcmdec: fix output buffer size check by calculating the actual output size prior to decoding.
  pcmdec: move codec-specific variable declarations to the corresponding codec blocks.
  pcmdec: return buf_size instead of src-buf.
  avcodec: remove the Zork PCM encoder.
  pcm_zork: use AV_SAMPLE_FMT_U8 instead of shifting all samples by 8.
  pcmenc: remove unneeded sample_fmt check.
  pcmdec: move number of channels check to pcm_decode_init()
  pcmdec: remove unnecessary check for sample_fmt change
  pcmdec: move DVD PCM bits_per_coded_sample check near to the code that sets the sample size.
  pcmdec: do not needlessly set *data_size to 0
  alacdec: remove unneeded NULL or zero-size packet checks.
  alacdec: simplify buffer allocation by using FF_ALLOC_OR_GOTO()
  alacdec: ask for a sample for unsupported sample depths.
  alacdec: cosmetics: use 'ch' instead of 'chan' to iterate channels
  alacdec: move some declarations to the top of the function
  alacdec: always use get_sbits_long() for uncompressed samples
  ...

Conflicts:
	libavcodec/pcm.c
	tests/ref/acodec/pcm

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 172 insertions, 210 deletions

diff --git a/libavcodec/alac.c b/libavcodec/alac.c
index 06bf6f8c78..4e143270a5 100644
--- a/libavcodec/alac.c
+++ b/libavcodec/alac.c
@@ -72,7 +72,7 @@ typedef struct {
 
     int32_t *outputsamples_buffer[MAX_CHANNELS];
 
-    int32_t *wasted_bits_buffer[MAX_CHANNELS];
+    int32_t *extra_bits_buffer[MAX_CHANNELS];
 
     /* stuff from setinfo */
     uint32_t setinfo_max_samples_per_frame; /* 0x1000 = 4096 */    /* max samples per frame? */
@@ -82,58 +82,9 @@ typedef struct {
     uint8_t setinfo_rice_kmodifier; /* 0x0e */
     /* end setinfo stuff */
 
-    int wasted_bits;
+    int extra_bits;                         /**< number of extra bits beyond 16-bit */
 } ALACContext;
 
-static void allocate_buffers(ALACContext *alac)
-{
-    int chan;
-    for (chan = 0; chan < MAX_CHANNELS; chan++) {
-        alac->predicterror_buffer[chan] =
-            av_malloc(alac->setinfo_max_samples_per_frame * 4);
-
-        alac->outputsamples_buffer[chan] =
-            av_malloc(alac->setinfo_max_samples_per_frame * 4);
-
-        alac->wasted_bits_buffer[chan] = av_malloc(alac->setinfo_max_samples_per_frame * 4);
-    }
-}
-
-static int alac_set_info(ALACContext *alac)
-{
-    const unsigned char *ptr = alac->avctx->extradata;
-
-    ptr += 4; /* size */
-    ptr += 4; /* alac */
-    ptr += 4; /* 0 ? */
-
-    if(AV_RB32(ptr) >= UINT_MAX/4){
-        av_log(alac->avctx, AV_LOG_ERROR, "setinfo_max_samples_per_frame too large\n");
-        return -1;
-    }
-
-    /* buffer size / 2 ? */
-    alac->setinfo_max_samples_per_frame = bytestream_get_be32(&ptr);
-    ptr++;                          /* ??? */
-    alac->setinfo_sample_size           = *ptr++;
-    if (alac->setinfo_sample_size > 32) {
-        av_log(alac->avctx, AV_LOG_ERROR, "setinfo_sample_size too large\n");
-        return -1;
-    }
-    alac->setinfo_rice_historymult      = *ptr++;
-    alac->setinfo_rice_initialhistory   = *ptr++;
-    alac->setinfo_rice_kmodifier        = *ptr++;
-    ptr++;                         /* channels? */
-    bytestream_get_be16(&ptr);      /* ??? */
-    bytestream_get_be32(&ptr);      /* max coded frame size */
-    bytestream_get_be32(&ptr);      /* bitrate ? */
-    bytestream_get_be32(&ptr);      /* samplerate */
-
-    allocate_buffers(alac);
-
-    return 0;
-}
-
 static inline int decode_scalar(GetBitContext *gb, int k, int limit, int readsamplesize){
     /* read x - number of 1s before 0 represent the rice */
     int x = get_unary_0_9(gb);
@@ -347,93 +298,56 @@ static void predictor_decompress_fir_adapt(int32_t *error_buffer,
     }
 }
 
-static void reconstruct_stereo_16(int32_t *buffer[MAX_CHANNELS],
-                                  int16_t *buffer_out,
-                                  int numchannels, int numsamples,
-                                  uint8_t interlacing_shift,
-                                  uint8_t interlacing_leftweight)
+static void decorrelate_stereo(int32_t *buffer[MAX_CHANNELS],
+                               int numsamples, uint8_t interlacing_shift,
+                               uint8_t interlacing_leftweight)
 {
     int i;
-    if (numsamples <= 0)
-        return;
-
-    /* weighted interlacing */
-    if (interlacing_leftweight) {
-        for (i = 0; i < numsamples; i++) {
-            int32_t a, b;
 
-            a = buffer[0][i];
-            b = buffer[1][i];
+    for (i = 0; i < numsamples; i++) {
+        int32_t a, b;
 
-            a -= (b * interlacing_leftweight) >> interlacing_shift;
-            b += a;
+        a = buffer[0][i];
+        b = buffer[1][i];
 
-            buffer_out[i*numchannels] = b;
-            buffer_out[i*numchannels + 1] = a;
-        }
+        a -= (b * interlacing_leftweight) >> interlacing_shift;
+        b += a;
 
-        return;
+        buffer[0][i] = b;
+        buffer[1][i] = a;
     }
+}
 
-    /* otherwise basic interlacing took place */
-    for (i = 0; i < numsamples; i++) {
-        int16_t left, right;
-
-        left = buffer[0][i];
-        right = buffer[1][i];
+static void append_extra_bits(int32_t *buffer[MAX_CHANNELS],
+                              int32_t *extra_bits_buffer[MAX_CHANNELS],
+                              int extra_bits, int numchannels, int numsamples)
+{
+    int i, ch;
 
-        buffer_out[i*numchannels] = left;
-        buffer_out[i*numchannels + 1] = right;
-    }
+    for (ch = 0; ch < numchannels; ch++)
+        for (i = 0; i < numsamples; i++)
+            buffer[ch][i] = (buffer[ch][i] << extra_bits) | extra_bits_buffer[ch][i];
 }
 
-static void decorrelate_stereo_24(int32_t *buffer[MAX_CHANNELS],
-                                  int32_t *buffer_out,
-                                  int32_t *wasted_bits_buffer[MAX_CHANNELS],
-                                  int wasted_bits,
-                                  int numchannels, int numsamples,
-                                  uint8_t interlacing_shift,
-                                  uint8_t interlacing_leftweight)
+static void interleave_stereo_16(int32_t *buffer[MAX_CHANNELS],
+                                 int16_t *buffer_out, int numsamples)
 {
     int i;
 
-    if (numsamples <= 0)
-        return;
-
-    /* weighted interlacing */
-    if (interlacing_leftweight) {
-        for (i = 0; i < numsamples; i++) {
-            int32_t a, b;
-
-            a = buffer[0][i];
-            b = buffer[1][i];
-
-            a -= (b * interlacing_leftweight) >> interlacing_shift;
-            b += a;
-
-            if (wasted_bits) {
-                b  = (b  << wasted_bits) | wasted_bits_buffer[0][i];
-                a  = (a  << wasted_bits) | wasted_bits_buffer[1][i];
-            }
-
-            buffer_out[i * numchannels]     = b << 8;
-            buffer_out[i * numchannels + 1] = a << 8;
-        }
-    } else {
-        for (i = 0; i < numsamples; i++) {
-            int32_t left, right;
-
-            left  = buffer[0][i];
-            right = buffer[1][i];
+    for (i = 0; i < numsamples; i++) {
+        *buffer_out++ = buffer[0][i];
+        *buffer_out++ = buffer[1][i];
+    }
+}
 
-            if (wasted_bits) {
-                left   = (left   << wasted_bits) | wasted_bits_buffer[0][i];
-                right  = (right  << wasted_bits) | wasted_bits_buffer[1][i];
-            }
+static void interleave_stereo_24(int32_t *buffer[MAX_CHANNELS],
+                                 int32_t *buffer_out, int numsamples)
+{
+    int i;
 
-            buffer_out[i * numchannels]     = left  << 8;
-            buffer_out[i * numchannels + 1] = right << 8;
-        }
+    for (i = 0; i < numsamples; i++) {
+        *buffer_out++ = buffer[0][i] << 8;
+        *buffer_out++ = buffer[1][i] << 8;
     }
 }
 
@@ -452,18 +366,14 @@ static int alac_decode_frame(AVCodecContext *avctx,
     int isnotcompressed;
     uint8_t interlacing_shift;
     uint8_t interlacing_leftweight;
-
-    /* short-circuit null buffers */
-    if (!inbuffer || !input_buffer_size)
-        return -1;
+    int i, ch;
 
     init_get_bits(&alac->gb, inbuffer, input_buffer_size * 8);
 
     channels = get_bits(&alac->gb, 3) + 1;
-    if (channels > MAX_CHANNELS) {
-        av_log(avctx, AV_LOG_ERROR, "channels > %d not supported\n",
-               MAX_CHANNELS);
-        return -1;
+    if (channels != avctx->channels) {
+        av_log(avctx, AV_LOG_ERROR, "frame header channel count mismatch\n");
+        return AVERROR_INVALIDDATA;
     }
 
     /* 2^result = something to do with output waiting.
@@ -476,7 +386,7 @@ static int alac_decode_frame(AVCodecContext *avctx,
     /* the output sample size is stored soon */
     hassize = get_bits1(&alac->gb);
 
-    alac->wasted_bits = get_bits(&alac->gb, 2) << 3;
+    alac->extra_bits = get_bits(&alac->gb, 2) << 3;
 
     /* whether the frame is compressed */
     isnotcompressed = get_bits1(&alac->gb);
@@ -491,17 +401,7 @@ static int alac_decode_frame(AVCodecContext *avctx,
     } else
         outputsamples = alac->setinfo_max_samples_per_frame;
 
-    switch (alac->setinfo_sample_size) {
-    case 16: avctx->sample_fmt    = AV_SAMPLE_FMT_S16;
-             alac->bytespersample = channels << 1;
-             break;
-    case 24: avctx->sample_fmt    = AV_SAMPLE_FMT_S32;
-             alac->bytespersample = channels << 2;
-             break;
-    default: av_log(avctx, AV_LOG_ERROR, "Sample depth %d is not supported.\n",
-                    alac->setinfo_sample_size);
-             return -1;
-    }
+    alac->bytespersample = channels * av_get_bytes_per_sample(avctx->sample_fmt);
 
     if(outputsamples > *outputsize / alac->bytespersample){
         av_log(avctx, AV_LOG_ERROR, "sample buffer too small\n");
@@ -509,7 +409,7 @@ static int alac_decode_frame(AVCodecContext *avctx,
     }
 
     *outputsize = outputsamples * alac->bytespersample;
-    readsamplesize = alac->setinfo_sample_size - (alac->wasted_bits) + channels - 1;
+    readsamplesize = alac->setinfo_sample_size - alac->extra_bits + channels - 1;
     if (readsamplesize > MIN_CACHE_BITS) {
         av_log(avctx, AV_LOG_ERROR, "readsamplesize too big (%d)\n", readsamplesize);
         return -1;
@@ -522,51 +422,49 @@ static int alac_decode_frame(AVCodecContext *avctx,
         int prediction_type[MAX_CHANNELS];
         int prediction_quantitization[MAX_CHANNELS];
         int ricemodifier[MAX_CHANNELS];
-        int i, chan;
 
         interlacing_shift = get_bits(&alac->gb, 8);
         interlacing_leftweight = get_bits(&alac->gb, 8);
 
-        for (chan = 0; chan < channels; chan++) {
-            prediction_type[chan] = get_bits(&alac->gb, 4);
-            prediction_quantitization[chan] = get_bits(&alac->gb, 4);
+        for (ch = 0; ch < channels; ch++) {
+            prediction_type[ch] = get_bits(&alac->gb, 4);
+            prediction_quantitization[ch] = get_bits(&alac->gb, 4);
 
-            ricemodifier[chan] = get_bits(&alac->gb, 3);
-            predictor_coef_num[chan] = get_bits(&alac->gb, 5);
+            ricemodifier[ch] = get_bits(&alac->gb, 3);
+            predictor_coef_num[ch] = get_bits(&alac->gb, 5);
 
             /* read the predictor table */
-            for (i = 0; i < predictor_coef_num[chan]; i++)
-                predictor_coef_table[chan][i] = (int16_t)get_bits(&alac->gb, 16);
+            for (i = 0; i < predictor_coef_num[ch]; i++)
+                predictor_coef_table[ch][i] = (int16_t)get_bits(&alac->gb, 16);
         }
 
-        if (alac->wasted_bits) {
-            int i, ch;
+        if (alac->extra_bits) {
             for (i = 0; i < outputsamples; i++) {
                 for (ch = 0; ch < channels; ch++)
-                    alac->wasted_bits_buffer[ch][i] = get_bits(&alac->gb, alac->wasted_bits);
+                    alac->extra_bits_buffer[ch][i] = get_bits(&alac->gb, alac->extra_bits);
             }
         }
-        for (chan = 0; chan < channels; chan++) {
+        for (ch = 0; ch < channels; ch++) {
             bastardized_rice_decompress(alac,
-                                        alac->predicterror_buffer[chan],
+                                        alac->predicterror_buffer[ch],
                                         outputsamples,
                                         readsamplesize,
                                         alac->setinfo_rice_initialhistory,
                                         alac->setinfo_rice_kmodifier,
-                                        ricemodifier[chan] * alac->setinfo_rice_historymult / 4,
+                                        ricemodifier[ch] * alac->setinfo_rice_historymult / 4,
                                         (1 << alac->setinfo_rice_kmodifier) - 1);
 
-            if (prediction_type[chan] == 0) {
+            if (prediction_type[ch] == 0) {
                 /* adaptive fir */
-                predictor_decompress_fir_adapt(alac->predicterror_buffer[chan],
-                                               alac->outputsamples_buffer[chan],
+                predictor_decompress_fir_adapt(alac->predicterror_buffer[ch],
+                                               alac->outputsamples_buffer[ch],
                                                outputsamples,
                                                readsamplesize,
-                                               predictor_coef_table[chan],
-                                               predictor_coef_num[chan],
-                                               prediction_quantitization[chan]);
+                                               predictor_coef_table[ch],
+                                               predictor_coef_num[ch],
+                                               prediction_quantitization[ch]);
             } else {
-                av_log(avctx, AV_LOG_ERROR, "FIXME: unhandled prediction type: %i\n", prediction_type[chan]);
+                av_log(avctx, AV_LOG_ERROR, "FIXME: unhandled prediction type: %i\n", prediction_type[ch]);
                 /* I think the only other prediction type (or perhaps this is
                  * just a boolean?) runs adaptive fir twice.. like:
                  * predictor_decompress_fir_adapt(predictor_error, tempout, ...)
@@ -577,44 +475,35 @@ static int alac_decode_frame(AVCodecContext *avctx,
         }
     } else {
         /* not compressed, easy case */
-        int i, chan;
-        if (alac->setinfo_sample_size <= 16) {
-        for (i = 0; i < outputsamples; i++)
-            for (chan = 0; chan < channels; chan++) {
-                int32_t audiobits;
-
-                audiobits = get_sbits_long(&alac->gb, alac->setinfo_sample_size);
-
-                alac->outputsamples_buffer[chan][i] = audiobits;
-            }
-        } else {
-            for (i = 0; i < outputsamples; i++) {
-                for (chan = 0; chan < channels; chan++) {
-                    alac->outputsamples_buffer[chan][i] = get_bits(&alac->gb,
-                                                          alac->setinfo_sample_size);
-                    alac->outputsamples_buffer[chan][i] = sign_extend(alac->outputsamples_buffer[chan][i],
-                                                                      alac->setinfo_sample_size);
-                }
+        for (i = 0; i < outputsamples; i++) {
+            for (ch = 0; ch < channels; ch++) {
+                alac->outputsamples_buffer[ch][i] = get_sbits_long(&alac->gb,
+                                                                   alac->setinfo_sample_size);
             }
         }
-        alac->wasted_bits = 0;
+        alac->extra_bits = 0;
         interlacing_shift = 0;
         interlacing_leftweight = 0;
     }
     if (get_bits(&alac->gb, 3) != 7)
         av_log(avctx, AV_LOG_ERROR, "Error : Wrong End Of Frame\n");
 
+    if (channels == 2 && interlacing_leftweight) {
+        decorrelate_stereo(alac->outputsamples_buffer, outputsamples,
+                           interlacing_shift, interlacing_leftweight);
+    }
+
+    if (alac->extra_bits) {
+        append_extra_bits(alac->outputsamples_buffer, alac->extra_bits_buffer,
+                          alac->extra_bits, alac->numchannels, outputsamples);
+    }
+
     switch(alac->setinfo_sample_size) {
     case 16:
         if (channels == 2) {
-            reconstruct_stereo_16(alac->outputsamples_buffer,
-                                  (int16_t*)outbuffer,
-                                  alac->numchannels,
-                                  outputsamples,
-                                  interlacing_shift,
-                                  interlacing_leftweight);
+            interleave_stereo_16(alac->outputsamples_buffer, outbuffer,
+                                 outputsamples);
         } else {
-            int i;
             for (i = 0; i < outputsamples; i++) {
                 ((int16_t*)outbuffer)[i] = alac->outputsamples_buffer[0][i];
             }
@@ -622,16 +511,9 @@ static int alac_decode_frame(AVCodecContext *avctx,
         break;
     case 24:
         if (channels == 2) {
-            decorrelate_stereo_24(alac->outputsamples_buffer,
-                                  outbuffer,
-                                  alac->wasted_bits_buffer,
-                                  alac->wasted_bits,
-                                  alac->numchannels,
-                                  outputsamples,
-                                  interlacing_shift,
-                                  interlacing_leftweight);
+            interleave_stereo_24(alac->outputsamples_buffer, outbuffer,
+                                 outputsamples);
         } else {
-            int i;
             for (i = 0; i < outputsamples; i++)
                 ((int32_t *)outbuffer)[i] = alac->outputsamples_buffer[0][i] << 8;
         }
@@ -644,11 +526,75 @@ static int alac_decode_frame(AVCodecContext *avctx,
     return input_buffer_size;
 }
 
+static av_cold int alac_decode_close(AVCodecContext *avctx)
+{
+    ALACContext *alac = avctx->priv_data;
+
+    int ch;
+    for (ch = 0; ch < alac->numchannels; ch++) {
+        av_freep(&alac->predicterror_buffer[ch]);
+        av_freep(&alac->outputsamples_buffer[ch]);
+        av_freep(&alac->extra_bits_buffer[ch]);
+    }
+
+    return 0;
+}
+
+static int allocate_buffers(ALACContext *alac)
+{
+    int ch;
+    for (ch = 0; ch < alac->numchannels; ch++) {
+        int buf_size = alac->setinfo_max_samples_per_frame * sizeof(int32_t);
+
+        FF_ALLOC_OR_GOTO(alac->avctx, alac->predicterror_buffer[ch],
+                         buf_size, buf_alloc_fail);
+
+        FF_ALLOC_OR_GOTO(alac->avctx, alac->outputsamples_buffer[ch],
+                         buf_size, buf_alloc_fail);
+
+        FF_ALLOC_OR_GOTO(alac->avctx, alac->extra_bits_buffer[ch],
+                         buf_size, buf_alloc_fail);
+    }
+    return 0;
+buf_alloc_fail:
+    alac_decode_close(alac->avctx);
+    return AVERROR(ENOMEM);
+}
+
+static int alac_set_info(ALACContext *alac)
+{
+    const unsigned char *ptr = alac->avctx->extradata;
+
+    ptr += 4; /* size */
+    ptr += 4; /* alac */
+    ptr += 4; /* 0 ? */
+
+    if(AV_RB32(ptr) >= UINT_MAX/4){
+        av_log(alac->avctx, AV_LOG_ERROR, "setinfo_max_samples_per_frame too large\n");
+        return -1;
+    }
+
+    /* buffer size / 2 ? */
+    alac->setinfo_max_samples_per_frame = bytestream_get_be32(&ptr);
+    ptr++;                          /* ??? */
+    alac->setinfo_sample_size           = *ptr++;
+    alac->setinfo_rice_historymult      = *ptr++;
+    alac->setinfo_rice_initialhistory   = *ptr++;
+    alac->setinfo_rice_kmodifier        = *ptr++;
+    alac->numchannels                   = *ptr++;
+    bytestream_get_be16(&ptr);      /* ??? */
+    bytestream_get_be32(&ptr);      /* max coded frame size */
+    bytestream_get_be32(&ptr);      /* bitrate ? */
+    bytestream_get_be32(&ptr);      /* samplerate */
+
+    return 0;
+}
+
 static av_cold int alac_decode_init(AVCodecContext * avctx)
 {
+    int ret;
     ALACContext *alac = avctx->priv_data;
     alac->avctx = avctx;
-    alac->numchannels = alac->avctx->channels;
 
     /* initialize from the extradata */
     if (alac->avctx->extradata_size != ALAC_EXTRADATA_SIZE) {
@@ -661,18 +607,34 @@ static av_cold int alac_decode_init(AVCodecContext * avctx)
         return -1;
     }
 
-    return 0;
-}
+    switch (alac->setinfo_sample_size) {
+    case 16: avctx->sample_fmt    = AV_SAMPLE_FMT_S16;
+             break;
+    case 24: avctx->sample_fmt    = AV_SAMPLE_FMT_S32;
+             break;
+    default: av_log_ask_for_sample(avctx, "Sample depth %d is not supported.\n",
+                                   alac->setinfo_sample_size);
+             return AVERROR_PATCHWELCOME;
+    }
 
-static av_cold int alac_decode_close(AVCodecContext *avctx)
-{
-    ALACContext *alac = avctx->priv_data;
+    if (alac->numchannels < 1) {
+        av_log(avctx, AV_LOG_WARNING, "Invalid channel count\n");
+        alac->numchannels = avctx->channels;
+    } else {
+        if (alac->numchannels > MAX_CHANNELS)
+            alac->numchannels = avctx->channels;
+        else
+            avctx->channels = alac->numchannels;
+    }
+    if (avctx->channels > MAX_CHANNELS) {
+        av_log(avctx, AV_LOG_ERROR, "Unsupported channel count: %d\n",
+               avctx->channels);
+        return AVERROR_PATCHWELCOME;
+    }
 
-    int chan;
-    for (chan = 0; chan < MAX_CHANNELS; chan++) {
-        av_freep(&alac->predicterror_buffer[chan]);
-        av_freep(&alac->outputsamples_buffer[chan]);
-        av_freep(&alac->wasted_bits_buffer[chan]);
+    if ((ret = allocate_buffers(alac)) < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Error allocating buffers\n");
+        return ret;
     }
 
     return 0;