resample: recover from integer overflow in update_filter()

Signed-off-by: Tristan Matthews <le.businessman@gmail.com>

1 files changed, 16 insertions, 5 deletions

diff --git a/libspeexdsp/resample.c b/libspeexdsp/resample.c
index b0eb7c5..edbd65b 100644
--- a/libspeexdsp/resample.c
+++ b/libspeexdsp/resample.c
@@ -77,6 +77,7 @@ static void speex_free (void *ptr) {free(ptr);}
 
 #include "stack_alloc.h"
 #include <math.h>
+#include <limits.h>
 
 #ifndef M_PI
 #define M_PI 3.14159265358979323846
@@ -629,13 +630,19 @@ static int update_filter(SpeexResamplerState *st)
    /* Choose the resampling type that requires the least amount of memory */
 #ifdef RESAMPLE_FULL_SINC_TABLE
    use_direct = 1;
+   if (INT_MAX/sizeof(spx_word16_t)/st->den_rate < st->filt_len)
+      goto fail;
 #else
-   use_direct = st->filt_len*st->den_rate <= st->filt_len*st->oversample+8;
+   use_direct = st->filt_len*st->den_rate <= st->filt_len*st->oversample+8
+                && INT_MAX/sizeof(spx_word16_t)/st->den_rate >= st->filt_len;
 #endif
    if (use_direct)
    {
       min_sinc_table_length = st->filt_len*st->den_rate;
    } else {
+      if ((INT_MAX/sizeof(spx_word16_t)-8)/st->oversample < st->filt_len)
+         goto fail;
+
       min_sinc_table_length = st->filt_len*st->oversample+8;
    }
    if (st->sinc_table_length < min_sinc_table_length)
@@ -682,16 +689,20 @@ static int update_filter(SpeexResamplerState *st)
       /*fprintf (stderr, "resampler uses interpolated sinc table and normalised cutoff %f\n", cutoff);*/
    }
 
-   
    /* Here's the place where we update the filter memory to take into account
       the change in filter length. It's probably the messiest part of the code
       due to handling of lots of corner cases. */
+
+   /* Adding buffer_size to filt_len won't overflow here because filt_len
+      could be multiplied by sizeof(spx_word16_t) above. */
    min_alloc_size = st->filt_len-1 + st->buffer_size;
    if (min_alloc_size > st->mem_alloc_size)
    {
-      spx_word16_t *mem = (spx_word16_t*)speex_realloc(st->mem, st->nb_channels*min_alloc_size * sizeof(spx_word16_t));
-      if (!mem)
-         goto fail;
+      spx_word16_t *mem;
+      if (INT_MAX/sizeof(spx_word16_t)/st->nb_channels < min_alloc_size)
+          goto fail;
+      else if (!(mem = (spx_word16_t*)speex_realloc(st->mem, st->nb_channels*min_alloc_size * sizeof(*mem))))
+          goto fail;
 
       st->mem = mem;
       st->mem_alloc_size = min_alloc_size;