diff options
author | Michael Niedermayer <michael@niedermayer.cc> | 2019-06-16 12:39:15 +0300 |
---|---|---|
committer | Michael Niedermayer <michael@niedermayer.cc> | 2019-08-04 22:16:28 +0300 |
commit | a33fd082661faa0a6eda7a4ede722fdb865ddc8b (patch) | |
tree | 95a71e470963ba9433609ac9167b4eca1dc96267 /libavcodec/apedec.c | |
parent | 5fa0b18c95a0324a933101a727106e7cbf6597b6 (diff) |
avcodec/apedec: Fix multiple integer overflows and undefined behaviorin filter_3800()
Fixes: left shift of negative value -4
Fixes: signed integer overflow: -15091694 * 167 cannot be represented in type 'int'
Fixes: signed integer overflow: 1898547155 + 453967445 cannot be represented in type 'int'
Fixes: 15258/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5759095564402688
Fixes: signed integer overflow: 962196438 * 31 cannot be represented in type 'int'
Fixes: 15364/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5718799845687296
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 267eb2ab7f87696e1a156ca9a5ff1b1628d170c1)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec/apedec.c')
-rw-r--r-- | libavcodec/apedec.c | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c index 7cf99a00b1..27425b17e6 100644 --- a/libavcodec/apedec.c +++ b/libavcodec/apedec.c @@ -859,9 +859,9 @@ static av_always_inline int filter_3800(APEPredictor *p, return predictionA; } d2 = p->buf[delayA]; - d1 = (p->buf[delayA] - p->buf[delayA - 1]) << 1; - d0 = p->buf[delayA] + ((p->buf[delayA - 2] - p->buf[delayA - 1]) << 3); - d3 = p->buf[delayB] * 2 - p->buf[delayB - 1]; + d1 = (p->buf[delayA] - p->buf[delayA - 1]) * 2U; + d0 = p->buf[delayA] + ((p->buf[delayA - 2] - p->buf[delayA - 1]) * 8U); + d3 = p->buf[delayB] * 2U - p->buf[delayB - 1]; d4 = p->buf[delayB]; predictionA = d0 * p->coeffsA[filter][0] + @@ -881,7 +881,7 @@ static av_always_inline int filter_3800(APEPredictor *p, p->coeffsB[filter][1] -= (((d4 >> 30) & 2) - 1) * sign; p->filterB[filter] = p->lastA[filter] + (predictionB >> shift); - p->filterA[filter] = p->filterB[filter] + ((p->filterA[filter] * 31) >> 5); + p->filterA[filter] = p->filterB[filter] + ((int)(p->filterA[filter] * 31U) >> 5); return p->filterA[filter]; } @@ -902,7 +902,7 @@ static void long_filter_high_3800(int32_t *buffer, int order, int shift, int len dotprod = 0; sign = APESIGN(buffer[i]); for (j = 0; j < order; j++) { - dotprod += delay[j] * coeffs[j]; + dotprod += delay[j] * (unsigned)coeffs[j]; coeffs[j] += ((delay[j] >> 31) | 1) * sign; } buffer[i] -= dotprod >> shift; |