From e38f280fece38e270a6462a02cc034f4116a7912 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 8 Oct 2017 21:41:54 +0200 Subject: avcodec/mpeg4videodec: Use 64 bit intermediates for sprite delta Fixes: runtime error: signed integer overflow: -104713 * 65536 cannot be represented in type 'int' Fixes: 3453/clusterfuzz-testcase-minimized-5555554657239040 Fixes: 3528/clusterfuzz-testcase-minimized-6283628420005888 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/mpeg4videodec.c | 79 +++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c index 8f85e9362d..cd39131d55 100644 --- a/libavcodec/mpeg4videodec.c +++ b/libavcodec/mpeg4videodec.c @@ -179,6 +179,7 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g int sprite_ref[4][2]; int virtual_ref[2][2]; int64_t sprite_offset[2][2]; + int64_t sprite_delta[2][2]; // only true for rectangle shapes const int vop_ref[4][2] = { { 0, 0 }, { s->width, 0 }, @@ -262,10 +263,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g sprite_offset[0][1] = sprite_offset[1][0] = sprite_offset[1][1] = 0; - s->sprite_delta[0][0] = a; - s->sprite_delta[0][1] = - s->sprite_delta[1][0] = 0; - s->sprite_delta[1][1] = a; + sprite_delta[0][0] = a; + sprite_delta[0][1] = + sprite_delta[1][0] = 0; + sprite_delta[1][1] = a; ctx->sprite_shift[0] = ctx->sprite_shift[1] = 0; break; @@ -276,10 +277,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g a * (vop_ref[0][0] / 2); sprite_offset[1][1] = ((sprite_ref[0][1] >> 1) | (sprite_ref[0][1] & 1)) - a * (vop_ref[0][1] / 2); - s->sprite_delta[0][0] = a; - s->sprite_delta[0][1] = - s->sprite_delta[1][0] = 0; - s->sprite_delta[1][1] = a; + sprite_delta[0][0] = a; + sprite_delta[0][1] = + sprite_delta[1][0] = 0; + sprite_delta[1][1] = a; ctx->sprite_shift[0] = ctx->sprite_shift[1] = 0; break; @@ -304,10 +305,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g ((int64_t)-r * sprite_ref[0][0] + virtual_ref[0][0]) * ((int64_t)-2 * vop_ref[0][1] + 1) + 2 * w2 * r * (int64_t) sprite_ref[0][1] - 16 * w2 + (1 << (alpha + rho + 1))); - s->sprite_delta[0][0] = (-r * sprite_ref[0][0] + virtual_ref[0][0]); - s->sprite_delta[0][1] = (+r * sprite_ref[0][1] - virtual_ref[0][1]); - s->sprite_delta[1][0] = (-r * sprite_ref[0][1] + virtual_ref[0][1]); - s->sprite_delta[1][1] = (-r * sprite_ref[0][0] + virtual_ref[0][0]); + sprite_delta[0][0] = (-r * sprite_ref[0][0] + virtual_ref[0][0]); + sprite_delta[0][1] = (+r * sprite_ref[0][1] - virtual_ref[0][1]); + sprite_delta[1][0] = (-r * sprite_ref[0][1] + virtual_ref[0][1]); + sprite_delta[1][1] = (-r * sprite_ref[0][0] + virtual_ref[0][0]); ctx->sprite_shift[0] = alpha + rho; ctx->sprite_shift[1] = alpha + rho + 2; @@ -332,28 +333,28 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g ((int64_t)-r * sprite_ref[0][1] + virtual_ref[1][1]) * w3 * (-2 * vop_ref[0][1] + 1) + (int64_t)2 * w2 * h3 * r * sprite_ref[0][1] - 16 * w2 * h3 + ((int64_t)1 << (alpha + beta + rho - min_ab + 1)); - s->sprite_delta[0][0] = (-r * sprite_ref[0][0] + virtual_ref[0][0]) * h3; - s->sprite_delta[0][1] = (-r * sprite_ref[0][0] + virtual_ref[1][0]) * w3; - s->sprite_delta[1][0] = (-r * sprite_ref[0][1] + virtual_ref[0][1]) * h3; - s->sprite_delta[1][1] = (-r * sprite_ref[0][1] + virtual_ref[1][1]) * w3; + sprite_delta[0][0] = (-r * (int64_t)sprite_ref[0][0] + virtual_ref[0][0]) * h3; + sprite_delta[0][1] = (-r * (int64_t)sprite_ref[0][0] + virtual_ref[1][0]) * w3; + sprite_delta[1][0] = (-r * (int64_t)sprite_ref[0][1] + virtual_ref[0][1]) * h3; + sprite_delta[1][1] = (-r * (int64_t)sprite_ref[0][1] + virtual_ref[1][1]) * w3; ctx->sprite_shift[0] = alpha + beta + rho - min_ab; ctx->sprite_shift[1] = alpha + beta + rho - min_ab + 2; break; } /* try to simplify the situation */ - if (s->sprite_delta[0][0] == a << ctx->sprite_shift[0] && - s->sprite_delta[0][1] == 0 && - s->sprite_delta[1][0] == 0 && - s->sprite_delta[1][1] == a << ctx->sprite_shift[0]) { + if (sprite_delta[0][0] == a << ctx->sprite_shift[0] && + sprite_delta[0][1] == 0 && + sprite_delta[1][0] == 0 && + sprite_delta[1][1] == a << ctx->sprite_shift[0]) { sprite_offset[0][0] >>= ctx->sprite_shift[0]; sprite_offset[0][1] >>= ctx->sprite_shift[0]; sprite_offset[1][0] >>= ctx->sprite_shift[1]; sprite_offset[1][1] >>= ctx->sprite_shift[1]; - s->sprite_delta[0][0] = a; - s->sprite_delta[0][1] = 0; - s->sprite_delta[1][0] = 0; - s->sprite_delta[1][1] = a; + sprite_delta[0][0] = a; + sprite_delta[0][1] = 0; + sprite_delta[1][0] = 0; + sprite_delta[1][1] = a; ctx->sprite_shift[0] = 0; ctx->sprite_shift[1] = 0; s->real_sprite_warping_points = 1; @@ -365,8 +366,8 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g if (shift_c < 0 || shift_y < 0 || FFABS( sprite_offset[0][i]) >= INT_MAX >> shift_y || FFABS( sprite_offset[1][i]) >= INT_MAX >> shift_c || - FFABS(s->sprite_delta[0][i]) >= INT_MAX >> shift_y || - FFABS(s->sprite_delta[1][i]) >= INT_MAX >> shift_y + FFABS( sprite_delta[0][i]) >= INT_MAX >> shift_y || + FFABS( sprite_delta[1][i]) >= INT_MAX >> shift_y ) { avpriv_request_sample(s->avctx, "Too large sprite shift, delta or offset"); goto overflow; @@ -376,22 +377,22 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g for (i = 0; i < 2; i++) { sprite_offset[0][i] *= 1 << shift_y; sprite_offset[1][i] *= 1 << shift_c; - s->sprite_delta[0][i] *= 1 << shift_y; - s->sprite_delta[1][i] *= 1 << shift_y; + sprite_delta[0][i] *= 1 << shift_y; + sprite_delta[1][i] *= 1 << shift_y; ctx->sprite_shift[i] = 16; } for (i = 0; i < 2; i++) { int64_t sd[2] = { - s->sprite_delta[i][0] - a * (1LL<<16), - s->sprite_delta[i][1] - a * (1LL<<16) + sprite_delta[i][0] - a * (1LL<<16), + sprite_delta[i][1] - a * (1LL<<16) }; - if (llabs(sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL)) >= INT_MAX || - llabs(sprite_offset[0][i] + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || - llabs(sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL) + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || - llabs(s->sprite_delta[i][0] * (w+16LL)) >= INT_MAX || - llabs(s->sprite_delta[i][1] * (w+16LL)) >= INT_MAX || + if (llabs(sprite_offset[0][i] + sprite_delta[i][0] * (w+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + sprite_delta[i][1] * (h+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + sprite_delta[i][0] * (w+16LL) + sprite_delta[i][1] * (h+16LL)) >= INT_MAX || + llabs(sprite_delta[i][0] * (w+16LL)) >= INT_MAX || + llabs(sprite_delta[i][1] * (w+16LL)) >= INT_MAX || llabs(sd[0]) >= INT_MAX || llabs(sd[1]) >= INT_MAX || llabs(sprite_offset[0][i] + sd[0] * (w+16LL)) >= INT_MAX || @@ -405,10 +406,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g s->real_sprite_warping_points = ctx->num_sprite_warping_points; } - s->sprite_offset[0][0] = sprite_offset[0][0]; - s->sprite_offset[0][1] = sprite_offset[0][1]; - s->sprite_offset[1][0] = sprite_offset[1][0]; - s->sprite_offset[1][1] = sprite_offset[1][1]; + for (i = 0; i < 4; i++) { + s->sprite_offset[i&1][i>>1] = sprite_offset[i&1][i>>1]; + s->sprite_delta [i&1][i>>1] = sprite_delta [i&1][i>>1]; + } return 0; overflow: -- cgit v1.2.3