From 30dff62b4fdacfefee61ff91f9b75985fee3dd9d Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 18 May 2022 02:10:52 +0200 Subject: avfilter/vf_signature: Fix integer overflow in filter_frame() Fixes: CID1403233 The second of the 2 changes may be unneeded but will help coverity Signed-off-by: Michael Niedermayer (cherry picked from commit dd6040675ec18d19429f882caea6bb306ed6677a) Signed-off-by: Michael Niedermayer --- libavfilter/vf_signature.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavfilter/vf_signature.c b/libavfilter/vf_signature.c index d07b213f31..8bdd7f55a2 100644 --- a/libavfilter/vf_signature.c +++ b/libavfilter/vf_signature.c @@ -223,7 +223,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) dw1 = inlink->w / 32; if (inlink->w % 32) dw2 = dw1 + 1; - denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1; + denom = (sc->divide) ? dh1 * (int64_t)dh2 * dw1 * dw2 : 1; for (i = 0; i < 32; i++) { rowcount = 0; @@ -249,7 +249,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) } } - denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2; + denom = (sc->divide) ? 1 : dh1 * (int64_t)dh2 * dw1 * dw2; for (i = 0; i < ELEMENT_COUNT; i++) { const ElemCat* elemcat = elements[i]; -- cgit v1.2.3