From cee3c9d29aceec8cddd829acd6dfb56dc5f60322 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 8 Nov 2015 14:13:42 +0100 Subject: avutil/softfloat: Fix overflows in shifts in av_cmp_sf() and av_gt_sf() Signed-off-by: Michael Niedermayer --- libavutil/softfloat.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'libavutil') diff --git a/libavutil/softfloat.h b/libavutil/softfloat.h index e5bfbbdfc6..ae11bdb39d 100644 --- a/libavutil/softfloat.h +++ b/libavutil/softfloat.h @@ -119,15 +119,19 @@ static inline av_const SoftFloat av_div_sf(SoftFloat a, SoftFloat b){ static inline av_const int av_cmp_sf(SoftFloat a, SoftFloat b){ int t= a.exp - b.exp; - if(t<0) return (a.mant >> (-t)) - b.mant ; - else return a.mant - (b.mant >> t); + if (t <-31) return - b.mant ; + else if (t < 0) return (a.mant >> (-t)) - b.mant ; + else if (t < 32) return a.mant - (b.mant >> t); + else return a.mant ; } static inline av_const int av_gt_sf(SoftFloat a, SoftFloat b) { int t= a.exp - b.exp; - if(t<0) return (a.mant >> (-t)) > b.mant ; - else return a.mant > (b.mant >> t); + if (t <-31) return 0; + else if (t < 0) return (a.mant >> (-t)) > b.mant ; + else if (t < 32) return a.mant > (b.mant >> t); + else return 1; } static inline av_const SoftFloat av_add_sf(SoftFloat a, SoftFloat b){ -- cgit v1.2.3