diff options
author | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-06 00:26:42 +0400 |
---|---|---|
committer | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-06 22:47:41 +0400 |
commit | 2254b559cbcfc0418135f09add37c0a5866b1981 (patch) | |
tree | 07783f2b1324c6fa17b8ea32c30d29c8fc017e06 /libswscale | |
parent | 018f39ef496c768f7cd580a96ae971e03c78205e (diff) |
swscale: make filterPos 32bit.
Fixes overflows for large image sizes.
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Diffstat (limited to 'libswscale')
-rw-r--r-- | libswscale/ppc/swscale_altivec.c | 2 | ||||
-rw-r--r-- | libswscale/swscale.c | 20 | ||||
-rw-r--r-- | libswscale/swscale_internal.h | 12 | ||||
-rw-r--r-- | libswscale/utils.c | 4 | ||||
-rw-r--r-- | libswscale/x86/scale.asm | 31 | ||||
-rw-r--r-- | libswscale/x86/swscale_mmx.c | 6 | ||||
-rw-r--r-- | libswscale/x86/swscale_template.c | 4 |
7 files changed, 41 insertions, 38 deletions
diff --git a/libswscale/ppc/swscale_altivec.c b/libswscale/ppc/swscale_altivec.c index 3041053096..5537707bd0 100644 --- a/libswscale/ppc/swscale_altivec.c +++ b/libswscale/ppc/swscale_altivec.c @@ -144,7 +144,7 @@ static void yuv2planeX_altivec(const int16_t *filter, int filterSize, static void hScale_altivec_real(SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { register int i; DECLARE_ALIGNED(16, int, tempo)[4]; diff --git a/libswscale/swscale.c b/libswscale/swscale.c index bd909bf163..a98a4bb90f 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -61,7 +61,7 @@ static av_always_inline void fillPlane(uint8_t* plane, int stride, static void hScale16To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t *_src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { int i; int32_t *dst = (int32_t *) _dst; @@ -84,7 +84,7 @@ static void hScale16To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t static void hScale16To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t *_src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { int i; const uint16_t *src = (const uint16_t *) _src; @@ -105,7 +105,7 @@ static void hScale16To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t // bilinear / bicubic scaling static void hScale8To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, - const int16_t *filter, const int16_t *filterPos, + const int16_t *filter, const int32_t *filterPos, int filterSize) { int i; @@ -123,7 +123,7 @@ static void hScale8To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t * } static void hScale8To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t *src, - const int16_t *filter, const int16_t *filterPos, + const int16_t *filter, const int32_t *filterPos, int filterSize) { int i; @@ -224,7 +224,7 @@ static void hyscale_fast_c(SwsContext *c, int16_t *dst, int dstWidth, static av_always_inline void hyscale(SwsContext *c, int16_t *dst, int dstWidth, const uint8_t *src_in[4], int srcW, int xInc, const int16_t *hLumFilter, - const int16_t *hLumFilterPos, int hLumFilterSize, + const int32_t *hLumFilterPos, int hLumFilterSize, uint8_t *formatConvBuffer, uint32_t *pal, int isAlpha) { @@ -268,7 +268,7 @@ static void hcscale_fast_c(SwsContext *c, int16_t *dst1, int16_t *dst2, static av_always_inline void hcscale(SwsContext *c, int16_t *dst1, int16_t *dst2, int dstWidth, const uint8_t *src_in[4], int srcW, int xInc, const int16_t *hChrFilter, - const int16_t *hChrFilterPos, int hChrFilterSize, + const int32_t *hChrFilterPos, int hChrFilterSize, uint8_t *formatConvBuffer, uint32_t *pal) { const uint8_t *src1 = src_in[1], *src2 = src_in[2]; @@ -312,10 +312,10 @@ static int swScale(SwsContext *c, const uint8_t* src[], const int chrXInc= c->chrXInc; const enum PixelFormat dstFormat= c->dstFormat; const int flags= c->flags; - int16_t *vLumFilterPos= c->vLumFilterPos; - int16_t *vChrFilterPos= c->vChrFilterPos; - int16_t *hLumFilterPos= c->hLumFilterPos; - int16_t *hChrFilterPos= c->hChrFilterPos; + int32_t *vLumFilterPos= c->vLumFilterPos; + int32_t *vChrFilterPos= c->vChrFilterPos; + int32_t *hLumFilterPos= c->hLumFilterPos; + int32_t *hChrFilterPos= c->hChrFilterPos; int16_t *vLumFilter= c->vLumFilter; int16_t *vChrFilter= c->vChrFilter; int16_t *hLumFilter= c->hLumFilter; diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index bc36826ea2..29fe4ff2c6 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -295,10 +295,10 @@ typedef struct SwsContext { int16_t *hChrFilter; ///< Array of horizontal filter coefficients for chroma planes. int16_t *vLumFilter; ///< Array of vertical filter coefficients for luma/alpha planes. int16_t *vChrFilter; ///< Array of vertical filter coefficients for chroma planes. - int16_t *hLumFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for luma/alpha planes. - int16_t *hChrFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for chroma planes. - int16_t *vLumFilterPos; ///< Array of vertical filter starting positions for each dst[i] for luma/alpha planes. - int16_t *vChrFilterPos; ///< Array of vertical filter starting positions for each dst[i] for chroma planes. + int32_t *hLumFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for luma/alpha planes. + int32_t *hChrFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for chroma planes. + int32_t *vLumFilterPos; ///< Array of vertical filter starting positions for each dst[i] for luma/alpha planes. + int32_t *vChrFilterPos; ///< Array of vertical filter starting positions for each dst[i] for chroma planes. int hLumFilterSize; ///< Horizontal filter size for luma/alpha pixels. int hChrFilterSize; ///< Horizontal filter size for chroma pixels. int vLumFilterSize; ///< Vertical filter size for luma/alpha pixels. @@ -508,10 +508,10 @@ typedef struct SwsContext { /** @{ */ void (*hyScale)(struct SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize); + const int32_t *filterPos, int filterSize); void (*hcScale)(struct SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize); + const int32_t *filterPos, int filterSize); /** @} */ /// Color range conversion function for luma plane if needed. diff --git a/libswscale/utils.c b/libswscale/utils.c index 7c89a70953..51dd331117 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -180,7 +180,7 @@ static double getSplineCoeff(double a, double b, double c, double d, double dist dist-1.0); } -static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSize, int xInc, +static int initFilter(int16_t **outFilter, int32_t **filterPos, int *outFilterSize, int xInc, int srcW, int dstW, int filterAlign, int one, int flags, int cpu_flags, SwsVector *srcFilter, SwsVector *dstFilter, double param[2], int is_horizontal) { @@ -196,7 +196,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi emms_c(); //FIXME this should not be required but it IS (even for non-MMX versions) // NOTE: the +3 is for the MMX(+1)/SSE(+3) scaler which reads over the end - FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(int16_t), fail); + FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(**filterPos), fail); if (FFABS(xInc - 0x10000) <10) { // unscaled int i; diff --git a/libswscale/x86/scale.asm b/libswscale/x86/scale.asm index 0d367f7a14..2387d0266b 100644 --- a/libswscale/x86/scale.asm +++ b/libswscale/x86/scale.asm @@ -38,7 +38,7 @@ SECTION .text ; (SwsContext *c, int{16,32}_t *dst, ; int dstW, const uint{8,16}_t *src, ; const int16_t *filter, -; const int16_t *filterPos, int filterSize); +; const int32_t *filterPos, int filterSize); ; ; Scale one horizontal line. Input is either 8-bits width or 16-bits width ; ($source_width can be either 8, 9, 10 or 16, difference is whether we have to @@ -53,6 +53,9 @@ SECTION .text cglobal hscale%1to%2_%4_%5, %6, 7, %7 %if ARCH_X86_64 movsxd r2, r2d +%define mov32 movsxd +%else ; x86-32 +%define mov32 mov %endif ; x86-64 %if %2 == 19 %if mmsize == 8 ; mmx @@ -95,14 +98,14 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %else ; %2 == 19 lea r1, [r1+r2*(4>>r2shr)] %endif ; %2 == 15/19 - lea r5, [r5+r2*(2>>r2shr)] + lea r5, [r5+r2*(4>>r2shr)] neg r2 .loop: %if %3 == 4 ; filterSize == 4 scaling ; load 2x4 or 4x4 source pixels into m0/m1 - movsx r0, word [r5+r2*2+0] ; filterPos[0] - movsx r6, word [r5+r2*2+2] ; filterPos[1] + mov32 r0, dword [r5+r2*4+0] ; filterPos[0] + mov32 r6, dword [r5+r2*4+4] ; filterPos[1] movlh m0, [r3+r0*srcmul] ; src[filterPos[0] + {0,1,2,3}] %if mmsize == 8 movlh m1, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}] @@ -112,8 +115,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %else ; %1 == 8 movd m4, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}] %endif - movsx r0, word [r5+r2*2+4] ; filterPos[2] - movsx r6, word [r5+r2*2+6] ; filterPos[3] + mov32 r0, dword [r5+r2*4+8] ; filterPos[2] + mov32 r6, dword [r5+r2*4+12] ; filterPos[3] movlh m1, [r3+r0*srcmul] ; src[filterPos[2] + {0,1,2,3}] %if %1 > 8 movhps m1, [r3+r6*srcmul] ; src[filterPos[3] + {0,1,2,3}] @@ -156,8 +159,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %endif ; mmx/sse2/ssse3/sse4 %else ; %3 == 8, i.e. filterSize == 8 scaling ; load 2x8 or 4x8 source pixels into m0, m1, m4 and m5 - movsx r0, word [r5+r2*1+0] ; filterPos[0] - movsx r6, word [r5+r2*1+2] ; filterPos[1] + mov32 r0, dword [r5+r2*2+0] ; filterPos[0] + mov32 r6, dword [r5+r2*2+4] ; filterPos[1] movbh m0, [r3+ r0 *srcmul] ; src[filterPos[0] + {0,1,2,3,4,5,6,7}] %if mmsize == 8 movbh m1, [r3+(r0+4)*srcmul] ; src[filterPos[0] + {4,5,6,7}] @@ -165,8 +168,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 movbh m5, [r3+(r6+4)*srcmul] ; src[filterPos[1] + {4,5,6,7}] %else ; mmsize == 16 movbh m1, [r3+ r6 *srcmul] ; src[filterPos[1] + {0,1,2,3,4,5,6,7}] - movsx r0, word [r5+r2*1+4] ; filterPos[2] - movsx r6, word [r5+r2*1+6] ; filterPos[3] + mov32 r0, dword [r5+r2*2+8] ; filterPos[2] + mov32 r6, dword [r5+r2*2+12] ; filterPos[3] movbh m4, [r3+ r0 *srcmul] ; src[filterPos[2] + {0,1,2,3,4,5,6,7}] movbh m5, [r3+ r6 *srcmul] ; src[filterPos[3] + {0,1,2,3,4,5,6,7}] %endif ; mmsize == 8/16 @@ -251,7 +254,7 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %define r1x r1 %define filter2 r6m %endif ; x86-32/64 - lea r5, [r5+r2*2] + lea r5, [r5+r2*4] %if %2 == 15 lea r1, [r1+r2*2] %else ; %2 == 19 @@ -261,8 +264,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 neg r2 .loop: - movsx r0, word [r5+r2*2+0] ; filterPos[0] - movsx r1x, word [r5+r2*2+2] ; filterPos[1] + mov32 r0, dword [r5+r2*4+0] ; filterPos[0] + mov32 r1x, dword [r5+r2*4+4] ; filterPos[1] ; FIXME maybe do 4px/iteration on x86-64 (x86-32 wouldn't have enough regs)? pxor m4, m4 pxor m5, m5 @@ -293,7 +296,7 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 jl .innerloop %ifidn %4, X4 - movsx r1x, word [r5+r2*2+2] ; filterPos[1] + mov32 r1x, dword [r5+r2*4+4] ; filterPos[1] movlh m0, [src_reg+r0 *srcmul] ; split last 4 srcpx of dstpx[0] sub r1x, r6 ; and first 4 srcpx of dstpx[1] %if %1 > 8 diff --git a/libswscale/x86/swscale_mmx.c b/libswscale/x86/swscale_mmx.c index 64d5f0fc9d..99b32623cb 100644 --- a/libswscale/x86/swscale_mmx.c +++ b/libswscale/x86/swscale_mmx.c @@ -93,8 +93,8 @@ void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufI int16_t **alpPixBuf= c->alpPixBuf; const int vLumBufSize= c->vLumBufSize; const int vChrBufSize= c->vChrBufSize; - int16_t *vLumFilterPos= c->vLumFilterPos; - int16_t *vChrFilterPos= c->vChrFilterPos; + int32_t *vLumFilterPos= c->vLumFilterPos; + int32_t *vChrFilterPos= c->vChrFilterPos; int16_t *vLumFilter= c->vLumFilter; int16_t *vChrFilter= c->vChrFilter; int32_t *lumMmxFilter= c->lumMmxFilter; @@ -204,7 +204,7 @@ extern void ff_hscale ## from_bpc ## to ## to_bpc ## _ ## filter_n ## _ ## opt( SwsContext *c, int16_t *data, \ int dstW, const uint8_t *src, \ const int16_t *filter, \ - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) #define SCALE_FUNCS(filter_n, opt) \ SCALE_FUNC(filter_n, 8, 15, opt); \ diff --git a/libswscale/x86/swscale_template.c b/libswscale/x86/swscale_template.c index 4db3fb3086..ad2b32f27b 100644 --- a/libswscale/x86/swscale_template.c +++ b/libswscale/x86/swscale_template.c @@ -1376,7 +1376,7 @@ static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst, int dstWidth, const uint8_t *src, int srcW, int xInc) { - int16_t *filterPos = c->hLumFilterPos; + int32_t *filterPos = c->hLumFilterPos; int16_t *filter = c->hLumFilter; void *mmx2FilterCode= c->lumMmx2FilterCode; int i; @@ -1472,7 +1472,7 @@ static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2, int dstWidth, const uint8_t *src1, const uint8_t *src2, int srcW, int xInc) { - int16_t *filterPos = c->hChrFilterPos; + int32_t *filterPos = c->hChrFilterPos; int16_t *filter = c->hChrFilter; void *mmx2FilterCode= c->chrMmx2FilterCode; int i; |