From 1ac5abb1d062b6ca983d494068bb9fd30390a941 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 20 Dec 2014 20:39:13 +0100 Subject: avfilter/vf_spp: fix overflows with depth > 8 Signed-off-by: Michael Niedermayer --- libavfilter/vf_spp.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'libavfilter/vf_spp.c') diff --git a/libavfilter/vf_spp.c b/libavfilter/vf_spp.c index 67dc80000a..4c7158ed62 100644 --- a/libavfilter/vf_spp.c +++ b/libavfilter/vf_spp.c @@ -186,16 +186,17 @@ static void store_slice_c(uint8_t *dst, const uint16_t *src, } } -static void store_slice16_c(uint16_t *dst, const uint16_t *src, +static void store_slice16_c(uint16_t *dst, const int16_t *src, int dst_linesize, int src_linesize, int width, int height, int log2_scale, - const uint8_t dither[8][8]) + const uint8_t dither[8][8], int depth) { int y, x; + unsigned int mask = -1<>1)) >> 5; \ - if (temp & 0x400) \ + if (temp & mask ) \ temp = ~(temp >> 31); \ dst[x + y*dst_linesize + pos] = temp; \ } while (0) @@ -242,7 +243,7 @@ static inline int norm_qscale(int qscale, int type) static void filter(SPPContext *p, uint8_t *dst, uint8_t *src, int dst_linesize, int src_linesize, int width, int height, - const uint8_t *qp_table, int qp_stride, int is_luma, int sample_bytes) + const uint8_t *qp_table, int qp_stride, int is_luma, int depth) { int x, y, i; const int count = 1 << p->log2_count; @@ -251,6 +252,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src, int16_t *block = (int16_t *)block_align; int16_t *block2 = (int16_t *)(block_align + 16); uint16_t *psrc16 = (uint16_t*)p->src; + const int sample_bytes = (depth+7) / 8; for (y = 0; y < height; y++) { int index = 8 + 8*linesize + y*linesize; @@ -305,7 +307,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src, store_slice16_c((uint16_t*)(dst + (y - 8) * dst_linesize), p->temp + 8 + y*linesize, dst_linesize/2, linesize, width, FFMIN(8, height + 8 - y), MAX_LEVEL - p->log2_count, - ldither); + ldither, depth); } } } @@ -366,7 +368,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) int qp_stride = 0; const int8_t *qp_table = NULL; const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); - const int sample_bytes = desc->comp[0].depth_minus1 < 8 ? 1 : 2; + const int depth = desc->comp[0].depth_minus1 + 1; /* if we are not in a constant user quantizer mode and we don't want to use * the quantizers from the B-frames (B-frames often have a higher QP), we @@ -426,11 +428,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) out->height = in->height; } - filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, sample_bytes); + filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, depth); if (out->data[2]) { - filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, sample_bytes); - filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, sample_bytes); + filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, depth); + filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, depth); } emms_c(); } -- cgit v1.2.3