diff options
author | Stefano Sabatini <stefano.sabatini-lala@poste.it> | 2011-05-13 17:42:31 +0400 |
---|---|---|
committer | Stefano Sabatini <stefano.sabatini-lala@poste.it> | 2011-05-15 14:27:07 +0400 |
commit | 10931720cd55d83e0b933b8a9bb0795fd9e48875 (patch) | |
tree | 9a4287662b23bc6f1b8baf850b9ee92f7dfec02a | |
parent | 5a153604c930792aa7f00c55cbf3c470f582dfb7 (diff) |
imgutils: generalize linesize computation for bitstream formats
Make it a subcase of the general algorithm used for the non-bitstream
case. Simplify, and make av_image_get_linesize() and
av_image_fill_linesizes() correctly return the right value when plane
!= 0.
In particular fix a crash occurring with:
-vf format=monow,showinfo,format=monow.
-rw-r--r-- | libavutil/imgutils.c | 19 |
1 files changed, 7 insertions, 12 deletions
diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c index 8eefa4d494..0df8de4255 100644 --- a/libavutil/imgutils.c +++ b/libavutil/imgutils.c @@ -48,14 +48,14 @@ int av_image_get_linesize(enum PixelFormat pix_fmt, int width, int plane) const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt]; int max_step [4]; /* max pixel step for each plane */ int max_step_comp[4]; /* the component for each plane which has the max pixel step */ - int s; - - if (desc->flags & PIX_FMT_BITSTREAM) - return (width * (desc->comp[0].step_minus1+1) + 7) >> 3; + int s, linesize; av_image_fill_max_pixsteps(max_step, max_step_comp, desc); s = (max_step_comp[plane] == 1 || max_step_comp[plane] == 2) ? desc->log2_chroma_w : 0; - return max_step[plane] * (((width + (1 << s) - 1)) >> s); + linesize = max_step[plane] * (((width + (1 << s) - 1)) >> s); + if (desc->flags & PIX_FMT_BITSTREAM) + linesize = (linesize + 7) >> 3; + return linesize; } int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int width) @@ -70,13 +70,6 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt if ((unsigned)pix_fmt >= PIX_FMT_NB || desc->flags & PIX_FMT_HWACCEL) return AVERROR(EINVAL); - if (desc->flags & PIX_FMT_BITSTREAM) { - if (width > (INT_MAX -7) / (desc->comp[0].step_minus1+1)) - return AVERROR(EINVAL); - linesizes[0] = (width * (desc->comp[0].step_minus1+1) + 7) >> 3; - return 0; - } - av_image_fill_max_pixsteps(max_step, max_step_comp, desc); for (i = 0; i < 4; i++) { int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0; @@ -84,6 +77,8 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt if (max_step[i] > INT_MAX / shifted_w) return AVERROR(EINVAL); linesizes[i] = max_step[i] * shifted_w; + if (desc->flags & PIX_FMT_BITSTREAM) + linesizes[i] = (linesizes[i] + 7) >> 3; } return 0; |