From c486cfab792e53de3f5fc9f5082909ce687a3fe8 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 10 Apr 2014 18:38:39 +0200 Subject: avcodec/imgconvert: fix nb_components and depth for PAL8 in get_pix_fmt_score() This causes us to favor RGB8 over PAL8 when FF_LOSS_COLORQUANT is used It probably makes sense to reinvestigate the exact scoring of pal8 when our pal8 support improves to be supperior to rgb8 Signed-off-by: Michael Niedermayer --- libavcodec/imgconvert.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'libavcodec/imgconvert.c') diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c index b830961fda..cabd3fd26b 100644 --- a/libavcodec/imgconvert.c +++ b/libavcodec/imgconvert.c @@ -130,13 +130,18 @@ static int get_pix_fmt_score(enum AVPixelFormat dst_pix_fmt, src_color = get_color_type(src_desc); dst_color = get_color_type(dst_desc); - nb_components = FFMIN(src_desc->nb_components, dst_desc->nb_components); - - for (i = 0; i < nb_components; i++) - if (src_desc->comp[i].depth_minus1 > dst_desc->comp[i].depth_minus1 && (consider & FF_LOSS_DEPTH)) { + if (dst_pix_fmt == AV_PIX_FMT_PAL8) + nb_components = FFMIN(src_desc->nb_components, 4); + else + nb_components = FFMIN(src_desc->nb_components, dst_desc->nb_components); + + for (i = 0; i < nb_components; i++) { + int depth_minus1 = (dst_pix_fmt == AV_PIX_FMT_PAL8) ? 7/nb_components : dst_desc->comp[i].depth_minus1; + if (src_desc->comp[i].depth_minus1 > depth_minus1 && (consider & FF_LOSS_DEPTH)) { loss |= FF_LOSS_DEPTH; - score -= 65536 >> dst_desc->comp[i].depth_minus1; + score -= 65536 >> depth_minus1; } + } if (consider & FF_LOSS_RESOLUTION) { if (dst_desc->log2_chroma_w > src_desc->log2_chroma_w) { -- cgit v1.2.3