diff options
| author | Vitor Sessak <vitor1001@gmail.com> | 2011-02-12 12:15:58 +0300 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2011-02-13 02:54:06 +0300 |
| commit | 0a18b64585ca3f9181229dcc4ccb62bc6b939679 (patch) | |
| tree | b83a6490320bfbdd132d17610cc3caed98bc0bdf /libavcodec | |
| parent | c906d974b7c3288bc25a1db87acd7129bcc5cd04 (diff) | |
Make tables generation insensitive to floating-point implementation
Using doubles make the double -> int cast well defined for all the values
used, with the exception of when s[i]==1.0, which is special-cased.
Signed-off-by: Mans Rullgard <mans@mansr.com>
(cherry picked from commit 47d62c965b7bcf4715127fbf9bffab7b4c5c88b7)
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/bink.c | 25 |
1 files changed, 16 insertions, 9 deletions
diff --git a/libavcodec/bink.c b/libavcodec/bink.c index a6cfb71f0c..928bf53b3b 100644 --- a/libavcodec/bink.c +++ b/libavcodec/bink.c @@ -1217,30 +1217,37 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac */ static av_cold void binkb_calc_quant() { - float s[64]; + double s[64]; int i, j; for (j = 0; j < 8; j++) { for (i = 0; i < 8; i++) { if (j && j != 4) if (i && i != 4) - s[j*8 + i] = cos(j * M_PI/16.0f) * cos(i * M_PI/16.0f) * 2.0f; + s[j*8 + i] = cos(j * M_PI/16.0) * cos(i * M_PI/16.0) * 2.0; else - s[j*8 + i] = cos(j * M_PI/16.0f) * sqrt(2.0f); + s[j*8 + i] = cos(j * M_PI/16.0) * sqrt(2.0); else if (i && i != 4) - s[j*8 + i] = cos(i * M_PI/16.0f) * sqrt(2.0f); + s[j*8 + i] = cos(i * M_PI/16.0) * sqrt(2.0); else - s[j*8 + i] = 1.0f; + s[j*8 + i] = 1.0; } } for (j = 0; j < 16; j++) { for (i = 0; i < 64; i++) { - binkb_intra_quant[j][i] = (1L << 12) * binkb_intra_seed[i] * s[i] * - binkb_num[j]/(float)binkb_den[j]; - binkb_inter_quant[j][i] = (1L << 12) * binkb_inter_seed[i] * s[i] * - binkb_num[j]/(float)binkb_den[j]; + if (s[i] == 1.0) { + binkb_intra_quant[j][i] = (1L << 12) * binkb_intra_seed[i] * + binkb_num[j]/binkb_den[j]; + binkb_inter_quant[j][i] = (1L << 12) * binkb_inter_seed[i] * + binkb_num[j]/binkb_den[j]; + } else { + binkb_intra_quant[j][i] = (1L << 12) * binkb_intra_seed[i] * s[i] * + binkb_num[j]/(double)binkb_den[j]; + binkb_inter_quant[j][i] = (1L << 12) * binkb_inter_seed[i] * s[i] * + binkb_num[j]/(double)binkb_den[j]; + } } } } |