diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2016-01-14 01:51:47 +0300 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2016-01-17 19:28:20 +0300 |
| commit | 5fbd97fc756a827f62f556c66272f851cc3c7f90 (patch) | |
| tree | 76ea5fc9f2a2f1e863fad8777395aeea28df1ed1 /libavcodec/diracdec.c | |
| parent | 69697be92200ff2222f8c59158f35b1771f0c002 (diff) | |
avcodec/diracdec: Fix qfactor/offset tables
It seems the previous tables where calculated with 32bit integers ignoring
overflows.
Also check for the max qindex, the value is choosen so that the qfactor/offset
fit in int32.
Fixes: 070b7914fd5dfe8f93248bea71363410/asan_static-oob_c8d034_2764_258e20f4a3c79158aecddb61a833d756.drc
Fixes out of array reads
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec/diracdec.c')
| -rw-r--r-- | libavcodec/diracdec.c | 108 |
1 files changed, 63 insertions, 45 deletions
diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c index 942376bb3c..ee07ff8bb3 100644 --- a/libavcodec/diracdec.c +++ b/libavcodec/diracdec.c @@ -253,53 +253,61 @@ static const uint8_t default_qmat[][4][4] = { { { 3, 1, 1, 0}, { 0, 4, 4, 2}, { 0, 6, 6, 5}, { 0, 9, 9, 7} }, }; -static const int32_t qscale_tab[128] = { - 4, 5, 6, 7, 8, 10, 11, 13, - 16, 19, 23, 27, 32, 38, 45, 54, - 64, 76, 91, 108, 128, 152, 181, 215, - 256, 304, 362, 431, 512, 609, 724, 861, - 1024, 1218, 1448, 1722, 2048, 2435, 2896, 3444, - 4096, 4871, 5793, 6889, 8192, 9742, 11585, 13777, - 16384, 19484, -13317, 27554, 32768, -1581, 9853, -10518, - 65536, -3164, -16782, -21037, 131072, -6328, 2922, 23552, - 262144, -12658, 5844, -18524, 524288, 15232, 11689, 28578, - 1048576, -10085, -13110, -8471, 2097152, -20170, 10267, -16943, - 4194304, 208, -15954, 31741, 8388608, 416, 4579, -2146, - 16777216, 832, 9158, -4293, 33554432, 1663, -18172, -8587, - 67108864, 3326, 143, -17175, 134217728, 6653, 285, 31276, -268435456, 13306, 570, -3075, 536870912, -13938, 1140, -6152, -1073741824, 12672, 2281, -12304, -2147483648, -15205, 4561, -24610, - 0, 10138, 9122, 16407, 0, -20274, -18243, -32813, +static const int32_t qscale_tab[116] = { + 4, 5, 6, 7, 8, 10, 11, 13, + 16, 19, 23, 27, 32, 38, 45, 54, + 64, 76, 91, 108, 128, 152, 181, 215, + 256, 304, 362, 431, 512, 609, 724, 861, + 1024, 1218, 1448, 1722, 2048, 2435, 2896, 3444, + 4096, 4871, 5793, 6889, 8192, 9742, 11585, 13777, + 16384, 19484, 23170, 27554, 32768, 38968, 46341, 55109, + 65536, 77936, 92682, 110218, 131072, 155872, 185364, 220436, + 262144, 311744, 370728, 440872, 524288, 623487, 741455, 881744, + 1048576, 1246974, 1482910, 1763488, 2097152, 2493948, 2965821, 3526975, + 4194304, 4987896, 5931642, 7053950, 8388608, 9975792, 11863283, 14107901, + 16777216, 19951585, 23726566, 28215802, 33554432, 39903169, 47453133, 56431603, + 67108864, 79806339, 94906266, 112863206, 134217728, 159612677, 189812531, 225726413, + 268435456, 319225354, 379625062, 451452825, 536870912, 638450708, 759250125, 902905651, +1073741824,1276901417,1518500250,1805811301,/*2147483648,2553802834,3037000500,3611622603, +4294967296*/ }; -static const int32_t qoffset_intra_tab[128] = { +static const int32_t qoffset_intra_tab[120] = { 1, 2, 3, 4, 4, 5, 6, 7, - 8, 10, 12, 14, 16, 19, 23, 27, - 32, 38, 46, 54, 64, 76, 91, 108, - 128, 152, 181, 216, 256, 305, 362, 431, - 512, 609, 724, 861, 1024, 1218, 1448, 1722, - 2048, 2436, 2897, 3445, 4096, 4871, 5793, 6889, - 8192, 9742, -6658, 13777, 16384, -790, 4927, -5258, - 32768, -1581, -8390, -10518, 65536, -3163, 1461, 11776, - 131072, -6328, 2922, -9261, 262144, 7616, 5845, 14289, - 524288, -5042, -6554, -4235, 1048576, -10084, 5134, -8471, - 2097152, 104, -7976, 15871, 4194304, 208, 2290, -1072, - 8388608, 416, 4579, -2146, 16777216, 832, -9085, -4293, - 33554432, 1663, 72, -8587, 67108864, 3327, 143, 15638, -134217728, 6653, 285, -1537, 268435456, -6968, 570, -3075, -536870912, 6336, 1141, -6151, -1073741823, -7602, 2281, -12304, - 0, 5069, 4561, 8204, 0, -10136, -9121, -16406, + 8, 10, 12, 14, 16, 19, 23, 27, + 32, 38, 46, 54, 64, 76, 91, 108, + 128, 152, 181, 216, 256, 305, 362, 431, + 512, 609, 724, 861, 1024, 1218, 1448, 1722, + 2048, 2436, 2897, 3445, 4096, 4871, 5793, 6889, + 8192, 9742, 11585, 13777, 16384, 19484, 23171, 27555, + 32768, 38968, 46341, 55109, 65536, 77936, 92682, 110218, + 131072, 155872, 185364, 220436, 262144, 311744, 370728, 440872, + 524288, 623487, 741455, 881744, 1048576, 1246974, 1482911, 1763488, + 2097152, 2493948, 2965821, 3526975, 4194304, 4987896, 5931642, 7053951, + 8388608, 9975793, 11863283, 14107901, 16777216, 19951585, 23726567, 28215802, + 33554432, 39903170, 47453133, 56431603, 67108864, 79806339, 94906266, 112863207, +134217728, 159612677, 189812531, 225726413, 268435456, 319225354, 379625063, 451452826, +536870912, 638450709, 759250125, 902905651,1073741824,1276901417,1518500250,1805811302, +/*2147483648, 2553802834, 3037000500, 3611622603, 4294967296,*/ }; -static const int qoffset_inter_tab[MAX_QUANT+1] = { - 1, 2, 2, 3, 3, 4, 4, 5, - 6, 7, 9, 10, 12, 14, 17, 20, - 24, 29, 34, 41, 48, 57, 68, 81, - 96, 114, 136, 162, 192, 228, 272, 323, - 384, 457, 543, 646, 768, 913, 1086, 1292, - 1536, 1827, 2172, 2583, 3072, 3653, 4344, 5166, - 6144, 7307, 8689, 10333, 12288, 14613, 17378, 20666, - 24576, 29226 +static const int qoffset_inter_tab[122] = { + 1, 2, 2, 3, 3, 4, 4, 5, + 6, 7, 9, 10, 12, 14, 17, 20, + 24, 29, 34, 41, 48, 57, 68, 81, + 96, 114, 136, 162, 192, 228, 272, 323, + 384, 457, 543, 646, 768, 913, 1086, 1292, + 1536, 1827, 2172, 2583, 3072, 3653, 4344, 5166, + 6144, 7307, 8689, 10333, 12288, 14613, 17378, 20666, + 24576, 29226, 34756, 41332, 49152, 58452, 69512, 82664, + 98304, 116904, 139023, 165327, 196608, 233808, 278046, 330654, + 393216, 467615, 556091, 661308, 786432, 935231, 1112183, 1322616, + 1572864, 1870461, 2224366, 2645231, 3145728, 3740922, 4448731, 5290463, + 6291456, 7481844, 8897462, 10580926, 12582912, 14963688, 17794925, 21161851, + 25165824, 29927377, 35589850, 42323702, 50331648, 59854754, 71179699, 84647405, +100663296, 119709508, 142359398, 169294809, 201326592, 239419016, 284718797, 338589619, +402653184, 478838031, 569437594, 677179238, 805306368, 957676063,1138875188,1354358476, +1610612736, 1915352125, /*2277750375, 2708716952, 3221225472, 3830704250,*/ }; /* magic number division by 3 from schroedinger */ @@ -601,7 +609,11 @@ static inline void codeblock(DiracContext *s, SubBand *b, b->quant = quant; } - b->quant = FFMIN(b->quant, MAX_QUANT); + if (b->quant > 115) { + av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", b->quant); + b->quant = 0; + return; + } qfactor = qscale_tab[b->quant]; /* TODO: context pointer? */ @@ -781,12 +793,18 @@ static void decode_subband(DiracContext *s, GetBitContext *gb, int quant, int top = b1->height * slice_y / s->num_y; int bottom = b1->height *(slice_y+1) / s->num_y; - int qfactor = qscale_tab[quant & 0x7f]; - int qoffset = qoffset_intra_tab[quant & 0x7f] + 2; + int qfactor, qoffset; uint8_t *buf1 = b1->ibuf + top * b1->stride; uint8_t *buf2 = b2 ? b2->ibuf + top * b2->stride: NULL; int x, y; + + if (quant > 115) { + av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", quant); + return; + } + qfactor = qscale_tab[quant & 0x7f]; + qoffset = qoffset_intra_tab[quant & 0x7f] + 2; /* we have to constantly check for overread since the spec explicitly requires this, with the meaning that all remaining coeffs are set to 0 */ if (get_bits_count(gb) >= bits_end) |