1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
#pragma once
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <limits>
#include "FbgemmBuild.h"
#include "QuantUtilsAvx2.h"
#include "Utils.h"
namespace fbgemm {
FBGEMM_API TensorQuantizationParams ChooseQuantizationParams(
float min,
float max,
std::int32_t qmin,
std::int32_t qmax,
bool preserve_sparsity = false,
bool force_scale_power_of_two = false);
FBGEMM_API void ChooseRequantizationMultiplier(
float real_multiplier,
std::int32_t* quantized_multiplier,
int* right_shift,
int requantization_multiplier_precision = 32);
////////////////////////////////////////////////////////////////////////////////
// Utility functions
/// Clamp src in T1 to the desired precision and convert it to T2
template <typename T1, typename T2 = std::uint8_t>
FBGEMM_API T2 clamp(T1 src, int precision, bool is_signed = false)
// TODO: T26263653 fix signed-integer-overflow undefined behavior
#if defined(__has_feature)
#if __has_feature(__address_sanitizer__)
__attribute__((__no_sanitize__("signed-integer-overflow")))
#endif
#endif
{
std::int32_t min = is_signed ? -(1LL << (precision - 1)) : 0;
std::int32_t max =
is_signed ? ((1LL << (precision - 1)) - 1) : (1LL << precision) - 1;
// Make sure T1 and T2 can represent the precision
assert(min >= std::numeric_limits<T1>::lowest());
assert(min >= std::numeric_limits<T2>::lowest());
assert(max <= std::numeric_limits<T1>::max());
assert(max <= std::numeric_limits<T2>::max());
return std::min<T1>(std::max<T1>(src, min), max);
}
/// Quantize src using zero_point and scale, clamp to the specified precision,
/// and convert it to type T
template <typename T>
FBGEMM_API T Quantize(
float src,
std::int32_t zero_point,
float scale,
int result_precision,
bool result_is_signed = std::is_signed<T>::value) {
const float transformed_val = zero_point + src / scale;
return clamp<std::int64_t, T>(
static_cast<std::int64_t>(std::nearbyint(transformed_val)),
result_precision,
result_is_signed);
}
template <typename T>
FBGEMM_API T Quantize(float src, const TensorQuantizationParams& qparams) {
return Quantize<T>(src, qparams.zero_point, qparams.scale, qparams.precision);
}
template <typename T>
FBGEMM_API void Quantize(
const float* src,
T* dst,
int len,
const TensorQuantizationParams& qparams);
/*
* @brief Quantize floating point data in src to type T
*
* @tparam T output quantized data type (int8_t, uint8_t and int32_t are
* supported)
*
* @tparam T LAYOUT layout of input tensor in src. (KCX and KXC are supported)
* KCX corresponds to KCRS or KCTRS (for weight tensors with
* time dimension)
* KXC corresponds to KRSC or KTRSC (for weight tensors with
* time dimension)
*
* @params K Output channels for weight tensors
* @params C Number of channels
* @params X R*S or T*R*S
* @params G Groups (if G == C the function performs channelwise quantization;
* if 1 < G < C the function performs groupwise quantization;
* if G == 1 the function performs per tensor quantization;)
* @params scales floating point scales.
* Size should be equal G
* @params zero_points zero points (should be reprsentable in type T).
* Size should be equal G
*/
template <typename T, layout_t LAYOUT = layout_t::KCX>
FBGEMM_API void QuantizeGroupwise(
const float* src,
int K,
int C,
int X,
int G,
const float* scales,
const std::int32_t* zero_points,
T* dst);
template <typename T>
FBGEMM_API float Dequantize(T src, const TensorQuantizationParams& qparams) {
return qparams.scale * (src - qparams.zero_point);
}
template <typename T>
FBGEMM_API void Dequantize(
const T* src,
float* dst,
int len,
const TensorQuantizationParams& qparams) {
for (std::size_t i = 0; i < len; i++) {
dst[i] = Dequantize(src[i], qparams);
}
}
////////////////////////////////////////////////////////////////////////////////
// Requantization (pure fixed-point)
FBGEMM_API std::int64_t
SaturatingRoundingMulWithShift(std::int32_t a, std::int32_t b, int right_shift);
template <typename T>
FBGEMM_API T Requantize(
std::int32_t src, // int32 input before requantization
std::int32_t zero_point,
std::int32_t multiplier,
int right_shift,
int result_precision,
bool result_is_signed = false) {
std::int64_t quantized_down =
zero_point + SaturatingRoundingMulWithShift(src, multiplier, right_shift);
return clamp<std::int64_t, T>(
quantized_down, result_precision, result_is_signed);
}
template <typename T>
FBGEMM_API T RequantizeFixedPoint(
std::int32_t src, // int32 input before requantization
const RequantizationParams& params) {
return Requantize<T>(
src,
params.target_qparams.zero_point,
params.multiplier,
params.right_shift,
params.target_qparams.precision);
}
template <typename T>
FBGEMM_API void RequantizeFixedPoint(
const std::int32_t* src,
T* dst,
int len,
const RequantizationParams& params);
////////////////////////////////////////////////////////////////////////////////
// Requantization (with floats)
template <typename T>
FBGEMM_API T Requantize(
std::int32_t src, // int32 input before requantization
std::int32_t zero_point,
float multiplier,
int result_precision,
bool result_is_signed = false) {
long quantized_down = zero_point + std::lrintf(src * multiplier);
return clamp<long, T>(quantized_down, result_precision, result_is_signed);
}
template <typename T>
FBGEMM_API T Requantize(
std::int32_t src, // int32 input before requantization
const RequantizationParams& params) {
return Requantize<T>(
src,
params.target_qparams.zero_point,
params.real_multiplier,
params.target_qparams.precision);
}
template <typename T>
FBGEMM_API void Requantize(
const std::int32_t* src,
T* dst,
int len,
const RequantizationParams& params);
} // namespace fbgemm
|
