1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
#include "intgemm.h"
#include "stats.h"
#include <stdlib.h>
#include <iostream>
namespace intgemm {
namespace {
// Return the maximum CPU model that's found and supported at compile time.
CPUType RealCPUID() {
#if defined(WASM)
// emscripten does SSE4.1 but we only use up to SSSE3.
return CPUType::SSSE3;
#elif defined(__INTEL_COMPILER)
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX512VNNI
if (_may_i_use_cpu_feature(_FEATURE_AVX512_VNNI)) return CPUType::AVX512VNNI;
# endif
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX512BW
if (_may_i_use_cpu_feature(_FEATURE_AVX512BW)) return CPUType::AVX512BW;
# endif
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX2
if (_may_i_use_cpu_feature(_FEATURE_AVX2)) return CPUType::AVX2;
# endif
if (_may_i_use_cpu_feature(_FEATURE_SSSE3)) return CPUType::SSSE3;
if (_may_i_use_cpu_feature(_FEATURE_SSE2)) return CPUType::SSE2;
return CPUType::UNSUPPORTED;
#else
// Not emscripten, not Intel compiler
# if defined(_MSC_VER)
int regs[4];
int &eax = regs[0], &ebx = regs[1], &ecx = regs[2], &edx = regs[3];
__cpuid(regs, 0);
int m = eax;
# else
/* gcc and clang.
* If intgemm is compiled by gcc 6.4.1 then dlopened into an executable
* compiled by gcc 7.3.0, there will be a undefined symbol __cpu_info.
* Work around this by calling the intrinsics more directly instead of
* __builtin_cpu_supports.
*
* clang 6.0.0-1ubuntu2 supports vnni but doesn't have
* __builtin_cpu_supports("avx512vnni")
* so use the hand-coded CPUID for clang.
*/
unsigned int m = __get_cpuid_max(0, 0);
unsigned int eax, ebx, ecx, edx;
# endif
if (m >= 7) {
# if defined(_MSC_VER)
__cpuid(regs, 7);
# else
__cpuid_count(7, 0, eax, ebx, ecx, edx);
# endif
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX512VNNI
if (ecx & (1 << 11)) return CPUType::AVX512VNNI;
# endif
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX512BW
if (ebx & (1 << 30)) return CPUType::AVX512BW;
# endif
# ifdef INTGEMM_COMPILER_SUPPORTS_AVX2
if (ebx & (1 << 5)) return CPUType::AVX2;
# endif
}
if (m >= 1) {
# if defined(_MSC_VER)
__cpuid(regs, 1);
# else
__cpuid_count(1, 0, eax, ebx, ecx, edx);
# endif
if (ecx & (1 << 9)) return CPUType::SSSE3;
if (edx & (1 << 26)) return CPUType::SSE2;
}
return CPUType::UNSUPPORTED;
#endif
}
CPUType EnvironmentCPUID() {
const char *env_override = getenv("INTGEMM_CPUID");
if (!env_override) return CPUType::AVX512VNNI; /* This will be capped to actual ID */
if (!strcmp(env_override, "AVX512VNNI")) return CPUType::AVX512VNNI;
if (!strcmp(env_override, "AVX512BW")) return CPUType::AVX512BW;
if (!strcmp(env_override, "AVX2")) return CPUType::AVX2;
if (!strcmp(env_override, "SSSE3")) return CPUType::SSSE3;
if (!strcmp(env_override, "SSE2")) return CPUType::SSE2;
std::cerr << "Unrecognized INTGEMM_CPUID " << env_override << std::endl;
return CPUType::AVX512VNNI;
}
} // namespace
CPUType GetCPUID() {
static const CPUType kCPU = std::min(RealCPUID(), EnvironmentCPUID());
return kCPU;
}
float Unsupported_MaxAbsolute(const float * /*begin*/, const float * /*end*/) {
throw UnsupportedCPU();
}
MeanStd Unsupported_VectorMeanStd(const float * /*begin*/, const float * /*end*/, bool /*absolute*/) {
throw UnsupportedCPU();
}
void (*Int16::Quantize)(const float *input, int16_t *output, float quant_mult, Index size) = ChooseCPU(AVX512BW::Kernels16::Quantize, AVX512BW::Kernels16::Quantize, AVX2::Kernels16::Quantize, SSE2::Kernels16::Quantize, SSE2::Kernels16::Quantize, Unsupported_16bit::Quantize);
void (*Int16::PrepareB)(const float *input, int16_t *output, float quant_mult, Index rows, Index cols) = ChooseCPU(AVX512BW::Kernels16::PrepareB, AVX512BW::Kernels16::PrepareB, AVX2::Kernels16::PrepareB, SSE2::Kernels16::PrepareB, SSE2::Kernels16::PrepareB, Unsupported_16bit::PrepareB);
void (*Int16::PrepareBQuantizedTransposed)(const int16_t *input, int16_t *output, Index inner, Index B_untransposed_cols) = ChooseCPU(AVX512BW::Kernels16::PrepareBQuantizedTransposed, AVX512BW::Kernels16::PrepareBQuantizedTransposed, AVX2::Kernels16::PrepareBQuantizedTransposed, SSE2::Kernels16::PrepareBQuantizedTransposed, SSE2::Kernels16::PrepareBQuantizedTransposed, Unsupported_16bit::PrepareBQuantizedTransposed);
void (*Int16::PrepareBTransposed)(const float *input, int16_t *output, float quant_mult, Index inner, Index B_untransposed_cols) = ChooseCPU(AVX512BW::Kernels16::PrepareBTransposed, AVX512BW::Kernels16::PrepareBTransposed, AVX2::Kernels16::PrepareBTransposed, SSE2::Kernels16::PrepareBTransposed, SSE2::Kernels16::PrepareBTransposed, Unsupported_16bit::PrepareBTransposed);
void (*Int16::SelectColumnsB)(const int16_t *input, int16_t *output, Index rows, const Index *cols_begin, const Index *cols_end) = ChooseCPU(AVX512BW::Kernels16::SelectColumnsB, AVX512BW::Kernels16::SelectColumnsB, AVX2::Kernels16::SelectColumnsB, SSE2::Kernels16::SelectColumnsB, SSE2::Kernels16::SelectColumnsB, Unsupported_16bit::SelectColumnsB);
const char *const Int16::kName = ChooseCPU(AVX512BW::Kernels16::kName, AVX512BW::Kernels16::kName, AVX2::Kernels16::kName, SSE2::Kernels16::kName, SSE2::Kernels16::kName, Unsupported_16bit::kName);
void (*Int8::Quantize)(const float *input, int8_t *output, float quant_mult, Index size) = ChooseCPU(AVX512VNNI::Kernels8::Quantize, AVX512BW::Kernels8::Quantize, AVX2::Kernels8::Quantize, SSSE3::Kernels8::Quantize, Unsupported_8bit::Quantize, Unsupported_8bit::Quantize);
void (*Int8::QuantizeU)(const float *input, uint8_t *output, float quant_mult, Index size) = ChooseCPU(AVX512VNNI::Kernels8::QuantizeU, AVX512BW::Kernels8::QuantizeU, AVX2::Kernels8::QuantizeU, SSSE3::Kernels8::QuantizeU, Unsupported_8bit::QuantizeU, Unsupported_8bit::QuantizeU);
void (*Int8::PrepareB)(const float *input, int8_t *output, float quant_mult, Index rows, Index cols) = ChooseCPU(AVX512VNNI::Kernels8::PrepareB, AVX512BW::Kernels8::PrepareB, AVX2::Kernels8::PrepareB, SSSE3::Kernels8::PrepareB, Unsupported_8bit::PrepareB, Unsupported_8bit::PrepareB);
void (*Int8::PrepareBQuantizedTransposed)(const int8_t *input, int8_t *output, Index inner, Index B_untransposed_cols) = ChooseCPU(AVX512BW::Kernels8::PrepareBQuantizedTransposed, AVX512BW::Kernels8::PrepareBQuantizedTransposed, AVX2::Kernels8::PrepareBQuantizedTransposed, SSSE3::Kernels8::PrepareBQuantizedTransposed, Unsupported_8bit::PrepareBQuantizedTransposed, Unsupported_8bit::PrepareBQuantizedTransposed);
void (*Int8::PrepareBTransposed)(const float *input, int8_t *output, float quant_mult, Index inner, Index B_untransposed_cols) = ChooseCPU(AVX512BW::Kernels8::PrepareBTransposed, AVX512BW::Kernels8::PrepareBTransposed, AVX2::Kernels8::PrepareBTransposed, SSSE3::Kernels8::PrepareBTransposed, Unsupported_8bit::PrepareBTransposed, Unsupported_8bit::PrepareBTransposed);
void (*Int8::SelectColumnsB)(const int8_t *input, int8_t *output, Index rows, const Index *cols_begin, const Index *cols_end) = ChooseCPU(AVX512VNNI::Kernels8::SelectColumnsB, AVX512BW::Kernels8::SelectColumnsB, AVX2::Kernels8::SelectColumnsB, SSSE3::Kernels8::SelectColumnsB, Unsupported_8bit::SelectColumnsB, Unsupported_8bit::SelectColumnsB);
const char *const Int8::kName = ChooseCPU(AVX512VNNI::Kernels8::kName, AVX512BW::Kernels8::kName, AVX2::Kernels8::kName, SSSE3::Kernels8::kName, Unsupported_8bit::kName, Unsupported_8bit::kName);
void (*Int8Shift::QuantizeU)(const float *input, uint8_t *output, float quant_mult, Index size) = ChooseCPU(AVX512VNNI::Kernels8::QuantizeU, AVX512BW::Kernels8::QuantizeU, AVX2::Kernels8::QuantizeU, SSSE3::Kernels8::QuantizeU, Unsupported_8bit::QuantizeU, Unsupported_8bit::QuantizeU);
const char *const Int8Shift::kName = ChooseCPU(AVX512VNNI::Kernels8::kName, AVX512BW::Kernels8::kName, AVX2::Kernels8::kName, SSSE3::Kernels8::kName, Unsupported_8bit::kName, Unsupported_8bit::kName);
const CPUType kCPU = ChooseCPU(CPUType::AVX512VNNI, CPUType::AVX512BW, CPUType::AVX2, CPUType::SSSE3, CPUType::SSE2, CPUType::UNSUPPORTED);
#if !defined(INTGEMM_COMPILER_SUPPORTS_AVX2)
namespace AVX2{
using SSE2::MaxAbsolute;
using SSE2::VectorMeanStd;
} // namespace AVX2
#endif
#if !defined(INTGEMM_COMPILER_SUPPORTS_AVX512BW)
namespace AVX512BW {
using AVX2::MaxAbsolute;
using AVX2::VectorMeanStd;
} // namespace AVX512BW
#endif
float (*MaxAbsolute)(const float *begin, const float *end) = ChooseCPU(AVX512BW::MaxAbsolute, AVX512BW::MaxAbsolute, AVX2::MaxAbsolute, SSE2::MaxAbsolute, SSE2::MaxAbsolute, Unsupported_MaxAbsolute);
MeanStd (*VectorMeanStd)(const float *begin, const float *end, bool absolute) = ChooseCPU(AVX512BW::VectorMeanStd, AVX512BW::VectorMeanStd, AVX2::VectorMeanStd, SSE2::VectorMeanStd, SSE2::VectorMeanStd, Unsupported_VectorMeanStd);
constexpr const char *const Unsupported_16bit::kName;
constexpr const char *const Unsupported_8bit::kName;
constexpr const char *const SSE2::Kernels16::kName;
constexpr const char *const SSSE3::Kernels8::kName;
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX2
constexpr const char *const AVX2::Kernels8::kName;
constexpr const char *const AVX2::Kernels16::kName;
#endif
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX512BW
constexpr const char *const AVX512BW::Kernels8::kName;
constexpr const char *const AVX512BW::Kernels16::kName;
#endif
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX512VNNI
constexpr const char *const AVX512VNNI::Kernels8::kName;
#endif
}
|
