1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
#pragma once
#include "intgemm/intgemm_config.h"
#include <exception>
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX2
#include <immintrin.h>
#endif
#include <emmintrin.h>
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
/* MSVC does not appear to have target attributes but is also fine with just
* using intrinsics anywhere.
*
* The Intel compiler has a bug whereby constructors with target attributes do
* not link. Like this program doesn't compile with icpc:
* class Foo {
* public:
* __attribute__ ((target ("avx2"))) Foo() {}
* };
* int main() { Foo a; }
*
* It appears to be erroneously activating function multiversioning when only
* one version of a constructor with target attributes is defined. Normal
* methods with one target attribute work fine. The Intel compiler also allows
* intrinsics without any target attributes so we just leave them blank.
*/
#define INTGEMM_SSE2
#define INTGEMM_SSSE3
#define INTGEMM_AVX2
#define INTGEMM_AVX512F
#define INTGEMM_AVX512BW
#define INTGEMM_AVX512DQ
#define INTGEMM_AVX512VNNI
#else
/* gcc and clang take lists of all the flavors */
#define INTGEMM_SSE2 __attribute__ ((target ("sse2")))
#define INTGEMM_SSSE3 __attribute__ ((target ("ssse3")))
#define INTGEMM_AVX2 __attribute__ ((target ("avx2")))
#define INTGEMM_AVX512F __attribute__ ((target ("avx512f")))
#define INTGEMM_AVX512BW __attribute__ ((target ("avx512f,avx512bw,avx512dq")))
#define INTGEMM_AVX512DQ __attribute__ ((target ("avx512f,avx512bw,avx512dq")))
#define INTGEMM_AVX512VNNI __attribute__ ((target ("avx512f,avx512bw,avx512dq,avx512vnni")))
#endif
namespace intgemm {
// This will be thrown if a CPU isn't supported by the routines (16-bit without SSE2 or 8-bit without SSSE3).
class UnsupportedCPU : public std::exception {
public:
UnsupportedCPU() {}
~UnsupportedCPU() throw() {}
const char *what() const throw() override {
return "Integer matrix multiplication has not been efficiently implemented for your CPU.";
}
};
typedef unsigned int Index;
// If you want to detect the CPU and dispatch yourself, here's what to use:
enum class CPUType {
UNSUPPORTED = 0,
SSE2 = 1,
SSSE3 = 2,
AVX2 = 3,
AVX512BW = 4,
AVX512VNNI = 5
};
// Running CPU type. This is defined in intgemm.cc (as the dispatcher).
extern const CPUType kCPU;
struct MeanStd {
float mean;
float stddev;
};
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX512VNNI
namespace AVX512VNNI {
typedef __m512i Register;
typedef __m512 FRegister;
} // namespace AVX512VNNI
#endif
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX512BW
namespace AVX512BW {
typedef __m512i Register;
typedef __m512 FRegister;
} // namespace AVX512BW
#endif
#ifdef INTGEMM_COMPILER_SUPPORTS_AVX2
namespace AVX2 {
typedef __m256i Register;
typedef __m256 FRegister;
} // namespace AVX2
#endif
namespace SSSE3 {
typedef __m128i Register;
typedef __m128 FRegister;
} // namespace SSSE3
namespace SSE2 {
typedef __m128i Register;
typedef __m128 FRegister;
} // namespace SSE2
} // namespace intgemm
|
