blob: 47fdb5ca961d38da78c1b3d578a0bf2e3bcb41c8 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#include "./utils.h"
#include <atomic>
#include <cstdlib>
#include <memory>
#include <stdexcept>
#include <vector>
#include "ctranslate2/utils.h"
namespace ctranslate2 {
namespace cuda {
const char* cublasGetStatusName(cublasStatus_t status)
{
switch (status)
{
case CUBLAS_STATUS_SUCCESS:
return "CUBLAS_STATUS_SUCCESS";
case CUBLAS_STATUS_NOT_INITIALIZED:
return "CUBLAS_STATUS_NOT_INITIALIZED";
case CUBLAS_STATUS_ALLOC_FAILED:
return "CUBLAS_STATUS_ALLOC_FAILED";
case CUBLAS_STATUS_INVALID_VALUE:
return "CUBLAS_STATUS_INVALID_VALUE";
case CUBLAS_STATUS_ARCH_MISMATCH:
return "CUBLAS_STATUS_ARCH_MISMATCH";
case CUBLAS_STATUS_MAPPING_ERROR:
return "CUBLAS_STATUS_MAPPING_ERROR";
case CUBLAS_STATUS_EXECUTION_FAILED:
return "CUBLAS_STATUS_EXECUTION_FAILED";
case CUBLAS_STATUS_INTERNAL_ERROR:
return "CUBLAS_STATUS_INTERNAL_ERROR";
case CUBLAS_STATUS_NOT_SUPPORTED:
return "CUBLAS_STATUS_NOT_SUPPORTED";
case CUBLAS_STATUS_LICENSE_ERROR:
return "CUBLAS_STATUS_LICENSE_ERROR";
default:
return "UNKNOWN";
}
}
// We assign the default CUDA stream to the main thread since it can interact with
// multiple devices (e.g. load replicas on each GPU). The main thread is created
// before the others, so it will be the first to see the flag below set to true.
static std::atomic<bool> is_main_thread(true);
class CudaStream {
public:
CudaStream() {
if (is_main_thread) {
is_main_thread = false;
_stream = cudaStreamDefault;
} else {
CUDA_CHECK(cudaGetDevice(&_device));
CUDA_CHECK(cudaStreamCreate(&_stream));
}
}
~CudaStream() {
if (_stream != cudaStreamDefault) {
ScopedDeviceSetter scoped_device_setter(Device::CUDA, _device);
cudaStreamDestroy(_stream);
}
}
cudaStream_t get() const {
return _stream;
}
private:
int _device;
cudaStream_t _stream;
};
class CublasHandle {
public:
CublasHandle() {
CUDA_CHECK(cudaGetDevice(&_device));
CUBLAS_CHECK(cublasCreate(&_handle));
CUBLAS_CHECK(cublasSetStream(_handle, get_cuda_stream()));
}
~CublasHandle() {
ScopedDeviceSetter scoped_device_setter(Device::CUDA, _device);
cublasDestroy(_handle);
}
cublasHandle_t get() const {
return _handle;
}
private:
int _device;
cublasHandle_t _handle;
};
// We create one cuBLAS/cuDNN handle per host thread. The handle is destroyed
// when the thread exits.
cudaStream_t get_cuda_stream() {
static thread_local CudaStream cuda_stream;
return cuda_stream.get();
}
cublasHandle_t get_cublas_handle() {
static thread_local CublasHandle cublas_handle;
return cublas_handle.get();
}
int get_gpu_count() {
int gpu_count = 0;
cudaError_t status = cudaGetDeviceCount(&gpu_count);
if (status != cudaSuccess)
return 0;
return gpu_count;
}
bool has_gpu() {
return get_gpu_count() > 0;
}
const cudaDeviceProp& get_device_properties(int device) {
static thread_local std::vector<std::unique_ptr<cudaDeviceProp>> cache;
if (device < 0) {
CUDA_CHECK(cudaGetDevice(&device));
}
if (device >= static_cast<int>(cache.size())) {
cache.resize(device + 1);
}
auto& device_prop = cache[device];
if (!device_prop) {
device_prop = std::make_unique<cudaDeviceProp>();
CUDA_CHECK(cudaGetDeviceProperties(device_prop.get(), device));
}
return *device_prop;
}
// See docs.nvidia.com/deeplearning/sdk/tensorrt-support-matrix/index.html
// for hardware support of reduced precision.
bool gpu_supports_int8(int device) {
const cudaDeviceProp& device_prop = get_device_properties(device);
return device_prop.major > 6 || (device_prop.major == 6 && device_prop.minor == 1);
}
bool gpu_has_int8_tensor_cores(int device) {
const cudaDeviceProp& device_prop = get_device_properties(device);
return device_prop.major > 7 || (device_prop.major == 7 && device_prop.minor >= 2);
}
bool gpu_has_fp16_tensor_cores(int device) {
const cudaDeviceProp& device_prop = get_device_properties(device);
return device_prop.major >= 7;
}
bool have_same_compute_capability(const std::vector<int>& devices) {
if (devices.size() > 1) {
int ref_major = -1;
int ref_minor = -1;
for (const int device : devices) {
const cudaDeviceProp& device_prop = get_device_properties(device);
const int major = device_prop.major;
const int minor = device_prop.minor;
if (ref_major < 0) {
ref_major = major;
ref_minor = minor;
} else if (major != ref_major || minor != ref_minor)
return false;
}
}
return true;
}
}
}
|
