1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
/* Misc utilities
(C) 2015-2017 Niall Douglas <http://www.nedproductions.biz/> (7 commits)
File Created: Dec 2015
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License in the accompanying file
Licence.txt or at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Distributed under the Boost Software License, Version 1.0.
(See accompanying file Licence.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)
*/
#include "../../../utils.hpp"
#include "import.hpp"
LLFIO_V2_NAMESPACE_BEGIN
namespace utils
{
// Stupid MSVC ...
namespace detail
{
using namespace LLFIO_V2_NAMESPACE::detail;
}
size_t page_size() noexcept
{
static size_t ret;
if(ret == 0u)
{
SYSTEM_INFO si{};
memset(&si, 0, sizeof(si));
GetSystemInfo(&si);
ret = si.dwPageSize;
}
return ret;
}
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 6387) // MSVC sanitiser warns that GetModuleHandleA() might fail (hah!)
#endif
const std::vector<size_t> &page_sizes(bool only_actually_available)
{
static spinlock lock;
std::lock_guard<decltype(lock)> g(lock);
static std::vector<size_t> pagesizes, pagesizes_available;
if(pagesizes.empty())
{
using GetLargePageMinimum_t = size_t(WINAPI *)(void);
SYSTEM_INFO si{};
memset(&si, 0, sizeof(si));
GetSystemInfo(&si);
pagesizes.push_back(si.dwPageSize);
pagesizes_available.push_back(si.dwPageSize);
auto GetLargePageMinimum_ = reinterpret_cast<GetLargePageMinimum_t>(GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "GetLargePageMinimum"));
if(GetLargePageMinimum_ != nullptr)
{
windows_nt_kernel::init();
using namespace windows_nt_kernel;
pagesizes.push_back(GetLargePageMinimum_());
/* Attempt to enable SeLockMemoryPrivilege */
HANDLE token;
if(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token) != 0)
{
auto untoken = make_scope_exit([&token]() noexcept { CloseHandle(token); });
TOKEN_PRIVILEGES privs{};
memset(&privs, 0, sizeof(privs));
privs.PrivilegeCount = 1;
if(LookupPrivilegeValueW(nullptr, L"SeLockMemoryPrivilege", &privs.Privileges[0].Luid))
{
privs.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
if((AdjustTokenPrivileges(token, FALSE, &privs, 0, nullptr, nullptr) != 0) && GetLastError() == S_OK)
{
pagesizes_available.push_back(GetLargePageMinimum_());
}
}
}
}
}
return only_actually_available ? pagesizes_available : pagesizes;
}
#ifdef _MSC_VER
#pragma warning(pop)
#endif
void random_fill(char *buffer, size_t bytes) noexcept
{
windows_nt_kernel::init();
using namespace windows_nt_kernel;
if(RtlGenRandom(buffer, static_cast<ULONG>(bytes)) == 0u)
{
LLFIO_LOG_FATAL(0, "llfio: Kernel crypto function failed");
std::terminate();
}
}
result<void> flush_modified_data() noexcept
{
windows_nt_kernel::init();
using namespace windows_nt_kernel;
static bool prived;
if(!prived)
{
HANDLE processToken;
if(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &processToken) == FALSE)
{
return win32_error();
}
auto unprocessToken = make_scope_exit([&processToken]() noexcept { CloseHandle(processToken); });
{
LUID luid{};
if(!LookupPrivilegeValueW(nullptr, L"SeProfileSingleProcessPrivilege", &luid))
{
return win32_error();
}
TOKEN_PRIVILEGES tp{};
tp.PrivilegeCount = 1;
tp.Privileges[0].Luid = luid;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
if(AdjustTokenPrivileges(processToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), static_cast<PTOKEN_PRIVILEGES>(nullptr), static_cast<PDWORD>(nullptr)) == 0)
{
return win32_error();
}
if(GetLastError() == ERROR_NOT_ALL_ASSIGNED)
{
return win32_error();
}
}
prived = true;
}
typedef enum _SYSTEM_MEMORY_LIST_COMMAND
{
MemoryCaptureAccessedBits,
MemoryCaptureAndResetAccessedBits,
MemoryEmptyWorkingSets,
MemoryFlushModifiedList,
MemoryPurgeStandbyList,
MemoryPurgeLowPriorityStandbyList,
MemoryCommandMax
} SYSTEM_MEMORY_LIST_COMMAND; // NOLINT
// Write all modified pages to storage
SYSTEM_MEMORY_LIST_COMMAND command = MemoryPurgeStandbyList;
NTSTATUS ntstat = NtSetSystemInformation(80 /*SystemMemoryListInformation*/, &command, sizeof(SYSTEM_MEMORY_LIST_COMMAND));
if(ntstat != STATUS_SUCCESS)
{
return ntkernel_error(ntstat);
}
return success();
}
result<void> drop_filesystem_cache() noexcept
{
windows_nt_kernel::init();
using namespace windows_nt_kernel;
static bool prived;
if(!prived)
{
HANDLE processToken;
if(OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &processToken) == FALSE)
{
return win32_error();
}
auto unprocessToken = make_scope_exit([&processToken]() noexcept { CloseHandle(processToken); });
{
LUID luid{};
if(!LookupPrivilegeValueW(nullptr, L"SeIncreaseQuotaPrivilege", &luid))
{
return win32_error();
}
TOKEN_PRIVILEGES tp{};
tp.PrivilegeCount = 1;
tp.Privileges[0].Luid = luid;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
if(AdjustTokenPrivileges(processToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), static_cast<PTOKEN_PRIVILEGES>(nullptr), static_cast<PDWORD>(nullptr)) == 0)
{
return win32_error();
}
if(GetLastError() == ERROR_NOT_ALL_ASSIGNED)
{
return win32_error();
}
}
prived = true;
}
// Flush modified data so dropping the cache drops everything
OUTCOME_TRYV(flush_modified_data());
// Drop filesystem cache
if(SetSystemFileCacheSize(static_cast<SIZE_T>(-1), static_cast<SIZE_T>(-1), 0) == 0)
{
return win32_error();
}
return success();
}
result<process_memory_usage> current_process_memory_usage() noexcept {
// Amazingly Win32 doesn't expose private working set, so to avoid having
// to iterate all the pages in the process and calculate, use a hidden
// NT kernel call
windows_nt_kernel::init();
using namespace windows_nt_kernel;
ULONG written = 0;
_VM_COUNTERS_EX2 vmc;
memset(&vmc, 0, sizeof(vmc));
NTSTATUS ntstat = NtQueryInformationProcess(GetCurrentProcess(), ProcessVmCounters, &vmc, sizeof(vmc), &written);
if(ntstat < 0)
{
return ntkernel_error(ntstat);
}
process_memory_usage ret;
/* Notes:
Apparently PrivateUsage is the commit charge on Windows. It always equals PagefileUsage.
It is the total amount of private anonymous pages committed.
SharedCommitUsage is amount of non-binary Shared memory committed.
Therefore total non-binary commit = PrivateUsage + SharedCommitUsage
WorkingSetSize is the total amount of program binaries, non-binary shared memory, and anonymous pages faulted in.
PrivateWorkingSetSize is the amount of private anonymous pages faulted into the process.
Therefore remainder is all shared working set faulted into the process.
*/
ret.total_address_space_in_use = vmc.VirtualSize;
ret.total_address_space_paged_in = vmc.WorkingSetSize;
ret.private_committed = vmc.PrivateUsage;
ret.private_paged_in = vmc.PrivateWorkingSetSize;
return ret;
}
namespace detail
{
large_page_allocation allocate_large_pages(size_t bytes)
{
large_page_allocation ret(calculate_large_page_allocation(bytes));
DWORD type = MEM_COMMIT | MEM_RESERVE;
if(ret.page_size_used > 65536)
{
type |= MEM_LARGE_PAGES;
}
ret.p = VirtualAlloc(nullptr, ret.actual_size, type, PAGE_READWRITE);
if(ret.p == nullptr)
{
if(ERROR_NOT_ENOUGH_MEMORY == GetLastError())
{
ret.p = VirtualAlloc(nullptr, ret.actual_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
}
}
#ifndef NDEBUG
else if(ret.page_size_used > 65536)
{
printf("llfio: Large page allocation successful\n");
}
#endif
return ret;
}
void deallocate_large_pages(void *p, size_t bytes)
{
(void) bytes;
if(VirtualFree(p, 0, MEM_RELEASE) == 0)
{
LLFIO_LOG_FATAL(p, "llfio: Freeing large pages failed");
std::terminate();
}
}
} // namespace detail
} // namespace utils
LLFIO_V2_NAMESPACE_END
|
