Welcome to mirror list, hosted at ThFree Co, Russian Federation.

index.html - github.com/windirstat/llfio.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
blob: 41618c8d89869ec162f7305cfd1cc677f04cdd4b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.17"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>LLFIO: Main Page</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="navtreedata.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/searchdata.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  <td id="projectalign" style="padding-left: 0.5em;">
   <div id="projectname">LLFIO
   &#160;<span id="projectnumber">v2.00 late beta</span>
   </div>
  </td>
 </tr>
 </tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.17 -->
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3&amp;dn=gpl-2.0.txt GPL-v2 */
var searchBox = new SearchBox("searchBox", "search",false,'Search');
/* @license-end */
</script>
<script type="text/javascript" src="menudata.js"></script>
<script type="text/javascript" src="menu.js"></script>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3&amp;dn=gpl-2.0.txt GPL-v2 */
$(function() {
  initMenu('',true,false,'search.php','Search');
  $(document).ready(function() { init_search(); });
});
/* @license-end */</script>
<div id="main-nav"></div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
  <div id="nav-tree">
    <div id="nav-tree-contents">
      <div id="nav-sync" class="sync"></div>
    </div>
  </div>
  <div id="splitbar" style="-moz-user-select:none;" 
       class="ui-resizable-handle">
  </div>
</div>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3&amp;dn=gpl-2.0.txt GPL-v2 */
$(document).ready(function(){initNavTree('index.html',''); initResizable(); });
/* @license-end */
</script>
<div id="doc-content">
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
</div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>

<div class="PageDoc"><div class="header">
  <div class="headertitle">
<div class="title">LLFIO Documentation</div>  </div>
</div><!--header-->
<div class="contents">
<div class="textblock"><center><table border="0" cellpadding="4">
<tr>
<td align="center"><a href="https://github.com/ned14/llfio">LLFIO</a><br  />
<a href="https://github.com/ned14/llfio">on GitHub</a>  </td><td align="center"><a href="https://my.cdash.org/index.php?project=Boost.AFIO">CTest summary</a><br  />
<a href="https://my.cdash.org/index.php?project=Boost.AFIO">dashboard</a>  </td><td align="center"><a href="https://github.com/ned14/llfio/actions?query=workflow%3A%22Unit+tests+Linux%22">Linux CI:</a><object type="image/svg+xml" data="https://github.com/ned14/llfio/workflows/Unit%20tests%20Linux/badge.svg?branch=master" style="pointer-events: none;"></object>  </td><td align="center"><a href="https://github.com/ned14/llfio/actions?query=workflow%3A%22Unit+tests+Mac+OS%22">Mac OS CI:</a><object type="image/svg+xml" data="https://github.com/ned14/llfio/workflows/Unit%20tests%20Mac%20OS/badge.svg?branch=master" style="pointer-events: none;"></object>  </td><td align="center"><a href="https://github.com/ned14/llfio/actions?query=workflow%3A%22Unit+tests+Windows%22">Windows CI:</a><object type="image/svg+xml" data="https://github.com/ned14/llfio/workflows/Unit%20tests%20Windows/badge.svg?branch=master" style="pointer-events: none;"></object>  </td><td align="center"><a href="https://github.com/ned14/llfio/releases">Prebuilt binaries</a>   </td></tr>
</table>
</center><p>Herein lies my proposed zero whole machine memory copy async file i/o and filesystem library for the C++ standard, intended for storage devices with ~1 microsecond 4Kb transfer latencies and those supporting Storage Class Memory (SCM)/Direct Access Storage (DAX). Its i/o overhead, including syscall overhead, has been benchmarked to 100 nanoseconds on Linux which corresponds to a theoretical maximum of 10M IOPS @ QD1, approx 40Gb/sec per thread. It has particularly strong support for writing portable filesystem algorithms which work well with directly mapped non-volatile storage such as Intel Optane.</p>
<p>It is a complete rewrite after a Boost peer review in August 2015. Its github source code repository lives at <a href="https://github.com/ned14/llfio">https://github.com/ned14/llfio</a>.</p>
<ul>
<li>LLFIO is the reference implementation for these C++ standardisations:<ul>
<li><code>llfio::path_view</code> is expected to enter the C++ 23 standard.</li>
<li><code>llfio::file_handle</code> and <code>llfio::mapped_file_handle</code> are on track for entering the C++ 23 standard.</li>
</ul>
</li>
<li>Portable to any conforming C++ 14 compiler with a working Filesystem TS in its STL.<ul>
<li>Note that VS2019 16.3 and libc++ 11 dropped support for Filesystem in C++ 14, so for those LLFIO's cmake forces on C++ 17.</li>
</ul>
</li>
<li>Fully clean with C++ 20.<ul>
<li>Will make use of any Coroutines, Concepts, Span, Byte etc if you have them, otherwise swaps in C++ 14 compatible alternatives.</li>
<li>NOTE that Ubuntu 18.04's libstdc++ 9 does not currently provide symbols for <code>&lt;codecvt&gt;</code> if you are building in C++ 20, so linking LLFIO programs on libstdc++ on that Linux if in C++ 20 will fail. Either use a different STL, manually rebuild libstdc++, or use C++ 17.</li>
</ul>
</li>
<li>Aims to support Microsoft Windows, Linux, Android, iOS, Mac OS and FreeBSD.<ul>
<li>Best effort to support older kernels up to their EOL (as of July 2020: &gt;= Windows 8.1, &gt;= Linux 2.6.32 (RHEL EOL), &gt;= Mac OS 10.13, &gt;= FreeBSD 11).</li>
</ul>
</li>
<li>Original error code is always preserved, even down to the original NT kernel error code if a NT kernel API was used.<ul>
<li>Optional configuration based on <a href="https://wg21.link/P1028">P1028</a> <em>SG14 status_code and standard error object for P0709 Zero-overhead deterministic exceptions</em>.</li>
</ul>
</li>
<li>Race free filesystem design used throughout (i.e. no TOCTOU).</li>
<li>Zero malloc, zero exception throw and zero whole system memory copy design used throughout, even down to paths (which can hit 64Kb!).</li>
<li>Comprehensive support for virtual and mapped memory of both SCM/DAX and page cached storage, including large, huge and super pages.</li>
</ul>
<dl class="section note"><dt>Note</dt><dd>Most of this code is of early mature quality. It has been shipping in production with multiple vendors for some years now, indeed amongst many big data solutions it powers the low level custom database component of the US Security and Exchange Commission's MIDAS solution which ingresses Terabytes of trade data per day. It is quite reliable on Windows and Linux (less well tested on Mac OS), so be careful when using it!</dd></dl>
<p>Examples of use: </p><table width="100%" border="0" cellpadding="4">
<tr>
<td width="50%" valign="top"><div class="fragment"><div class="line">  <span class="keyword">namespace </span><a class="code" href="namespacellfio__v2__xxx.html">llfio</a> = <a class="code" href="group__config.html#gac9f7f0153adb9034d26c4554728f817a">LLFIO_V2_NAMESPACE</a>;</div>
<div class="line"> </div>
<div class="line">  <span class="comment">// Make me a 1 trillion element sparsely allocated integer array!</span></div>
<div class="line">  <a class="code" href="classllfio__v2__xxx_1_1mapped__file__handle.html">llfio::mapped_file_handle</a> mfh = <a class="code" href="namespacellfio__v2__xxx.html#a58d5390cad390de24a80748c8cd7dc5b">llfio::mapped_temp_inode</a>().value();</div>
<div class="line"> </div>
<div class="line">  <span class="comment">// On an extents based filing system, doesn&#39;t actually allocate any physical</span></div>
<div class="line">  <span class="comment">// storage but does map approximately 4Tb of all bits zero data into memory</span></div>
<div class="line">  (void) mfh.<a class="code" href="classllfio__v2__xxx_1_1mapped__file__handle.html#a01a5da3834fd354c5c3d38284b84ef75">truncate</a>(1000000000000ULL * <span class="keyword">sizeof</span>(<span class="keywordtype">int</span>));</div>
<div class="line"> </div>
<div class="line">  <span class="comment">// Create a typed view of the one trillion integers</span></div>
<div class="line">  llfio::attached&lt;int&gt; one_trillion_int_array(mfh);</div>
<div class="line"> </div>
<div class="line">  <span class="comment">// Write and read as you see fit, if you exceed physical RAM it&#39;ll be paged out</span></div>
<div class="line">  one_trillion_int_array[0] = 5;</div>
<div class="line">  one_trillion_int_array[999999999999ULL] = 6;</div>
</div><!-- fragment --> </td><td width="50%" valign="top"><div class="fragment"></div><!-- fragment -->  </td></tr>
</table>
<p>See <a href="https://github.com/ned14/llfio/blob/master/programs/fs-probe/fs_probe_results.yaml">https://github.com/ned14/llfio/blob/master/programs/fs-probe/fs_probe_results.yaml</a> for a database of latencies for various previously tested OS, filing systems and storage devices.</p>
<p>Todo list for already implemented parts: <a href="https://ned14.github.io/llfio/todo.html">https://ned14.github.io/llfio/todo.html</a></p>
<p>&#160;</p>
<center><span style="font-size: large; text-decoration: underline;"><a class="el" href="md__build.html">Build instructions can found here</a></span></center> <p>&#160;</p>
<h2><a class="anchor" id="autotoc_md18"></a>
v2 architecture and design implemented:</h2>
<table class="markdownTable">
<tr class="markdownTableHead">
<th class="markdownTableHeadNone">NEW in v2 </th><th class="markdownTableHeadNone">Boost peer review feedback </th><th class="markdownTableHeadNone"></th></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Universal native handle/fd abstraction instead of <code>void *</code>.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Perfectly/Ideally low memory (de)allocation per op (usually none).  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">noexcept API throughout returning error_code for failure instead of throwing exceptions.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">LLFIO v1 handle type split into hierarchy of types:<ol>
<li>
handle - provides open, close, get path, clone, set/unset append only, change caching, characteristics</li>
<li>
fs_handle - handles with an inode number</li>
<li>
path_handle - a race free anchor to a subset of the filesystem</li>
<li>
directory_handle - enumerates the filesystem</li>
<li>
io_handle - adds synchronous scatter-gather i/o, byte range locking</li>
<li>
file_handle - adds open/create file, get and set maximum extent</li>
<li>
mapped_file_handle - adds low latency memory mapped scatter-gather i/o</li>
</ol>
</td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Cancelable i/o (made possible thanks to dropping XP support).  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">All shared_ptr usage removed as all use of multiple threads removed.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Use of std::vector to transport scatter-gather sequences replaced with C++ 20 <code>span&lt;&gt;</code> borrowed views.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Completion callbacks are now some arbitrary type <code>U&amp;&amp;</code> instead of a future continuation. Type erasure for its storage is bound into the one single memory allocation for everything needed to execute the op, and so therefore overhead is optimal.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Filing system algorithms made generic and broken out into public <code>llfio::algorithms</code> template library (the LLFIO FTL).  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Abstraction of native handle management via bitfield specified "characteristics".  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Storage profiles, a YAML database of behaviours of hardware, OS and filing system combinations.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Absolute and interval deadline timed i/o throughout (made possible thanks to dropping XP support).  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Dependency on ASIO/Networking TS removed completely.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Four choices of algorithm implementing a shared filing system mutex.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Uses CMake, CTest, CDash and CPack with automatic usage of C++ Modules or precompiled headers where available.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Far more comprehensive memory map and virtual memory facilities.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Much more granular, micro level unit testing of individual functions.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Much more granular, micro level internal logging of every code path taken.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Path views used throughout, thus avoiding string copying and allocation in <code>std::filesystem::path</code>.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Paths are equally interpreted as UTF-8 on all platforms.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">We never store nor retain a path, as they are inherently racy and are best avoided.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Parent handle caching is hard coded in, it is now an optional user applied templated adapter class.  </td></tr>
</table>
<p>Todo:</p>
<table class="markdownTable">
<tr class="markdownTableHead">
<th class="markdownTableHeadNone">NEW in v2 </th><th class="markdownTableHeadNone">Boost peer review feedback </th><th class="markdownTableHeadNone"></th></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">clang AST assisted SWIG bindings for other languages.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Statistical tracking of operation latencies so realtime IOPS can be measured.  </td></tr>
</table>
<h2><a class="anchor" id="autotoc_md19"></a>
Planned features implemented:</h2>
<table class="markdownTable">
<tr class="markdownTableHead">
<th class="markdownTableHeadNone">NEW in v2 </th><th class="markdownTableHeadNone">Windows </th><th class="markdownTableHeadNone">POSIX </th><th class="markdownTableHeadNone"></th></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Native handle cloning.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ (up from four) </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Maximum possible (seven) forms of kernel caching.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Absolute path open.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Relative "anchored" path open enabling race free file system.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Win32 path support (260 path limit).  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">NT kernel path support (32,768 path limit).  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Synchronous universal scatter-gather i/o.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ (POSIX AIO support) </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Asynchronous universal scatter-gather i/o.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">i/o deadlines and cancellation.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Retrieving and setting the current maximum extent (size) of an open file.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Retrieving the current path of an open file irrespective of where it has been renamed to by third parties.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">statfs_t ported over from LLFIO v1.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">utils namespace ported over from LLFIO v1.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>shared_fs_mutex</code> shared/exclusive entities locking based on lock files  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Byte range shared/exclusive locking.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>shared_fs_mutex</code> shared/exclusive entities locking based on byte ranges  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>shared_fs_mutex</code> shared/exclusive entities locking based on atomic append  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Memory mapped files and virtual memory management (<code>section_handle</code>, <code>map_handle</code> and <code>mapped_file_handle</code>)  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>shared_fs_mutex</code> shared/exclusive entities locking based on memory maps  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Universal portable UTF-8 path views.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">"Hole punching" and hole enumeration ported over from LLFIO v1.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Directory handles and very fast directory enumeration ported over from LLFIO v1.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>shared_fs_mutex</code> shared/exclusive entities locking based on safe byte ranges  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Set random or sequential i/o (prefetch).  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>llfio::algorithm::trivial_vector&lt;T&gt;</code> with constant time reallocation if <code>T</code> is trivially copyable.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"><code>symlink_handle</code>.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Large, huge and massive page size support for memory allocation and (POSIX only) file maps.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">A mechanism for writing a <code>stat_t</code> onto an inode.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Graph based directory hierarchy traveral algorithm.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Graph based directory hierarchy summary algorithm.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Graph based reliable directory hierarchy deletion algorithm.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone">Intelligent file contents cloning between file handles.  </td></tr>
</table>
<p>Todo thereafter in order of priority:</p>
<table class="markdownTable">
<tr class="markdownTableHead">
<th class="markdownTableHeadNone">NEW in v2 </th><th class="markdownTableHeadNone">Windows </th><th class="markdownTableHeadNone">POSIX </th><th class="markdownTableHeadNone"></th></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Page allocator based on an index of linked list of free pages. See notes.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Optionally concurrent B+ tree index based on page allocator for key-value store.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Attributes extending <code>span&lt;buffers_type&gt;</code> with DMA colouring.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Coroutine generator for iterating a file's contents in DMA friendly way.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Ranges &amp; Concurrency based reliable directory hierarchy copy algorithm.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Ranges &amp; Concurrency based reliable directory hierarchy update (two and three way) algorithm.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Linux io_uring support for native non-blocking <code>O_DIRECT</code> i/o  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"><code>std::pmr::memory_resource</code> adapting a file backing if on C++ 17.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Extended attributes support.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Algorithm to replace all duplicate content with hard links.  </td></tr>
<tr class="markdownTableRowOdd">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Algorithm to figure out all paths for a hard linked inode.  </td></tr>
<tr class="markdownTableRowEven">
<td class="markdownTableBodyNone">✔ </td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone"></td><td class="markdownTableBodyNone">Algorithm to compare two or three directory enumerations and give differences.  </td></tr>
</table>
<p>Features possibly to be added after a Boost peer review:</p><ul>
<li>Directory change monitoring.</li>
<li>Permissions support (ACLs).</li>
</ul>
<table width="100%" border="0" cellpadding="4">
<tr>
<th colspan="3">Why you might need LLFIO<hr  />
  </th></tr>
<tr>
<td valign="top" width="33%">Manufacturer claimed 4Kb transfer latencies for the physical hardware:<ul>
<li>Spinning rust hard drive latency @ QD1: <b>9000us</b></li>
<li>SATA flash drive latency @ QD1: <b>800us</b></li>
<li>NVMe flash drive latency @ QD1: <b>300us</b></li>
<li>RTT UDP packet latency over a LAN: <b>60us</b></li>
<li>NVMe Optane drive latency @ QD1: <b>60us</b></li>
<li><code>memcpy(4Kb)</code> latency: <b>5us</b> (main memory) to <b>1.3us</b> (L3 cache)</li>
<li>RTT PCIe latency: <b>0.5us</b>  </li>
</ul>
</td><td valign="top" width="33%">100% read QD1 4Kb direct transfer latencies for the software with LLFIO:<ul>
<li>&lt; 99% spinning rust hard drive latency: Windows <b>187,231us</b> FreeBSD <b>9,836us</b> Linux <b>26,468us</b></li>
<li>&lt; 99% SATA flash drive latency: Windows <b>290us</b> Linux <b>158us</b></li>
<li>&lt; 99% NVMe drive latency: Windows <b>37us</b> FreeBSD <b>70us</b> Linux <b>30us</b>  </li>
</ul>
</td><td valign="top" width="33%">75% read 25% write QD4 4Kb direct transfer latencies for the software with LLFIO:<ul>
<li>&lt; 99% spinning rust hard drive latency: Windows <b>48,185us</b> FreeBSD <b>61,834us</b> Linux <b>104,507us</b></li>
<li>&lt; 99% SATA flash drive latency: Windows <b>1,812us</b> Linux <b>1,416us</b></li>
<li>&lt; 99% NVMe drive latency: Windows <b>50us</b> FreeBSD <b>143us</b> Linux <b>40us</b>   </li>
</ul>
</td></tr>
</table>
<p>Max bandwidth for the physical hardware:</p><ul>
<li>DDR4 2133: <b>30Gb/sec</b> (main memory)</li>
<li>x4 PCIe 4.0: <b>7.5Gb/sec</b> (arrives end of 2017, the 2018 NVMe drives will use PCIe 4.0)</li>
<li>x4 PCIe 3.0: <b>3.75Gb/sec</b> (985Mb/sec per PCIe lane)</li>
<li>2017 XPoint drive (x4 PCIe 3.0): <b>2.5Gb/sec</b></li>
<li>2017 NVMe flash drive (x4 PCIe 3.0): <b>2Gb/sec</b></li>
<li>10Gbit LAN: <b>1.2Gb/sec</b> </li>
</ul>
</div></div><!-- PageDoc -->
</div><!-- contents -->
</div><!-- doc-content -->
<div class="ttc" id="aclassllfio__v2__xxx_1_1mapped__file__handle_html_a01a5da3834fd354c5c3d38284b84ef75"><div class="ttname"><a href="classllfio__v2__xxx_1_1mapped__file__handle.html#a01a5da3834fd354c5c3d38284b84ef75">llfio_v2_xxx::mapped_file_handle::truncate</a></div><div class="ttdeci">virtual result&lt; extent_type &gt; truncate(extent_type newsize) noexcept override</div><div class="ttdoc">Resize the current maximum permitted extent of the mapped file to the given extent,...</div></div>
<div class="ttc" id="anamespacellfio__v2__xxx_html_a58d5390cad390de24a80748c8cd7dc5b"><div class="ttname"><a href="namespacellfio__v2__xxx.html#a58d5390cad390de24a80748c8cd7dc5b">llfio_v2_xxx::mapped_temp_inode</a></div><div class="ttdeci">result&lt; mapped_file_handle &gt; mapped_temp_inode(mapped_file_handle::size_type reservation=0, const path_handle &amp;dir=path_discovery::storage_backed_temporary_files_directory(), mapped_file_handle::mode _mode=mapped_file_handle::mode::write, mapped_file_handle::flag flags=mapped_file_handle::flag::none) noexcept</div><div class="ttdef"><b>Definition:</b> mapped_file_handle.hpp:790</div></div>
<div class="ttc" id="agroup__config_html_gac9f7f0153adb9034d26c4554728f817a"><div class="ttname"><a href="group__config.html#gac9f7f0153adb9034d26c4554728f817a">LLFIO_V2_NAMESPACE</a></div><div class="ttdeci">#define LLFIO_V2_NAMESPACE</div><div class="ttdoc">The namespace of this LLFIO v2 which will be some unknown inline namespace starting with v2_ inside t...</div><div class="ttdef"><b>Definition:</b> config.hpp:178</div></div>
<div class="ttc" id="anamespacellfio__v2__xxx_html"><div class="ttname"><a href="namespacellfio__v2__xxx.html">llfio_v2_xxx</a></div><div class="ttdoc">The LLFIO namespace.</div><div class="ttdef"><b>Definition:</b> config.hpp:159</div></div>
<div class="ttc" id="aclassllfio__v2__xxx_1_1mapped__file__handle_html"><div class="ttname"><a href="classllfio__v2__xxx_1_1mapped__file__handle.html">llfio_v2_xxx::mapped_file_handle</a></div><div class="ttdoc">A memory mapped regular file or device.</div><div class="ttdef"><b>Definition:</b> mapped_file_handle.hpp:155</div></div>
<!-- start footer part -->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
  <ul>
    <li class="footer">Generated by
    <a href="http://www.doxygen.org/index.html">
    <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.17 </li>
  </ul>
</div>
</body>
</html>