1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
|
/* A handle to a file
(C) 2015-2017 Niall Douglas <http://www.nedproductions.biz/> (20 commits)
File Created: Dec 2015
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License in the accompanying file
Licence.txt or at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Distributed under the Boost Software License, Version 1.0.
(See accompanying file Licence.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)
*/
#ifndef LLFIO_FILE_HANDLE_H
#define LLFIO_FILE_HANDLE_H
#include "lockable_io_handle.hpp"
#include "path_discovery.hpp"
#include "utils.hpp"
//! \file file_handle.hpp Provides file_handle
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4251) // dll interface
#endif
LLFIO_V2_NAMESPACE_EXPORT_BEGIN
class io_service;
/*! \class file_handle
\brief A handle to a regular file or device, kept data layout compatible with
async_file_handle.
<table>
<tr><th></th><th>Cost of opening</th><th>Cost of i/o</th><th>Concurrency and Atomicity</th><th>Other remarks</th></tr>
<tr><td>`file_handle`</td><td>Least</td><td>Syscall</td><td>POSIX guarantees (usually)</td><td>Least gotcha</td></tr>
<tr><td>`async_file_handle`</td><td>More</td><td>Most (syscall + malloc/free + reactor)</td><td>POSIX guarantees (usually)</td><td>Makes no sense to use with cached i/o as it's a very expensive way to call `memcpy()`</td></tr>
<tr><td>`mapped_file_handle`</td><td>Most</td><td>Least</td><td>None</td><td>Cannot be used with uncached i/o</td></tr>
</table>
*/
class LLFIO_DECL file_handle : public lockable_io_handle, public fs_handle
{
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC const handle &_get_handle() const noexcept final { return *this; }
public:
using path_type = io_handle::path_type;
using extent_type = io_handle::extent_type;
using size_type = io_handle::size_type;
using mode = io_handle::mode;
using creation = io_handle::creation;
using caching = io_handle::caching;
using flag = io_handle::flag;
using buffer_type = io_handle::buffer_type;
using const_buffer_type = io_handle::const_buffer_type;
using buffers_type = io_handle::buffers_type;
using const_buffers_type = io_handle::const_buffers_type;
template <class T> using io_request = io_handle::io_request<T>;
template <class T> using io_result = io_handle::io_result<T>;
using dev_t = fs_handle::dev_t;
using ino_t = fs_handle::ino_t;
using path_view_type = fs_handle::path_view_type;
protected:
io_service *_service{nullptr};
public:
//! Default constructor
constexpr file_handle() {} // NOLINT
//! Construct a handle from a supplied native handle
constexpr file_handle(native_handle_type h, dev_t devid, ino_t inode, caching caching = caching::none, flag flags = flag::none)
: lockable_io_handle(std::move(h), caching, flags)
, fs_handle(devid, inode)
, _service(nullptr)
{
}
//! No copy construction (use clone())
file_handle(const file_handle &) = delete;
//! No copy assignment
file_handle &operator=(const file_handle &) = delete;
//! Implicit move construction of file_handle permitted
constexpr file_handle(file_handle &&o) noexcept
: lockable_io_handle(std::move(o))
, fs_handle(std::move(o))
, _service(o._service)
{
o._service = nullptr;
}
//! Explicit conversion from handle and io_handle permitted
explicit constexpr file_handle(handle &&o, dev_t devid, ino_t inode) noexcept
: lockable_io_handle(std::move(o))
, fs_handle(devid, inode)
, _service(nullptr)
{
}
//! Move assignment of file_handle permitted
file_handle &operator=(file_handle &&o) noexcept
{
this->~file_handle();
new(this) file_handle(std::move(o));
return *this;
}
//! Swap with another instance
LLFIO_MAKE_FREE_FUNCTION
void swap(file_handle &o) noexcept
{
file_handle temp(std::move(*this));
*this = std::move(o);
o = std::move(temp);
}
/*! Create a file handle opening access to a file on path
\param base Handle to a base location on the filing system. Pass `{}` to indicate that path will be absolute.
\param path The path relative to base to open.
\param _mode How to open the file.
\param _creation How to create the file.
\param _caching How to ask the kernel to cache the file.
\param flags Any additional custom behaviours.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<file_handle> file(const path_handle &base, path_view_type path, mode _mode = mode::read, creation _creation = creation::open_existing, caching _caching = caching::all, flag flags = flag::none) noexcept;
/*! Create a file handle creating a randomly named file on a path.
The file is opened exclusively with `creation::only_if_not_exist` so it
will never collide with nor overwrite any existing file. Note also
that caching defaults to temporary which hints to the OS to only
flush changes to physical storage as lately as possible.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
static inline result<file_handle> random_file(const path_handle &dirpath, mode _mode = mode::write, caching _caching = caching::temporary, flag flags = flag::none) noexcept
{
try
{
for(;;)
{
auto randomname = utils::random_string(32);
randomname.append(".random");
result<file_handle> ret = file(dirpath, randomname, _mode, creation::only_if_not_exist, _caching, flags);
if(ret || (!ret && ret.error() != errc::file_exists))
{
return ret;
}
}
}
catch(...)
{
return error_from_exception();
}
}
/*! Create a file handle creating the named file on some path which
the OS declares to be suitable for temporary files. Most OSs are
very lazy about flushing changes made to these temporary files.
Note the default flags are to have the newly created file deleted
on first handle close.
Note also that an empty name is equivalent to calling
`random_file(path_discovery::storage_backed_temporary_files_directory())` and the creation
parameter is ignored.
\note If the temporary file you are creating is not going to have its
path sent to another process for usage, this is the WRONG function
to use. Use `temp_inode()` instead, it is far more secure.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
static inline result<file_handle> temp_file(path_view_type name = path_view_type(), mode _mode = mode::write, creation _creation = creation::if_needed, caching _caching = caching::temporary, flag flags = flag::unlink_on_first_close) noexcept
{
auto &tempdirh = path_discovery::storage_backed_temporary_files_directory();
return name.empty() ? random_file(tempdirh, _mode, _caching, flags) : file(tempdirh, name, _mode, _creation, _caching, flags);
}
/*! \em Securely create a file handle creating a temporary anonymous inode in
the filesystem referred to by \em dirpath. The inode created has
no name nor accessible path on the filing system and ceases to
exist as soon as the last handle is closed, making it ideal for use as
a temporary file where other processes do not need to have access
to its contents via some path on the filing system (a classic use case
is for backing shared memory maps).
\errors Any of the values POSIX open() or CreateFile() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<file_handle> temp_inode(const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), mode _mode = mode::write, flag flags = flag::none) noexcept;
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC ~file_handle() override
{
if(_v)
{
(void) file_handle::close();
}
}
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> close() noexcept override
{
LLFIO_LOG_FUNCTION_CALL(this);
if(_flags & flag::unlink_on_first_close)
{
auto ret = unlink();
if(!ret)
{
// File may have already been deleted, if so ignore
if(ret.error() != errc::no_such_file_or_directory)
{
return std::move(ret).error();
}
}
}
#ifndef NDEBUG
if(_v)
{
// Tell handle::close() that we have correctly executed
_v.behaviour |= native_handle_type::disposition::_child_close_executed;
}
#endif
return io_handle::close();
}
/*! Reopen this handle (copy constructor is disabled to avoid accidental copying),
optionally race free reopening the handle with different access or caching.
Microsoft Windows provides a syscall for cloning an existing handle but with new
access. On POSIX, if not changing the mode, we change caching via `fcntl()`, if
changing the mode we must loop calling `current_path()`,
trying to open the path returned and making sure it is the same inode.
\errors Any of the values POSIX dup() or DuplicateHandle() can return.
\mallocs On POSIX if changing the mode, we must loop calling `current_path()` and
trying to open the path returned. Thus many allocations may occur.
*/
result<file_handle> reopen(mode mode_ = mode::unchanged, caching caching_ = caching::unchanged, deadline d = std::chrono::seconds(30)) const noexcept;
LLFIO_DEADLINE_TRY_FOR_UNTIL(reopen)
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<handle> clone() const noexcept override
{
// For file handles, we need a deeper clone than duph()
OUTCOME_TRY(ret, reopen());
return handle(ret.release());
}
//! The i/o service this handle is attached to, if any
io_service *service() const noexcept { return _service; }
using io_handle::read;
//! Convenience initialiser list based overload for `read()`
LLFIO_MAKE_FREE_FUNCTION
io_result<size_type> read(extent_type offset, std::initializer_list<buffer_type> lst, deadline d = deadline()) noexcept
{
buffer_type *_reqs = reinterpret_cast<buffer_type *>(alloca(sizeof(buffer_type) * lst.size()));
memcpy(_reqs, lst.begin(), sizeof(buffer_type) * lst.size());
io_request<buffers_type> reqs(buffers_type(_reqs, lst.size()), offset);
auto ret = read(reqs, d);
if(ret)
{
return ret.bytes_transferred();
}
return std::move(ret).error();
}
LLFIO_DEADLINE_TRY_FOR_UNTIL(read)
/*! Return the current maximum permitted extent of the file.
\errors Any of the values POSIX fstat() or GetFileInformationByHandleEx() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<extent_type> maximum_extent() const noexcept;
/*! Resize the current maximum permitted extent of the file to the given extent, avoiding any
new allocation of physical storage where supported. Note that on extents based filing systems
this will succeed even if there is insufficient free space on the storage medium.
\return The bytes actually truncated to.
\param newsize The bytes to truncate the file to.
\errors Any of the values POSIX ftruncate() or SetFileInformationByHandle() can return.
*/
LLFIO_MAKE_FREE_FUNCTION
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<extent_type> truncate(extent_type newsize) noexcept;
/*! \brief Returns a list of currently valid extents for this open file. WARNING: racy!
\return A vector of pairs of extent offset + extent length representing the valid extents
in this file. Filing systems which do not support extents return a single extent matching
the length of the file rather than returning an error.
*/
LLFIO_MAKE_FREE_FUNCTION
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<std::vector<std::pair<extent_type, extent_type>>> extents() const noexcept;
/*! \brief Efficiently zero, and possibly deallocate, data on storage.
On most major operating systems and with recent filing systems which are "extents based", one can
deallocate the physical storage of a file, causing the space deallocated to appear all bits zero.
This call attempts to deallocate whole pages (usually 4Kb) entirely, and memset's any excess to all
bits zero. This call works on most Linux filing systems with a recent kernel, Microsoft Windows
with NTFS, and FreeBSD with ZFS. On other systems it simply writes zeros.
\return The bytes zeroed.
\param offset The offset to start zeroing from.
\param bytes The number of bytes to zero.
\param d An optional deadline by which the i/o must complete, else it is cancelled.
Note function may return significantly after this deadline if the i/o takes long to cancel.
\errors Any of the values POSIX write() can return, `errc::timed_out`, `errc::operation_canceled`.
`errc::not_supported` may be returned if deadline i/o is not possible with this particular
handle configuration (e.g. writing to regular files on POSIX or writing to a non-overlapped
HANDLE on Windows).
\mallocs The default synchronous implementation in file_handle performs no memory allocation.
The asynchronous implementation in async_file_handle may perform one calloc and one free.
*/
LLFIO_MAKE_FREE_FUNCTION
LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<extent_type> zero(extent_type offset, extent_type bytes, deadline d = deadline()) noexcept;
LLFIO_DEADLINE_TRY_FOR_UNTIL(zero)
};
//! \brief Constructor for `file_handle`
template <> struct construct<file_handle>
{
const path_handle &base;
file_handle::path_view_type _path;
file_handle::mode _mode = file_handle::mode::read;
file_handle::creation _creation = file_handle::creation::open_existing;
file_handle::caching _caching = file_handle::caching::all;
file_handle::flag flags = file_handle::flag::none;
result<file_handle> operator()() const noexcept { return file_handle::file(base, _path, _mode, _creation, _caching, flags); }
};
// BEGIN make_free_functions.py
//! Swap with another instance
inline void swap(file_handle &self, file_handle &o) noexcept
{
return self.swap(std::forward<decltype(o)>(o));
}
/*! Create a file handle opening access to a file on path
\param base Handle to a base location on the filing system. Pass `{}` to indicate that path will be absolute.
\param path The path relative to base to open.
\param _mode How to open the file.
\param _creation How to create the file.
\param _caching How to ask the kernel to cache the file.
\param flags Any additional custom behaviours.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
inline result<file_handle> file(const path_handle &base, file_handle::path_view_type path, file_handle::mode _mode = file_handle::mode::read, file_handle::creation _creation = file_handle::creation::open_existing, file_handle::caching _caching = file_handle::caching::all,
file_handle::flag flags = file_handle::flag::none) noexcept
{
return file_handle::file(std::forward<decltype(base)>(base), std::forward<decltype(path)>(path), std::forward<decltype(_mode)>(_mode), std::forward<decltype(_creation)>(_creation), std::forward<decltype(_caching)>(_caching), std::forward<decltype(flags)>(flags));
}
/*! Create a file handle creating a randomly named file on a path.
The file is opened exclusively with `creation::only_if_not_exist` so it
will never collide with nor overwrite any existing file. Note also
that caching defaults to temporary which hints to the OS to only
flush changes to physical storage as lately as possible.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
inline result<file_handle> random_file(const path_handle &dirpath, file_handle::mode _mode = file_handle::mode::write, file_handle::caching _caching = file_handle::caching::temporary, file_handle::flag flags = file_handle::flag::none) noexcept
{
return file_handle::random_file(std::forward<decltype(dirpath)>(dirpath), std::forward<decltype(_mode)>(_mode), std::forward<decltype(_caching)>(_caching), std::forward<decltype(flags)>(flags));
}
/*! Create a file handle creating the named file on some path which
the OS declares to be suitable for temporary files. Most OSs are
very lazy about flushing changes made to these temporary files.
Note the default flags are to have the newly created file deleted
on first handle close.
Note also that an empty name is equivalent to calling
`random_file(path_discovery::storage_backed_temporary_files_directory())` and the creation
parameter is ignored.
\note If the temporary file you are creating is not going to have its
path sent to another process for usage, this is the WRONG function
to use. Use `temp_inode()` instead, it is far more secure.
\errors Any of the values POSIX open() or CreateFile() can return.
*/
inline result<file_handle> temp_file(file_handle::path_view_type name = file_handle::path_view_type(), file_handle::mode _mode = file_handle::mode::write, file_handle::creation _creation = file_handle::creation::if_needed, file_handle::caching _caching = file_handle::caching::temporary,
file_handle::flag flags = file_handle::flag::unlink_on_first_close) noexcept
{
return file_handle::temp_file(std::forward<decltype(name)>(name), std::forward<decltype(_mode)>(_mode), std::forward<decltype(_creation)>(_creation), std::forward<decltype(_caching)>(_caching), std::forward<decltype(flags)>(flags));
}
/*! \em Securely create a file handle creating a temporary anonymous inode in
the filesystem referred to by \em dirpath. The inode created has
no name nor accessible path on the filing system and ceases to
exist as soon as the last handle is closed, making it ideal for use as
a temporary file where other processes do not need to have access
to its contents via some path on the filing system (a classic use case
is for backing shared memory maps).
\errors Any of the values POSIX open() or CreateFile() can return.
*/
inline result<file_handle> temp_inode(const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), file_handle::mode _mode = file_handle::mode::write, file_handle::flag flags = file_handle::flag::none) noexcept
{
return file_handle::temp_inode(std::forward<decltype(dirh)>(dirh), std::forward<decltype(_mode)>(_mode), std::forward<decltype(flags)>(flags));
}
//! \overload
inline file_handle::io_result<file_handle::size_type> read(file_handle &self, file_handle::extent_type offset, std::initializer_list<file_handle::buffer_type> lst, deadline d = deadline()) noexcept
{
return self.read(std::forward<decltype(offset)>(offset), std::forward<decltype(lst)>(lst), std::forward<decltype(d)>(d));
}
/*! Return the current maximum permitted extent of the file.
\errors Any of the values POSIX fstat() or GetFileInformationByHandleEx() can return.
*/
inline result<file_handle::extent_type> maximum_extent(const file_handle &self) noexcept
{
return self.maximum_extent();
}
/*! Resize the current maximum permitted extent of the file to the given extent, avoiding any
new allocation of physical storage where supported. Note that on extents based filing systems
this will succeed even if there is insufficient free space on the storage medium.
\return The bytes actually truncated to.
\param self The object whose member function to call.
\param newsize The bytes to truncate the file to.
\errors Any of the values POSIX ftruncate() or SetFileInformationByHandle() can return.
*/
inline result<file_handle::extent_type> truncate(file_handle &self, file_handle::extent_type newsize) noexcept
{
return self.truncate(std::forward<decltype(newsize)>(newsize));
}
/*! \brief Returns a list of currently valid extents for this open file. WARNING: racy!
*/
inline result<std::vector<std::pair<file_handle::extent_type, file_handle::extent_type>>> extents(const file_handle &self) noexcept
{
return self.extents();
}
/*! \brief Efficiently zero, and possibly deallocate, data on storage.
On most major operating systems and with recent filing systems which are "extents based", one can
deallocate the physical storage of a file, causing the space deallocated to appear all bits zero.
This call attempts to deallocate whole pages (usually 4Kb) entirely, and memset's any excess to all
bits zero. This call works on most Linux filing systems with a recent kernel, Microsoft Windows
with NTFS, and FreeBSD with ZFS. On other systems it simply writes zeros.
\return The bytes zeroed.
\param self The object whose member function to call.
\param offset The offset to start zeroing from.
\param bytes The number of bytes to zero.
\param d An optional deadline by which the i/o must complete, else it is cancelled.
Note function may return significantly after this deadline if the i/o takes long to cancel.
\errors Any of the values POSIX write() can return, `errc::timed_out`, `errc::operation_canceled`. `errc::not_supported` may be
returned if deadline i/o is not possible with this particular handle configuration (e.g.
writing to regular files on POSIX or writing to a non-overlapped HANDLE on Windows).
\mallocs The default synchronous implementation in file_handle performs no memory allocation.
The asynchronous implementation in async_file_handle may perform one calloc and one free.
*/
inline result<file_handle::extent_type> zero(file_handle &self, file_handle::extent_type offset, file_handle::extent_type bytes, deadline d = deadline()) noexcept
{
return self.zero(std::forward<decltype(offset)>(offset), std::forward<decltype(bytes)>(bytes), std::forward<decltype(d)>(d));
}
// END make_free_functions.py
LLFIO_V2_NAMESPACE_END
#if LLFIO_HEADERS_ONLY == 1 && !defined(DOXYGEN_SHOULD_SKIP_THIS)
#define LLFIO_INCLUDED_BY_HEADER 1
#ifdef _WIN32
#include "detail/impl/windows/file_handle.ipp"
#else
#include "detail/impl/posix/file_handle.ipp"
#endif
#undef LLFIO_INCLUDED_BY_HEADER
#endif
// Needs to be here as path discovery uses file_handle
#if LLFIO_HEADERS_ONLY == 1 && !defined(DOXYGEN_SHOULD_SKIP_THIS)
#define LLFIO_INCLUDED_BY_HEADER 1
#include "detail/impl/path_discovery.ipp"
#undef LLFIO_INCLUDED_BY_HEADER
#endif
#ifdef _MSC_VER
#pragma warning(pop)
#endif
#endif
|