diff options
-rw-r--r-- | cmake/tests.cmake | 1 | ||||
-rw-r--r-- | include/llfio/revision.hpp | 6 | ||||
-rw-r--r-- | include/llfio/v2.0/detail/impl/posix/map_handle.ipp | 2 | ||||
-rw-r--r-- | include/llfio/v2.0/detail/impl/posix/utils.ipp | 206 | ||||
-rw-r--r-- | include/llfio/v2.0/detail/impl/windows/import.hpp | 106 | ||||
-rw-r--r-- | include/llfio/v2.0/detail/impl/windows/map_handle.ipp | 26 | ||||
-rw-r--r-- | include/llfio/v2.0/detail/impl/windows/utils.ipp | 47 | ||||
-rw-r--r-- | include/llfio/v2.0/map_handle.hpp | 2 | ||||
-rw-r--r-- | include/llfio/v2.0/utils.hpp | 29 |
9 files changed, 395 insertions, 30 deletions
diff --git a/cmake/tests.cmake b/cmake/tests.cmake index f8310cb7..c25ee384 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -29,6 +29,7 @@ set(llfio_TESTS "test/tests/symlink_handle_create_close/kernel_symlink_handle.cpp.hpp" "test/tests/symlink_handle_create_close/runner.cpp" "test/tests/trivial_vector.cpp" + "test/tests/utils.cpp" ) # DO NOT EDIT, GENERATED BY SCRIPT set(llfio_COMPILE_TESTS diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp index 5835ef7a..a5fb1ef3 100644 --- a/include/llfio/revision.hpp +++ b/include/llfio/revision.hpp @@ -1,4 +1,4 @@ // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time -#define LLFIO_PREVIOUS_COMMIT_REF 75cc45b6d26b3f96075e4b5e54b26be4c7eb6f51 -#define LLFIO_PREVIOUS_COMMIT_DATE "2020-01-14 14:00:36 +00:00" -#define LLFIO_PREVIOUS_COMMIT_UNIQUE 75cc45b6 +#define LLFIO_PREVIOUS_COMMIT_REF 03da2db2ef9c9316dbdf541ee29ff0a5a9993da9 +#define LLFIO_PREVIOUS_COMMIT_DATE "2020-01-15 14:57:43 +00:00" +#define LLFIO_PREVIOUS_COMMIT_UNIQUE 03da2db2 diff --git a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp index 0cdaf9b5..7d930dfe 100644 --- a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp @@ -129,7 +129,7 @@ result<section_handle::extent_type> section_handle::truncate(extent_type newsize map_handle::~map_handle() { - if(_v) + if(_addr != nullptr) { // Unmap the view auto ret = map_handle::close(); diff --git a/include/llfio/v2.0/detail/impl/posix/utils.ipp b/include/llfio/v2.0/detail/impl/posix/utils.ipp index 8c033899..79f5bd92 100644 --- a/include/llfio/v2.0/detail/impl/posix/utils.ipp +++ b/include/llfio/v2.0/detail/impl/posix/utils.ipp @@ -30,6 +30,10 @@ Distributed under the Boost Software License, Version 1.0. #include <sys/mman.h> +#ifdef __linux__ +#include <unistd.h> // for preadv +#endif + LLFIO_V2_NAMESPACE_BEGIN namespace utils @@ -79,7 +83,7 @@ namespace utils if(-1 != ih) { char buffer[4096], *hugepagesize, *hugepages; - buffer[ ::read(ih, buffer, sizeof(buffer) - 1)] = 0; + buffer[::read(ih, buffer, sizeof(buffer) - 1)] = 0; ::close(ih); hugepagesize = strstr(buffer, "Hugepagesize:"); hugepages = strstr(buffer, "HugePages_Total:"); @@ -202,6 +206,206 @@ namespace utils return false; } + result<process_memory_usage> current_process_memory_usage() noexcept + { +#ifdef __linux__ + try + { + /* /proc/[pid]/status: + + total_address_space_in_use = VmSize + total_address_space_paged_in = VmRSS + private_committed = ??? MISSING + private_paged_in = RssAnon + + /proc/[pid]/smaps: + + total_address_space_in_use = Sum of Size + total_address_space_paged_in = Sum of Rss + private_committed = Sum of Size for all entries with VmFlags containing ac, and inode = 0? + private_paged_in = (Sum of Anonymous - Sum of LazyFree) for all entries with VmFlags containing ac, and inode = 0? + */ + std::vector<char> buffer(65536); + for(;;) + { + int ih = ::open("/proc/self/smaps", O_RDONLY); + if(ih == -1) + { + return posix_error(); + } + size_t totalbytesread = 0; + for(;;) + { + auto bytesread = ::read(ih, buffer.data() + totalbytesread, buffer.size() - totalbytesread); + if(bytesread < 0) + { + ::close(ih); + return posix_error(); + } + if(bytesread == 0) + { + break; + } + totalbytesread += bytesread; + } + ::close(ih); + if(totalbytesread < buffer.size()) + { + buffer.resize(totalbytesread); + break; + } + buffer.resize(buffer.size() * 2); + } + const string_view totalview(buffer.data(), buffer.size()); + //std::cerr << totalview << std::endl; + std::vector<string_view> anon_entries, non_anon_entries; + anon_entries.reserve(32); + non_anon_entries.reserve(32); + auto sizeidx = totalview.find("\nSize:"); + while(sizeidx < totalview.size()) + { + auto itemtopidx = totalview.rfind("\n", sizeidx - 1); + if(string_view::npos == itemtopidx) + { + itemtopidx = 0; + } + // hexaddr-hexaddr flags offset dev:id inode [path] + size_t begin, end, offset, inode = 1; + char f1, f2, f3, f4, f5, f6, f7, f8; + sscanf(totalview.data() + itemtopidx, "%zx-%zx %c%c%c%c %zx %c%c:%c%c %zu", &begin, &end, &f1, &f2, &f3, &f4, &offset, &f5, &f6, &f7, &f8, &inode); + sizeidx = totalview.find("\nSize:", sizeidx + 1); + if(string_view::npos == sizeidx) + { + sizeidx = totalview.size(); + } + auto itemendidx = totalview.rfind("\n", sizeidx - 1); + if(string_view::npos == itemendidx) + { + abort(); + } + const string_view item(totalview.substr(itemtopidx + 1, itemendidx - itemtopidx - 1)); + auto vmflagsidx = item.rfind("\n"); + if(string_view::npos == itemendidx) + { + abort(); + } + if(0 != memcmp(item.data() + vmflagsidx, "\nVmFlags:", 9)) + { + return errc::illegal_byte_sequence; + } + // Is there " ac" after vmflagsidx? + if(string_view::npos != item.find(" ac", vmflagsidx) && inode == 0) + { + //std::cerr << "Adding anon entry at offset " << itemtopidx << std::endl; + anon_entries.push_back(item); + } + else + { + //std::cerr << "Adding non-anon entry at offset " << itemtopidx << std::endl; + non_anon_entries.push_back(item); + } + } + auto parse = [](string_view item, string_view what) ->result<uint64_t> { auto idx = item.find(what); + if(string_view::npos == idx) + { + return (uint64_t) -1; + } + idx += what.size(); + for(; item[idx] == ' '; idx++) + ; + auto eidx = idx; + for(; item[eidx] != '\n'; eidx++) + ; + string_view unit(item.substr(eidx - 2, 2)); + uint64_t value = atoll(item.data() + idx); + if(unit == "kB") + { + value *= 1024ULL; + } + else if(unit == "mB") + { + value *= 1024ULL * 1024; + } + else if(unit == "gB") + { + value *= 1024ULL * 1024 * 1024; + } + else if(unit == "tB") + { + value *= 1024ULL * 1024 * 1024 * 1024; + } + else if(unit == "pB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "eB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "zB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "yB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; + } + else + { + return errc::illegal_byte_sequence; + } + return value; + }; + process_memory_usage ret; + //std::cerr << "Anon entries:"; + for(auto &i : anon_entries) + { + OUTCOME_TRY(size, parse(i, "\nSize:")); + OUTCOME_TRY(rss, parse(i, "\nRss:")); + OUTCOME_TRY(anonymous, parse(i, "\nAnonymous:")); + OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:")); + if(size != (uint64_t) -1 && rss != (uint64_t) -1 && anonymous != (uint64_t) -1) + { + ret.total_address_space_in_use += size; + ret.total_address_space_paged_in += rss; + ret.private_committed += size; + ret.private_paged_in += anonymous; + if(lazyfree != (uint64_t) -1) + { + ret.total_address_space_paged_in -= lazyfree; + ret.private_paged_in -= lazyfree; + } + } + //std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; + } + //std::cerr << "\n\nNon-anon entries:"; + for(auto &i : non_anon_entries) + { + OUTCOME_TRY(size, parse(i, "\nSize:")); + OUTCOME_TRY(rss, parse(i, "\nRss:")); + OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:")); + if(size != (uint64_t) -1 && rss != (uint64_t) -1) + { + ret.total_address_space_in_use += size; + ret.total_address_space_paged_in += rss; + if(lazyfree != (uint64_t) -1) + { + ret.total_address_space_paged_in -= lazyfree; + } + } + //std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; + } + return ret; + } + catch(...) + { + return error_from_exception(); + } +#else +#error Unknown platform +#endif + } + namespace detail { large_page_allocation allocate_large_pages(size_t bytes) diff --git a/include/llfio/v2.0/detail/impl/windows/import.hpp b/include/llfio/v2.0/detail/impl/windows/import.hpp index 6eba31bb..d6cb4672 100644 --- a/include/llfio/v2.0/detail/impl/windows/import.hpp +++ b/include/llfio/v2.0/detail/impl/windows/import.hpp @@ -356,6 +356,103 @@ namespace windows_nt_kernel using NtFreeVirtualMemory_t = NTSTATUS(NTAPI *)(_In_ HANDLE ProcessHandle, _Inout_ PVOID *BaseAddress, _Inout_ PSIZE_T RegionSize, _In_ ULONG FreeType); + typedef struct _VM_COUNTERS_EX2 + { + SIZE_T PeakVirtualSize; + SIZE_T VirtualSize; + ULONG PageFaultCount; + SIZE_T PeakWorkingSetSize; + SIZE_T WorkingSetSize; + SIZE_T QuotaPeakPagedPoolUsage; + SIZE_T QuotaPagedPoolUsage; + SIZE_T QuotaPeakNonPagedPoolUsage; + SIZE_T QuotaNonPagedPoolUsage; + SIZE_T PagefileUsage; + SIZE_T PeakPagefileUsage; + SIZE_T PrivateUsage; + SIZE_T PrivateWorkingSetSize; + SIZE_T SharedCommitUsage; + } VM_COUNTERS_EX2, *PVM_COUNTERS_EX2; + + typedef enum _PROCESSINFOCLASS + { + ProcessBasicInformation, + ProcessQuotaLimits, + ProcessIoCounters, + ProcessVmCounters, + ProcessTimes, + ProcessBasePriority, + ProcessRaisePriority, + ProcessDebugPort, + ProcessExceptionPort, + ProcessAccessToken, + ProcessLdtInformation, + ProcessLdtSize, + ProcessDefaultHardErrorMode, + ProcessIoPortHandlers, + ProcessPooledUsageAndLimits, + ProcessWorkingSetWatch, + ProcessUserModeIOPL, + ProcessEnableAlignmentFaultFixup, + ProcessPriorityClass, + ProcessWx86Information, + ProcessHandleCount, + ProcessAffinityMask, + ProcessPriorityBoost, + ProcessDeviceMap, + ProcessSessionInformation, + ProcessForegroundInformation, + ProcessWow64Information, + ProcessImageFileName, + ProcessLUIDDeviceMapsEnabled, + ProcessBreakOnTermination, + ProcessDebugObjectHandle, + ProcessDebugFlags, + ProcessHandleTracing, + ProcessIoPriority, + ProcessExecuteFlags, + ProcessResourceManagement, + ProcessCookie, + ProcessImageInformation, + ProcessCycleTime, + ProcessPagePriority, + ProcessInstrumentationCallback, + ProcessThreadStackAllocation, + ProcessWorkingSetWatchEx, + ProcessImageFileNameWin32, + ProcessImageFileMapping, + ProcessAffinityUpdateMode, + ProcessMemoryAllocationMode, + ProcessGroupInformation, + ProcessTokenVirtualizationEnabled, + ProcessConsoleHostProcess, + ProcessWindowInformation, + ProcessHandleInformation, + ProcessMitigationPolicy, + ProcessDynamicFunctionTableInformation, + ProcessHandleCheckingMode, + ProcessKeepAliveCount, + ProcessRevokeFileHandles, + ProcessWorkingSetControl, + ProcessHandleTable, + ProcessCheckStackExtentsMode, + ProcessCommandLineInformation, + ProcessProtectionInformation, + ProcessMemoryExhaustion, + ProcessFaultInformation, + ProcessTelemetryIdInformation, + ProcessCommitReleaseInformation, + ProcessDefaultCpuSetsInformation, + ProcessAllowedCpuSetsInformation, + ProcessReserved1Information, + ProcessReserved2Information, + ProcessSubsystemProcess, + ProcessJobMemoryInformation, + MaxProcessInfoClass + } PROCESSINFOCLASS; + + using NtQueryInformationProcess_t = NTSTATUS(NTAPI *)(_In_ HANDLE ProcessHandle, _In_ PROCESSINFOCLASS ProcessInformationClass, _Out_ PVOID ProcessInformation, _In_ ULONG ProcessInformationLength, _Out_opt_ PULONG ReturnLength); + using RtlGenRandom_t = BOOLEAN(NTAPI *)(_Out_ PVOID RandomBuffer, _In_ ULONG RandomBufferLength); @@ -613,6 +710,7 @@ namespace windows_nt_kernel static NtSetSystemInformation_t NtSetSystemInformation; static NtAllocateVirtualMemory_t NtAllocateVirtualMemory; static NtFreeVirtualMemory_t NtFreeVirtualMemory; + static NtQueryInformationProcess_t NtQueryInformationProcess; static RtlGenRandom_t RtlGenRandom; static OpenProcessToken_t OpenProcessToken; static LookupPrivilegeValue_t LookupPrivilegeValue; @@ -830,6 +928,14 @@ namespace windows_nt_kernel abort(); } } + if(NtQueryInformationProcess == nullptr) + { + if((NtQueryInformationProcess = reinterpret_cast<NtQueryInformationProcess_t>(GetProcAddress(ntdllh, "NtQueryInformationProcess"))) == nullptr) + { + abort(); + } + } + HMODULE advapi32 = LoadLibraryA("ADVAPI32.DLL"); if(RtlGenRandom == nullptr) diff --git a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp index 00efc7d1..f18e4f52 100644 --- a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp @@ -447,7 +447,7 @@ static inline result<void> win32_release_allocations(byte *addr, size_t bytes, U map_handle::~map_handle() { - if(_v) + if(_addr != nullptr) { // Unmap the view auto ret = map_handle::close(); @@ -575,17 +575,13 @@ result<map_handle> map_handle::map(size_type bytes, bool /*unused*/, section_han if(_flag & section_handle::flag::prefault) { using namespace windows_nt_kernel; - // Start an asynchronous prefetch + // Start an asynchronous prefetch, so it might fault the whole lot in at once buffer_type b{static_cast<byte *>(addr), bytes}; (void) prefetch(span<buffer_type>(&b, 1)); - // If this kernel doesn't support that API, manually poke every page in the new map - if(PrefetchVirtualMemory_ == nullptr) + volatile auto *a = static_cast<volatile char *>(addr); + for(size_t n = 0; n < bytes; n += pagesize) { - volatile auto *a = static_cast<volatile char *>(addr); - for(size_t n = 0; n < bytes; n += pagesize) - { - a[n]; - } + a[n]; } } return ret; @@ -622,17 +618,13 @@ result<map_handle> map_handle::map(section_handle §ion, size_type bytes, ext // Windows has no way of getting the kernel to prefault maps on creation, so ... if(ret.value()._flag & section_handle::flag::prefault) { - // Start an asynchronous prefetch + // Start an asynchronous prefetch, so it might fault the whole lot in at once buffer_type b{static_cast<byte *>(addr), _bytes}; (void) prefetch(span<buffer_type>(&b, 1)); - // If this kernel doesn't support that API, manually poke every page in the new map - if(PrefetchVirtualMemory_ == nullptr) + volatile auto *a = static_cast<volatile char *>(addr); + for(size_t n = 0; n < _bytes; n += pagesize) { - volatile auto *a = static_cast<volatile char *>(addr); - for(size_t n = 0; n < _bytes; n += pagesize) - { - a[n]; - } + a[n]; } } return ret; diff --git a/include/llfio/v2.0/detail/impl/windows/utils.ipp b/include/llfio/v2.0/detail/impl/windows/utils.ipp index 2defa87d..6d384a8a 100644 --- a/include/llfio/v2.0/detail/impl/windows/utils.ipp +++ b/include/llfio/v2.0/detail/impl/windows/utils.ipp @@ -24,8 +24,8 @@ Distributed under the Boost Software License, Version 1.0. #include "../../../utils.hpp" -#include "quickcpplib/spinlock.hpp" #include "import.hpp" +#include "quickcpplib/spinlock.hpp" LLFIO_V2_NAMESPACE_BEGIN @@ -141,7 +141,16 @@ namespace utils } prived = true; } - typedef enum _SYSTEM_MEMORY_LIST_COMMAND { MemoryCaptureAccessedBits, MemoryCaptureAndResetAccessedBits, MemoryEmptyWorkingSets, MemoryFlushModifiedList, MemoryPurgeStandbyList, MemoryPurgeLowPriorityStandbyList, MemoryCommandMax } SYSTEM_MEMORY_LIST_COMMAND; // NOLINT + typedef enum _SYSTEM_MEMORY_LIST_COMMAND + { + MemoryCaptureAccessedBits, + MemoryCaptureAndResetAccessedBits, + MemoryEmptyWorkingSets, + MemoryFlushModifiedList, + MemoryPurgeStandbyList, + MemoryPurgeLowPriorityStandbyList, + MemoryCommandMax + } SYSTEM_MEMORY_LIST_COMMAND; // NOLINT // Write all modified pages to storage SYSTEM_MEMORY_LIST_COMMAND command = MemoryPurgeStandbyList; @@ -197,6 +206,40 @@ namespace utils return success(); } + result<process_memory_usage> current_process_memory_usage() noexcept { + // Amazingly Win32 doesn't expose private working set, so to avoid having + // to iterate all the pages in the process and calculate, use a hidden + // NT kernel call + windows_nt_kernel::init(); + using namespace windows_nt_kernel; + ULONG written = 0; + _VM_COUNTERS_EX2 vmc; + memset(&vmc, 0, sizeof(vmc)); + NTSTATUS ntstat = NtQueryInformationProcess(GetCurrentProcess(), ProcessVmCounters, &vmc, sizeof(vmc), &written); + if(ntstat < 0) + { + return ntkernel_error(ntstat); + } + process_memory_usage ret; + /* Notes: + + Apparently PrivateUsage is the commit charge on Windows. It always equals PagefileUsage. + It is the total amount of private anonymous pages committed. + SharedCommitUsage is amount of non-binary Shared memory committed. + Therefore total non-binary commit = PrivateUsage + SharedCommitUsage + + WorkingSetSize is the total amount of program binaries, non-binary shared memory, and anonymous pages faulted in. + PrivateWorkingSetSize is the amount of private anonymous pages faulted into the process. + Therefore remainder is all shared working set faulted into the process. + */ + ret.total_address_space_in_use = vmc.VirtualSize; + ret.total_address_space_paged_in = vmc.WorkingSetSize; + + ret.private_committed = vmc.PrivateUsage; + ret.private_paged_in = vmc.PrivateWorkingSetSize; + return ret; + } + namespace detail { large_page_allocation allocate_large_pages(size_t bytes) diff --git a/include/llfio/v2.0/map_handle.hpp b/include/llfio/v2.0/map_handle.hpp index 08ddc7e7..f56ad783 100644 --- a/include/llfio/v2.0/map_handle.hpp +++ b/include/llfio/v2.0/map_handle.hpp @@ -574,7 +574,7 @@ public: system entirely, including the extents for them in any backing storage. On newer Linux kernels the kernel can additionally swap whole 4Kb pages for freshly zeroed ones making this a very efficient way of zeroing large ranges of memory. - On Windows, this call currently only works for non-backed memory due to lacking kernel support. + On Windows, this call currently only has an effect for non-backed memory due to lacking kernel support. \errors Any of the errors returnable by madvise() or DiscardVirtualMemory or the zero() function. */ diff --git a/include/llfio/v2.0/utils.hpp b/include/llfio/v2.0/utils.hpp index 370a03a9..dbd31389 100644 --- a/include/llfio/v2.0/utils.hpp +++ b/include/llfio/v2.0/utils.hpp @@ -1,5 +1,5 @@ /* Misc utilities -(C) 2015-2017 Niall Douglas <http://www.nedproductions.biz/> (8 commits) +(C) 2015-2020 Niall Douglas <http://www.nedproductions.biz/> (8 commits) File Created: Dec 2015 @@ -47,7 +47,7 @@ namespace utils LLFIO_HEADERS_ONLY_FUNC_SPEC size_t page_size() noexcept; /*! \brief Round a value to its next lowest page size multiple - */ + */ template <class T> inline T round_down_to_page_size(T i, size_t pagesize) noexcept { assert(pagesize > 0); @@ -55,7 +55,7 @@ namespace utils return i; } /*! \brief Round a value to its next highest page size multiple - */ + */ template <class T> inline T round_up_to_page_size(T i, size_t pagesize) noexcept { assert(pagesize > 0); @@ -141,7 +141,7 @@ namespace utils } /*! \brief Tries to flush all modified data to the physical device. - */ + */ LLFIO_HEADERS_ONLY_FUNC_SPEC result<void> flush_modified_data() noexcept; /*! \brief Tries to flush all modified data to the physical device, and then drop the OS filesystem cache, @@ -155,10 +155,28 @@ namespace utils #ifndef _WIN32 /*! \brief Returns true if this POSIX is running under Microsoft's Subsystem for Linux. - */ + */ LLFIO_HEADERS_ONLY_FUNC_SPEC bool running_under_wsl() noexcept; #endif + /*! \brief Memory usage statistics for a process. + */ + struct process_memory_usage + { + //! The total virtual address space in use. + size_t total_address_space_in_use{0}; + //! The total memory currently paged into the process. Always `<= total_address_space_in_use`. Also known as "working set", or "resident set size including shared". + size_t total_address_space_paged_in{0}; + + //! The total anonymous memory committed. Also known as "commit charge". + size_t private_committed{0}; + //! The total anonymous memory currently paged into the process. Always `<= private_committed`. Also known as "active anonymous pages". + size_t private_paged_in{0}; + }; + /*! \brief Retrieve the current memory usage statistics for this process. + */ + LLFIO_HEADERS_ONLY_FUNC_SPEC result<process_memory_usage> current_process_memory_usage() noexcept; + namespace detail { struct large_page_allocation @@ -189,6 +207,7 @@ namespace utils LLFIO_HEADERS_ONLY_FUNC_SPEC large_page_allocation allocate_large_pages(size_t bytes); LLFIO_HEADERS_ONLY_FUNC_SPEC void deallocate_large_pages(void *p, size_t bytes); } // namespace detail + /*! \class page_allocator \brief An STL allocator which allocates large TLB page memory. \ingroup utils |