From b5cd957f42d33f5960014dddbca7414502638a2d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 20 Dec 2023 18:05:29 -0600 Subject: Extended lfs_fs_gc to compact metadata, compact_thresh This extends lfs_fs_gc to now handle three things: 1. Calls mkconsistent if not already consistent 2. Compacts metadata > compact_thresh 3. Populates the block allocator Which should be all of the janitorial work that can be done without additional on-disk data structures. Normally, metadata compaction occurs when an mdir is full, and results in mdirs that are at most block_size/2. Now, if you call lfs_fs_gc, littlefs will eagerly compact any mdirs that exceed the compact_thresh configuration option. Because the resulting mdirs are at most block_size/2, it only makes sense for compact_thresh to be >= block_size/2 and <= block_size. Additionally, there are some special values: - compact_thresh=0 => defaults to ~88% block_size, may change - compact_thresh=-1 => disables metadata compaction during lfs_fs_gc Note that compact_thresh only affects lfs_fs_gc. Normal compactions still only occur when full. --- runners/test_runner.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'runners/test_runner.c') diff --git a/runners/test_runner.c b/runners/test_runner.c index 13befdc..c6e933e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1346,6 +1346,7 @@ static void run_powerloss_none( .block_cycles = BLOCK_CYCLES, .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, + .compact_thresh = COMPACT_THRESH, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1422,6 +1423,7 @@ static void run_powerloss_linear( .block_cycles = BLOCK_CYCLES, .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, + .compact_thresh = COMPACT_THRESH, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1515,6 +1517,7 @@ static void run_powerloss_log( .block_cycles = BLOCK_CYCLES, .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, + .compact_thresh = COMPACT_THRESH, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1606,6 +1609,7 @@ static void run_powerloss_cycles( .block_cycles = BLOCK_CYCLES, .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, + .compact_thresh = COMPACT_THRESH, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1795,6 +1799,7 @@ static void run_powerloss_exhaustive( .block_cycles = BLOCK_CYCLES, .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, + .compact_thresh = COMPACT_THRESH, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif -- cgit v1.2.3 From 8b8fd14187d8b798606f579c49fbd8a0b7d4355c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 20 Dec 2023 22:56:26 -0600 Subject: Added inline_max, to optionally limit the size of inlined files Inlined files live in metadata and decrease storage requirements, but may be limited to improve metadata-related performance. This is especially important given the current plague of metadata performance. Though decreasing inline_max may make metadata more dense and increase block usage, so it's important to benchmark if optimizing for speed. The underlying limits of inlined files haven't changed: 1. Inlined files need to fit in RAM, so <= cache_size 2. Inlined files need to fit in a single attr, so <= attr_max 3. Inlined files need to fit in 1/8 of a block to avoid metadata overflow issues, this is after limiting by metadata_max, so <= min(metadata_max, block_size)/8 By default, the largest possible inline_max is used. This preserves backwards compatibility and is probably a good default for most use cases. This does have the awkward effect of requiring inline_max=-1 to indicate disabled inlined files, but I don't think there's a good way around this. --- runners/test_runner.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'runners/test_runner.c') diff --git a/runners/test_runner.c b/runners/test_runner.c index c6e933e..ff52673 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1347,6 +1347,7 @@ static void run_powerloss_none( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1424,6 +1425,7 @@ static void run_powerloss_linear( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1518,6 +1520,7 @@ static void run_powerloss_log( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1610,6 +1613,7 @@ static void run_powerloss_cycles( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1800,6 +1804,7 @@ static void run_powerloss_exhaustive( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif -- cgit v1.2.3