diff options
author | Christopher Haster <geky@geky.net> | 2023-12-21 07:56:26 +0300 |
---|---|---|
committer | Christopher Haster <geky@geky.net> | 2024-01-19 22:00:27 +0300 |
commit | 8b8fd14187d8b798606f579c49fbd8a0b7d4355c (patch) | |
tree | dfe01c6d2355c4250dfe3041e588b27b7c54e9a0 | |
parent | 09972a1710a68f245ed2c8cd6f229748edfc1994 (diff) |
Added inline_max, to optionally limit the size of inlined filesinline-max
Inlined files live in metadata and decrease storage requirements, but
may be limited to improve metadata-related performance. This is
especially important given the current plague of metadata performance.
Though decreasing inline_max may make metadata more dense and increase
block usage, so it's important to benchmark if optimizing for speed.
The underlying limits of inlined files haven't changed:
1. Inlined files need to fit in RAM, so <= cache_size
2. Inlined files need to fit in a single attr, so <= attr_max
3. Inlined files need to fit in 1/8 of a block to avoid metadata
overflow issues, this is after limiting by metadata_max,
so <= min(metadata_max, block_size)/8
By default, the largest possible inline_max is used. This preserves
backwards compatibility and is probably a good default for most use
cases.
This does have the awkward effect of requiring inline_max=-1 to
indicate disabled inlined files, but I don't think there's a good
way around this.
-rw-r--r-- | lfs.c | 35 | ||||
-rw-r--r-- | lfs.h | 10 | ||||
-rw-r--r-- | runners/bench_runner.c | 1 | ||||
-rw-r--r-- | runners/bench_runner.h | 15 | ||||
-rw-r--r-- | runners/test_runner.c | 5 | ||||
-rw-r--r-- | runners/test_runner.h | 17 | ||||
-rw-r--r-- | tests/test_files.toml | 21 |
7 files changed, 79 insertions, 25 deletions
@@ -3524,11 +3524,7 @@ static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, lfs_size_t nsize = size; if ((file->flags & LFS_F_INLINE) && - lfs_max(file->pos+nsize, file->ctz.size) > - lfs_min(0x3fe, lfs_min( - lfs->cfg->cache_size, - (lfs->cfg->metadata_max ? - lfs->cfg->metadata_max : lfs->cfg->block_size) / 8))) { + lfs_max(file->pos+nsize, file->ctz.size) > lfs->inline_max) { // inline file doesn't fit anymore int err = lfs_file_outline(lfs, file); if (err) { @@ -3725,10 +3721,7 @@ static int lfs_file_rawtruncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { lfs_off_t oldsize = lfs_file_rawsize(lfs, file); if (size < oldsize) { // revert to inline file? - if (size <= lfs_min(0x3fe, lfs_min( - lfs->cfg->cache_size, - (lfs->cfg->metadata_max ? - lfs->cfg->metadata_max : lfs->cfg->block_size) / 8))) { + if (size <= lfs->inline_max) { // flush+seek to head lfs_soff_t res = lfs_file_rawseek(lfs, file, 0, LFS_SEEK_SET); if (res < 0) { @@ -4259,6 +4252,27 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(lfs->cfg->metadata_max <= lfs->cfg->block_size); + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= lfs->cfg->cache_size); + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= lfs->attr_max); + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= ((lfs->cfg->metadata_max) + ? lfs->cfg->metadata_max + : lfs->cfg->block_size)/8); + lfs->inline_max = lfs->cfg->inline_max; + if (lfs->inline_max == (lfs_size_t)-1) { + lfs->inline_max = 0; + } else if (lfs->inline_max == 0) { + lfs->inline_max = lfs_min( + lfs->cfg->cache_size, + lfs_min( + lfs->attr_max, + ((lfs->cfg->metadata_max) + ? lfs->cfg->metadata_max + : lfs->cfg->block_size)/8)); + } + // setup default state lfs->root[0] = LFS_BLOCK_NULL; lfs->root[1] = LFS_BLOCK_NULL; @@ -4482,6 +4496,9 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs->attr_max = superblock.attr_max; + + // we also need to update inline_max in case attr_max changed + lfs->inline_max = lfs_min(lfs->inline_max, lfs->attr_max); } // this is where we get the block_count from disk if block_count=0 @@ -272,6 +272,15 @@ struct lfs_config { // Defaults to block_size when zero. lfs_size_t metadata_max; + // Optional upper limit on inlined files in bytes. Inlined files live in + // metadata and decrease storage requirements, but may be limited to + // improve metadata-related performance. Must be <= cache_size, <= + // attr_max, and <= block_size/8. Defaults to the largest possible + // inline_max when zero. + // + // Set to -1 to disable inlined files. + lfs_size_t inline_max; + #ifdef LFS_MULTIVERSION // On-disk version to use when writing in the form of 16-bit major version // + 16-bit minor version. This limiting metadata to what is supported by @@ -451,6 +460,7 @@ typedef struct lfs { lfs_size_t name_max; lfs_size_t file_max; lfs_size_t attr_max; + lfs_size_t inline_max; #ifdef LFS_MIGRATE struct lfs1 *lfs1; diff --git a/runners/bench_runner.c b/runners/bench_runner.c index 58cae68..387889d 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -1322,6 +1322,7 @@ void perm_run( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, }; struct lfs_emubd_config bdcfg = { diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 986ac90..174733c 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -96,11 +96,12 @@ intmax_t bench_define(size_t define); #define CACHE_SIZE_i 6 #define LOOKAHEAD_SIZE_i 7 #define COMPACT_THRESH_i 8 -#define BLOCK_CYCLES_i 9 -#define ERASE_VALUE_i 10 -#define ERASE_CYCLES_i 11 -#define BADBLOCK_BEHAVIOR_i 12 -#define POWERLOSS_BEHAVIOR_i 13 +#define INLINE_MAX_i 9 +#define BLOCK_CYCLES_i 10 +#define ERASE_VALUE_i 11 +#define ERASE_CYCLES_i 12 +#define BADBLOCK_BEHAVIOR_i 13 +#define POWERLOSS_BEHAVIOR_i 14 #define READ_SIZE bench_define(READ_SIZE_i) #define PROG_SIZE bench_define(PROG_SIZE_i) @@ -111,6 +112,7 @@ intmax_t bench_define(size_t define); #define CACHE_SIZE bench_define(CACHE_SIZE_i) #define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i) #define COMPACT_THRESH bench_define(COMPACT_THRESH_i) +#define INLINE_MAX bench_define(INLINE_MAX_i) #define BLOCK_CYCLES bench_define(BLOCK_CYCLES_i) #define ERASE_VALUE bench_define(ERASE_VALUE_i) #define ERASE_CYCLES bench_define(ERASE_CYCLES_i) @@ -127,6 +129,7 @@ intmax_t bench_define(size_t define); BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ BENCH_DEF(LOOKAHEAD_SIZE, 16) \ BENCH_DEF(COMPACT_THRESH, 0) \ + BENCH_DEF(INLINE_MAX, 0) \ BENCH_DEF(BLOCK_CYCLES, -1) \ BENCH_DEF(ERASE_VALUE, 0xff) \ BENCH_DEF(ERASE_CYCLES, 0) \ @@ -134,7 +137,7 @@ intmax_t bench_define(size_t define); BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) #define BENCH_GEOMETRY_DEFINE_COUNT 4 -#define BENCH_IMPLICIT_DEFINE_COUNT 14 +#define BENCH_IMPLICIT_DEFINE_COUNT 15 #endif diff --git a/runners/test_runner.c b/runners/test_runner.c index c6e933e..ff52673 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1347,6 +1347,7 @@ static void run_powerloss_none( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1424,6 +1425,7 @@ static void run_powerloss_linear( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1518,6 +1520,7 @@ static void run_powerloss_log( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1610,6 +1613,7 @@ static void run_powerloss_cycles( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif @@ -1800,6 +1804,7 @@ static void run_powerloss_exhaustive( .cache_size = CACHE_SIZE, .lookahead_size = LOOKAHEAD_SIZE, .compact_thresh = COMPACT_THRESH, + .inline_max = INLINE_MAX, #ifdef LFS_MULTIVERSION .disk_version = DISK_VERSION, #endif diff --git a/runners/test_runner.h b/runners/test_runner.h index 9609999..0f0e594 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -89,12 +89,13 @@ intmax_t test_define(size_t define); #define CACHE_SIZE_i 6 #define LOOKAHEAD_SIZE_i 7 #define COMPACT_THRESH_i 8 -#define BLOCK_CYCLES_i 9 -#define ERASE_VALUE_i 10 -#define ERASE_CYCLES_i 11 -#define BADBLOCK_BEHAVIOR_i 12 -#define POWERLOSS_BEHAVIOR_i 13 -#define DISK_VERSION_i 14 +#define INLINE_MAX_i 9 +#define BLOCK_CYCLES_i 10 +#define ERASE_VALUE_i 11 +#define ERASE_CYCLES_i 12 +#define BADBLOCK_BEHAVIOR_i 13 +#define POWERLOSS_BEHAVIOR_i 14 +#define DISK_VERSION_i 15 #define READ_SIZE TEST_DEFINE(READ_SIZE_i) #define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) @@ -105,6 +106,7 @@ intmax_t test_define(size_t define); #define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) #define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i) #define COMPACT_THRESH TEST_DEFINE(COMPACT_THRESH_i) +#define INLINE_MAX TEST_DEFINE(INLINE_MAX_i) #define BLOCK_CYCLES TEST_DEFINE(BLOCK_CYCLES_i) #define ERASE_VALUE TEST_DEFINE(ERASE_VALUE_i) #define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i) @@ -122,6 +124,7 @@ intmax_t test_define(size_t define); TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ TEST_DEF(LOOKAHEAD_SIZE, 16) \ TEST_DEF(COMPACT_THRESH, 0) \ + TEST_DEF(INLINE_MAX, 0) \ TEST_DEF(BLOCK_CYCLES, -1) \ TEST_DEF(ERASE_VALUE, 0xff) \ TEST_DEF(ERASE_CYCLES, 0) \ @@ -130,7 +133,7 @@ intmax_t test_define(size_t define); TEST_DEF(DISK_VERSION, 0) #define TEST_GEOMETRY_DEFINE_COUNT 4 -#define TEST_IMPLICIT_DEFINE_COUNT 15 +#define TEST_IMPLICIT_DEFINE_COUNT 16 #endif diff --git a/tests/test_files.toml b/tests/test_files.toml index afb0811..1c86cd8 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -1,5 +1,6 @@ [cases.test_files_simple] +defines.INLINE_MAX = [0, -1, 8] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -25,6 +26,7 @@ code = ''' [cases.test_files_large] defines.SIZE = [32, 8192, 262144, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 33, 1, 1023] +defines.INLINE_MAX = [0, -1, 8] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -67,6 +69,7 @@ code = ''' defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] +defines.INLINE_MAX = [0, -1, 8] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -152,6 +155,7 @@ code = ''' defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] +defines.INLINE_MAX = [0, -1, 8] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -232,6 +236,7 @@ code = ''' defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] +defines.INLINE_MAX = [0, -1, 8] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -303,6 +308,7 @@ code = ''' [cases.test_files_reentrant_write] defines.SIZE = [32, 0, 7, 2049] defines.CHUNKSIZE = [31, 16, 65] +defines.INLINE_MAX = [0, -1, 8] reentrant = true code = ''' lfs_t lfs; @@ -354,11 +360,20 @@ code = ''' [cases.test_files_reentrant_write_sync] defines = [ # append (O(n)) - {MODE='LFS_O_APPEND', SIZE=[32, 0, 7, 2049], CHUNKSIZE=[31, 16, 65]}, + {MODE='LFS_O_APPEND', + SIZE=[32, 0, 7, 2049], + CHUNKSIZE=[31, 16, 65], + INLINE_MAX=[0, -1, 8]}, # truncate (O(n^2)) - {MODE='LFS_O_TRUNC', SIZE=[32, 0, 7, 200], CHUNKSIZE=[31, 16, 65]}, + {MODE='LFS_O_TRUNC', + SIZE=[32, 0, 7, 200], + CHUNKSIZE=[31, 16, 65], + INLINE_MAX=[0, -1, 8]}, # rewrite (O(n^2)) - {MODE=0, SIZE=[32, 0, 7, 200], CHUNKSIZE=[31, 16, 65]}, + {MODE=0, + SIZE=[32, 0, 7, 200], + CHUNKSIZE=[31, 16, 65], + INLINE_MAX=[0, -1, 8]}, ] reentrant = true code = ''' |