[FL-1682] Faster file receiving function. Storage management scripts. (#647)

* Storage-cli: the file write function is splitted into a function for working with text and function for raw data
* Storage-cli: read_chunks, renamed write_raw to write_chunk
* Storage-cli: fix typo
* SD Hal: fixed wrong read/write block address
* HAL-console: printf
* Storage benchmark: more popular sizes
* Toolbox: md5
* Storage-cli: better read_chunks function, md5 hash function
* Notification: fixed incorrect error message when loading settings
* Storage-cli: stat command
* Storage-cli: stat, "/" is also storage
* Scripts: add storage managment script
* Scripts, storage lib: send command with known response syntax instead of SOH
* Scripts: python3 from env
* Storage-cli: fixed mess with error texts
* Storage-cli: write, show welcome message only if we didn't have any errors
* Storage: poorly fixed folders copying
* Storage: close an old file if an error occurred while opening a new file
* Storage-cli: fixed storage info spacing
* Scripts: use positional arguments in storage.
* Scripts: explicit string encoding and decoding, documentation in comments.

Co-authored-by: あく <alleteam@gmail.com>

2 files changed, 271 insertions, 99 deletions

diff --git a/applications/storage/storage-cli.c b/applications/storage/storage-cli.c
index a67378a3..f5520e0e 100644
--- a/applications/storage/storage-cli.c
+++ b/applications/storage/storage-cli.c
@@ -1,6 +1,7 @@
 #include <furi.h>
 #include <cli/cli.h>
 #include <lib/toolbox/args.h>
+#include <lib/toolbox/md5.h>
 #include <storage/storage.h>
 #include <storage/storage-sd-api.h>
 #include <furi-hal-version.h>
@@ -25,32 +26,23 @@ void storage_cli_print_usage() {
     printf("\tformat\t - format filesystem\r\n");
     printf("\tlist\t - list files and dirs\r\n");
     printf("\tremove\t - delete the file or directory\r\n");
-    printf("\tread\t - read data from file and print file size and content to cli\r\n");
+    printf("\tread\t - read text from file and print file size and content to cli\r\n");
     printf(
-        "\twrite\t - read data from cli and append it to file, <args> should contain how many bytes you want to write\r\n");
+        "\tread_chunks\t - read data from file and print file size and content to cli, <args> should contain how many bytes you want to read in block\r\n");
+    printf("\twrite\t - read text from cli and append it to file, stops by ctrl+c\r\n");
+    printf(
+        "\twrite_chunk\t - read data from cli and append it to file, <args> should contain how many bytes you want to write\r\n");
     printf("\tcopy\t - copy file to new file, <args> must contain new path\r\n");
     printf("\trename\t - move file to new file, <args> must contain new path\r\n");
     printf("\tmkdir\t - creates a new directory\r\n");
+    printf("\tmd5\t - md5 hash of the file\r\n");
+    printf("\tstat\t - info about file or dir\r\n");
 };
 
 void storage_cli_print_error(FS_Error error) {
     printf("Storage error: %s\r\n", storage_error_get_desc(error));
 }
 
-void storage_cli_print_path_error(string_t path, FS_Error error) {
-    printf(
-        "Storage error for path \"%s\": %s\r\n",
-        string_get_cstr(path),
-        storage_error_get_desc(error));
-}
-
-void storage_cli_print_file_error(string_t path, File* file) {
-    printf(
-        "Storage error for path \"%s\": %s\r\n",
-        string_get_cstr(path),
-        storage_file_get_error_desc(file));
-}
-
 void storage_cli_info(Cli* cli, string_t path) {
     Storage* api = furi_record_open("storage");
 
@@ -60,10 +52,10 @@ void storage_cli_info(Cli* cli, string_t path) {
         FS_Error error = storage_common_fs_info(api, "/int", &total_space, &free_space);
 
         if(error != FSE_OK) {
-            storage_cli_print_path_error(path, error);
+            storage_cli_print_error(error);
         } else {
             printf(
-                "Label: %s\r\nType: LittleFS\r\n%lu KB total\r\n%lu KB free\r\n",
+                "Label: %s\r\nType: LittleFS\r\n%luKB total\r\n%luKB free\r\n",
                 furi_hal_version_get_name_ptr() ? furi_hal_version_get_name_ptr() : "Unknown",
                 (uint32_t)(total_space / 1024),
                 (uint32_t)(free_space / 1024));
@@ -73,10 +65,10 @@ void storage_cli_info(Cli* cli, string_t path) {
         FS_Error error = storage_sd_info(api, &sd_info);
 
         if(error != FSE_OK) {
-            storage_cli_print_path_error(path, error);
+            storage_cli_print_error(error);
         } else {
             printf(
-                "Label: %s\r\nType: %s\r\n%lu KB total\r\n%lu KB free\r\n",
+                "Label: %s\r\nType: %s\r\n%luKB total\r\n%luKB free\r\n",
                 sd_info.label,
                 sd_api_get_fs_type_text(sd_info.fs_type),
                 sd_info.kb_total,
@@ -91,7 +83,7 @@ void storage_cli_info(Cli* cli, string_t path) {
 
 void storage_cli_format(Cli* cli, string_t path) {
     if(string_cmp_str(path, "/int") == 0) {
-        storage_cli_print_path_error(path, FSE_NOT_IMPLEMENTED);
+        storage_cli_print_error(FSE_NOT_IMPLEMENTED);
     } else if(string_cmp_str(path, "/ext") == 0) {
         printf("Formatting SD card, all data will be lost. Are you sure (y/n)?\r\n");
         char answer = cli_getc(cli);
@@ -102,7 +94,7 @@ void storage_cli_format(Cli* cli, string_t path) {
             FS_Error error = storage_sd_format(api);
 
             if(error != FSE_OK) {
-                storage_cli_print_path_error(path, error);
+                storage_cli_print_error(error);
             } else {
                 printf("SD card was successfully formatted.\r\n");
             }
@@ -142,7 +134,7 @@ void storage_cli_list(Cli* cli, string_t path) {
                 printf("\tEmpty\r\n");
             }
         } else {
-            storage_cli_print_file_error(path, file);
+            storage_cli_print_error(storage_file_get_error(file));
         }
 
         storage_dir_close(file);
@@ -172,7 +164,7 @@ void storage_cli_read(Cli* cli, string_t path) {
 
         free(data);
     } else {
-        storage_cli_print_file_error(path, file);
+        storage_cli_print_error(storage_file_get_error(file));
     }
 
     storage_file_close(file);
@@ -181,52 +173,126 @@ void storage_cli_read(Cli* cli, string_t path) {
     furi_record_close("storage");
 }
 
-void storage_cli_write(Cli* cli, string_t path, string_t args) {
+void storage_cli_write(Cli* cli, string_t path) {
     Storage* api = furi_record_open("storage");
     File* file = storage_file_alloc(api);
 
-    uint32_t size;
-    int parsed_count = sscanf(string_get_cstr(args), "%lu", &size);
+    const uint16_t buffer_size = 512;
+    uint8_t* buffer = furi_alloc(buffer_size);
 
-    if(parsed_count == EOF || parsed_count != 1) {
-        storage_cli_print_usage();
-    } else {
-        if(storage_file_open(file, string_get_cstr(path), FSAM_WRITE, FSOM_OPEN_APPEND)) {
-            const uint16_t write_size = 8;
-            uint32_t readed_index = 0;
-            uint8_t* data = furi_alloc(write_size);
+    if(storage_file_open(file, string_get_cstr(path), FSAM_WRITE, FSOM_OPEN_APPEND)) {
+        printf("Just write your text data. New line by Ctrl+Enter, exit by Ctrl+C.\r\n");
 
-            while(true) {
-                data[readed_index % write_size] = cli_getc(cli);
-                printf("%c", data[readed_index % write_size]);
-                fflush(stdout);
-                readed_index++;
+        uint32_t readed_index = 0;
 
-                if(((readed_index % write_size) == 0)) {
-                    uint16_t writed_size = storage_file_write(file, data, write_size);
+        while(true) {
+            uint8_t symbol = cli_getc(cli);
 
-                    if(writed_size != write_size) {
-                        storage_cli_print_file_error(path, file);
-                        break;
-                    }
-                } else if(readed_index == size) {
-                    uint16_t writed_size = storage_file_write(file, data, size % write_size);
+            if(symbol == CliSymbolAsciiETX) {
+                uint16_t write_size = readed_index % buffer_size;
+
+                if(write_size > 0) {
+                    uint16_t writed_size = storage_file_write(file, buffer, write_size);
 
-                    if(writed_size != (size % write_size)) {
-                        storage_cli_print_file_error(path, file);
-                        break;
+                    if(writed_size != write_size) {
+                        storage_cli_print_error(storage_file_get_error(file));
                     }
+                    break;
                 }
+            }
+
+            buffer[readed_index % buffer_size] = symbol;
+            printf("%c", buffer[readed_index % buffer_size]);
+            fflush(stdout);
+            readed_index++;
 
-                if(readed_index == size) {
+            if(((readed_index % buffer_size) == 0)) {
+                uint16_t writed_size = storage_file_write(file, buffer, buffer_size);
+
+                if(writed_size != buffer_size) {
+                    storage_cli_print_error(storage_file_get_error(file));
                     break;
                 }
             }
-            printf("\r\n");
+        }
+        printf("\r\n");
+
+    } else {
+        storage_cli_print_error(storage_file_get_error(file));
+    }
+    storage_file_close(file);
+
+    free(buffer);
+    storage_file_free(file);
+    furi_record_close("storage");
+}
+
+void storage_cli_read_chunks(Cli* cli, string_t path, string_t args) {
+    Storage* api = furi_record_open("storage");
+    File* file = storage_file_alloc(api);
+
+    uint32_t buffer_size;
+    int parsed_count = sscanf(string_get_cstr(args), "%lu", &buffer_size);
+
+    if(parsed_count == EOF || parsed_count != 1) {
+        storage_cli_print_usage();
+    } else if(storage_file_open(file, string_get_cstr(path), FSAM_READ, FSOM_OPEN_EXISTING)) {
+        uint8_t* data = furi_alloc(buffer_size);
+        uint64_t file_size = storage_file_size(file);
+
+        printf("Size: %lu\r\n", (uint32_t)file_size);
+
+        while(file_size > 0) {
+            printf("\r\nReady?\r\n");
+            cli_getc(cli);
+
+            uint16_t readed_size = storage_file_read(file, data, buffer_size);
+            for(uint16_t i = 0; i < readed_size; i++) {
+                putchar(data[i]);
+            }
+            file_size -= readed_size;
+        }
+        printf("\r\n");
+
+        free(data);
+    } else {
+        storage_cli_print_error(storage_file_get_error(file));
+    }
+
+    storage_file_close(file);
+    storage_file_free(file);
+
+    furi_record_close("storage");
+}
+
+void storage_cli_write_chunk(Cli* cli, string_t path, string_t args) {
+    Storage* api = furi_record_open("storage");
+    File* file = storage_file_alloc(api);
+
+    uint32_t buffer_size;
+    int parsed_count = sscanf(string_get_cstr(args), "%lu", &buffer_size);
+
+    if(parsed_count == EOF || parsed_count != 1) {
+        storage_cli_print_usage();
+    } else {
+        if(storage_file_open(file, string_get_cstr(path), FSAM_WRITE, FSOM_OPEN_APPEND)) {
+            printf("Ready\r\n");
 
-            free(data);
+            uint8_t* buffer = furi_alloc(buffer_size);
+
+            for(uint32_t i = 0; i < buffer_size; i++) {
+                buffer[i] = cli_getc(cli);
+            }
+
+            uint16_t writed_size = storage_file_write(file, buffer, buffer_size);
+
+            if(writed_size != buffer_size) {
+                storage_cli_print_error(storage_file_get_error(file));
+            }
+
+            free(buffer);
         } else {
-            storage_cli_print_file_error(path, file);
+            storage_cli_print_error(storage_file_get_error(file));
         }
         storage_file_close(file);
     }
@@ -235,6 +301,45 @@ void storage_cli_write(Cli* cli, string_t path, string_t args) {
     furi_record_close("storage");
 }
 
+void storage_cli_stat(Cli* cli, string_t path) {
+    Storage* api = furi_record_open("storage");
+
+    if(string_cmp_str(path, "/") == 0) {
+        printf("Storage\r\n");
+    } else if(
+        string_cmp_str(path, "/ext") == 0 || string_cmp_str(path, "/int") == 0 ||
+        string_cmp_str(path, "/any") == 0) {
+        uint64_t total_space;
+        uint64_t free_space;
+        FS_Error error =
+            storage_common_fs_info(api, string_get_cstr(path), &total_space, &free_space);
+
+        if(error != FSE_OK) {
+            storage_cli_print_error(error);
+        } else {
+            printf(
+                "Storage, %luKB total, %luKB free\r\n",
+                (uint32_t)(total_space / 1024),
+                (uint32_t)(free_space / 1024));
+        }
+    } else {
+        FileInfo fileinfo;
+        FS_Error error = storage_common_stat(api, string_get_cstr(path), &fileinfo);
+
+        if(error == FSE_OK) {
+            if(fileinfo.flags & FSF_DIRECTORY) {
+                printf("Directory\r\n");
+            } else {
+                printf("File, size: %lub\r\n", (uint32_t)(fileinfo.size));
+            }
+        } else {
+            storage_cli_print_error(error);
+        }
+    }
+
+    furi_record_close("storage");
+}
+
 void storage_cli_copy(Cli* cli, string_t old_path, string_t args) {
     Storage* api = furi_record_open("storage");
     string_t new_path;
@@ -297,6 +402,43 @@ void storage_cli_mkdir(Cli* cli, string_t path) {
     furi_record_close("storage");
 }
 
+void storage_cli_md5(Cli* cli, string_t path) {
+    Storage* api = furi_record_open("storage");
+    File* file = storage_file_alloc(api);
+
+    if(storage_file_open(file, string_get_cstr(path), FSAM_READ, FSOM_OPEN_EXISTING)) {
+        const uint16_t read_size = 512;
+        const uint8_t hash_size = 16;
+        uint8_t* data = malloc(read_size);
+        uint8_t* hash = malloc(sizeof(uint8_t) * hash_size);
+        md5_context* md5_ctx = malloc(sizeof(md5_context));
+
+        md5_starts(md5_ctx);
+        while(true) {
+            uint16_t readed_size = storage_file_read(file, data, read_size);
+            if(readed_size == 0) break;
+            md5_update(md5_ctx, data, readed_size);
+        }
+        md5_finish(md5_ctx, hash);
+        free(md5_ctx);
+
+        for(uint8_t i = 0; i < hash_size; i++) {
+            printf("%02x", hash[i]);
+        }
+        printf("\r\n");
+
+        free(hash);
+        free(data);
+    } else {
+        storage_cli_print_error(storage_file_get_error(file));
+    }
+
+    storage_file_close(file);
+    storage_file_free(file);
+
+    furi_record_close("storage");
+}
+
 void storage_cli(Cli* cli, string_t args, void* context) {
     string_t cmd;
     string_t path;
@@ -334,8 +476,18 @@ void storage_cli(Cli* cli, string_t args, void* context) {
             break;
         }
 
+        if(string_cmp_str(cmd, "read_chunks") == 0) {
+            storage_cli_read_chunks(cli, path, args);
+            break;
+        }
+
         if(string_cmp_str(cmd, "write") == 0) {
-            storage_cli_write(cli, path, args);
+            storage_cli_write(cli, path);
+            break;
+        }
+
+        if(string_cmp_str(cmd, "write_chunk") == 0) {
+            storage_cli_write_chunk(cli, path, args);
             break;
         }
 
@@ -359,6 +511,16 @@ void storage_cli(Cli* cli, string_t args, void* context) {
             break;
         }
 
+        if(string_cmp_str(cmd, "md5") == 0) {
+            storage_cli_md5(cli, path);
+            break;
+        }
+
+        if(string_cmp_str(cmd, "stat") == 0) {
+            storage_cli_stat(cli, path);
+            break;
+        }
+
         storage_cli_print_usage();
     } while(false);
 
diff --git a/applications/storage/storage-processing.c b/applications/storage/storage-processing.c
index fd4cc08c..0ab7f551 100644
--- a/applications/storage/storage-processing.c
+++ b/applications/storage/storage-processing.c
@@ -307,43 +307,67 @@ static FS_Error storage_process_common_remove(Storage* app, const char* path) {
     return ret;
 }
 
+static FS_Error storage_process_common_mkdir(Storage* app, const char* path) {
+    FS_Error ret = FSE_OK;
+    StorageType type = storage_get_type_by_path(path);
+
+    if(storage_type_is_not_valid(type)) {
+        ret = FSE_INVALID_NAME;
+    } else {
+        StorageData* storage = storage_get_storage_by_type(app, type);
+        FS_CALL(storage, common.mkdir(storage, remove_vfs(path)));
+    }
+
+    return ret;
+}
+
 static FS_Error storage_process_common_copy(Storage* app, const char* old, const char* new) {
     FS_Error ret = FSE_INTERNAL;
     File file_old;
     File file_new;
 
-    do {
-        if(!storage_process_file_open(app, &file_old, old, FSAM_READ, FSOM_OPEN_EXISTING)) {
-            ret = storage_file_get_error(&file_old);
-            storage_process_file_close(app, &file_old);
-            break;
-        }
-
-        if(!storage_process_file_open(app, &file_new, new, FSAM_WRITE, FSOM_CREATE_NEW)) {
-            ret = storage_file_get_error(&file_new);
-            storage_process_file_close(app, &file_new);
-            break;
-        }
-
-        const uint16_t buffer_size = 64;
-        uint8_t* buffer = malloc(buffer_size);
-        uint16_t readed_size = 0;
-        uint16_t writed_size = 0;
-
-        while(true) {
-            readed_size = storage_process_file_read(app, &file_old, buffer, buffer_size);
-            ret = storage_file_get_error(&file_old);
-            if(readed_size == 0) break;
+    FileInfo fileinfo;
+    ret = storage_process_common_stat(app, old, &fileinfo);
 
-            writed_size = storage_process_file_write(app, &file_new, buffer, readed_size);
-            ret = storage_file_get_error(&file_new);
-            if(writed_size < readed_size) break;
+    if(ret == FSE_OK) {
+        if(fileinfo.flags & FSF_DIRECTORY) {
+            ret = storage_process_common_mkdir(app, new);
+        } else {
+            do {
+                if(!storage_process_file_open(app, &file_old, old, FSAM_READ, FSOM_OPEN_EXISTING)) {
+                    ret = storage_file_get_error(&file_old);
+                    storage_process_file_close(app, &file_old);
+                    break;
+                }
+
+                if(!storage_process_file_open(app, &file_new, new, FSAM_WRITE, FSOM_CREATE_NEW)) {
+                    ret = storage_file_get_error(&file_new);
+                    storage_process_file_close(app, &file_new);
+                    storage_process_file_close(app, &file_old);
+                    break;
+                }
+
+                const uint16_t buffer_size = 64;
+                uint8_t* buffer = malloc(buffer_size);
+                uint16_t readed_size = 0;
+                uint16_t writed_size = 0;
+
+                while(true) {
+                    readed_size = storage_process_file_read(app, &file_old, buffer, buffer_size);
+                    ret = storage_file_get_error(&file_old);
+                    if(readed_size == 0) break;
+
+                    writed_size = storage_process_file_write(app, &file_new, buffer, readed_size);
+                    ret = storage_file_get_error(&file_new);
+                    if(writed_size < readed_size) break;
+                }
+
+                free(buffer);
+                storage_process_file_close(app, &file_old);
+                storage_process_file_close(app, &file_new);
+            } while(false);
         }
-
-        free(buffer);
-        storage_process_file_close(app, &file_old);
-        storage_process_file_close(app, &file_new);
-    } while(false);
+    }
 
     return ret;
 }
@@ -370,20 +394,6 @@ static FS_Error storage_process_common_rename(Storage* app, const char* old, con
     return ret;
 }
 
-static FS_Error storage_process_common_mkdir(Storage* app, const char* path) {
-    FS_Error ret = FSE_OK;
-    StorageType type = storage_get_type_by_path(path);
-
-    if(storage_type_is_not_valid(type)) {
-        ret = FSE_INVALID_NAME;
-    } else {
-        StorageData* storage = storage_get_storage_by_type(app, type);
-        FS_CALL(storage, common.mkdir(storage, remove_vfs(path)));
-    }
-
-    return ret;
-}
-
 static FS_Error storage_process_common_fs_info(
     Storage* app,
     const char* fs_path,