diff --git a/makefile b/makefile index 270e1a5..39ed478 100644 --- a/makefile +++ b/makefile @@ -34,6 +34,7 @@ CFILES_TEST = main.c \ test_dev.c \ test_check.c \ test_hydrogen.c \ + test_bugreports.c \ testsuites.c \ testrunner.c include files.mk diff --git a/src/default/spiffs_config.h b/src/default/spiffs_config.h index c7c6c57..5a6574a 100644 --- a/src/default/spiffs_config.h +++ b/src/default/spiffs_config.h @@ -22,19 +22,19 @@ // Set generic spiffs debug output call. #ifndef SPIFFS_DGB -#define SPIFFS_DBG(...) +#define SPIFFS_DBG(...) //printf(__VA_ARGS__) #endif // Set spiffs debug output call for garbage collecting. #ifndef SPIFFS_GC_DGB -#define SPIFFS_GC_DBG(...) +#define SPIFFS_GC_DBG(...) //printf(__VA_ARGS__) #endif // Set spiffs debug output call for caching. #ifndef SPIFFS_CACHE_DGB -#define SPIFFS_CACHE_DBG(...) +#define SPIFFS_CACHE_DBG(...) //printf("CA: " __VA_ARGS__) #endif // Set spiffs debug output call for system consistency checks. #ifndef SPIFFS_CHECK_DGB -#define SPIFFS_CHECK_DBG(...) +#define SPIFFS_CHECK_DBG(...) //printf(__VA_ARGS__) #endif // Enable/disable API functions to determine exact number of bytes @@ -69,7 +69,7 @@ // Define maximum number of gc runs to perform to reach desired free pages. #ifndef SPIFFS_GC_MAX_RUNS -#define SPIFFS_GC_MAX_RUNS 3 +#define SPIFFS_GC_MAX_RUNS 5 #endif // Enable/disable statistics on gc. Debug/test purpose only. diff --git a/src/spiffs.h b/src/spiffs.h index f42c76a..c34e477 100644 --- a/src/spiffs.h +++ b/src/spiffs.h @@ -388,6 +388,12 @@ s32_t SPIFFS_rename(spiffs *fs, char *old, char *newPath); */ s32_t SPIFFS_errno(spiffs *fs); +/** + * Clears last error. + * @param fs the file system struct + */ +void SPIFFS_clearerr(spiffs *fs); + /** * Opens a directory stream corresponding to the given name. * The stream is positioned at the first entry in the directory. @@ -419,6 +425,21 @@ struct spiffs_dirent *SPIFFS_readdir(spiffs_DIR *d, struct spiffs_dirent *e); */ s32_t SPIFFS_check(spiffs *fs); + +/** + * Returns number of total bytes available and number of used bytes. + * This is an estimation, and depends on if there a many files with little + * data or few files with much data. + * NB: If used number of bytes exceeds total bytes, a SPIFFS_check should + * run. This indicates a power loss in midst of things. In worst case + * (repeated powerlosses in mending or gc) you might have to delete some files. + * + * @param fs the file system struct + * @param total total number of bytes in filesystem + * @param used used number of bytes in filesystem + */ +s32_t SPIFFS_info(spiffs *fs, u32_t *total, u32_t *used); + #if SPIFFS_TEST_VISUALISATION /** * Prints out a visualization of the filesystem. diff --git a/src/spiffs_gc.c b/src/spiffs_gc.c index b34fe51..e122624 100644 --- a/src/spiffs_gc.c +++ b/src/spiffs_gc.c @@ -119,13 +119,13 @@ s32_t spiffs_gc_check( spiffs *fs, u32_t len) { s32_t res; - u32_t free_pages = - (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count + s32_t free_pages = + (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count-2) - fs->stats_p_allocated - fs->stats_p_deleted; int tries = 0; if (fs->free_blocks > 3 && - len < free_pages * SPIFFS_DATA_PAGE_SIZE(fs)) { + (s32_t)len < free_pages * (s32_t)SPIFFS_DATA_PAGE_SIZE(fs)) { return SPIFFS_OK; } @@ -168,16 +168,22 @@ s32_t spiffs_gc_check( SPIFFS_CHECK_RES(res); free_pages = - (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count + (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count - 2) - fs->stats_p_allocated - fs->stats_p_deleted; } while (++tries < SPIFFS_GC_MAX_RUNS && (fs->free_blocks <= 2 || - len > free_pages*SPIFFS_DATA_PAGE_SIZE(fs))); - SPIFFS_GC_DBG("gc_check: finished\n"); + (s32_t)len > free_pages*(s32_t)SPIFFS_DATA_PAGE_SIZE(fs))); - //printf("gcing finished %i dirty, blocks %i free, %i pages free, %i tries, res %i\n", - // fs->stats_p_allocated + fs->stats_p_deleted, - // fs->free_blocks, free_pages, tries, res); + free_pages = + (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count - 2) + - fs->stats_p_allocated - fs->stats_p_deleted; + if ((s32_t)len > free_pages*(s32_t)SPIFFS_DATA_PAGE_SIZE(fs)) { + res = SPIFFS_ERR_FULL; + } + + SPIFFS_GC_DBG("gc_check: finished, %i dirty, blocks %i free, %i pages free, %i tries, res %i\n", + fs->stats_p_allocated + fs->stats_p_deleted, + fs->free_blocks, free_pages, tries, res); return res; } diff --git a/src/spiffs_hydrogen.c b/src/spiffs_hydrogen.c index f6a3a1f..34178e5 100644 --- a/src/spiffs_hydrogen.c +++ b/src/spiffs_hydrogen.c @@ -106,6 +106,10 @@ s32_t SPIFFS_errno(spiffs *fs) { return fs->err_code; } +void SPIFFS_clearerr(spiffs *fs) { + fs->err_code = SPIFFS_OK; +} + s32_t SPIFFS_creat(spiffs *fs, char *path, spiffs_mode mode) { (void)mode; SPIFFS_API_CHECK_MOUNT(fs); @@ -314,8 +318,6 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { #endif } - SPIFFS_DBG("SPIFFS_write %i %04x offs:%i len %i\n", fh, fd->obj_id, offset, len); - #if SPIFFS_CACHE_WR if ((fd->flags & SPIFFS_DIRECT) == 0) { if (len < (s32_t)SPIFFS_CFG_LOG_PAGE_SZ(fs)) { @@ -334,6 +336,7 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) { spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix), fd->cache_page->offset, fd->cache_page->size); spiffs_cache_fd_release(fs, fd->cache_page); + SPIFFS_API_CHECK_RES(fs, res); } else { // writing within cache alloc_cpage = 0; @@ -760,6 +763,29 @@ s32_t SPIFFS_check(spiffs *fs) { return res; } +s32_t SPIFFS_info(spiffs *fs, u32_t *total, u32_t *used) { + s32_t res = SPIFFS_OK; + SPIFFS_API_CHECK_MOUNT(fs); + SPIFFS_LOCK(fs); + + u32_t pages_per_block = SPIFFS_PAGES_PER_BLOCK(fs); + u32_t blocks = fs->block_count; + u32_t obj_lu_pages = SPIFFS_OBJ_LOOKUP_PAGES(fs); + u32_t data_page_size = SPIFFS_DATA_PAGE_SIZE(fs); + u32_t total_data_pages = (blocks - 2) * (pages_per_block - obj_lu_pages) + 1; // -2 for spare blocks, +1 for emergency page + + if (total) { + *total = total_data_pages * data_page_size; + } + + if (used) { + *used = fs->stats_p_allocated * data_page_size; + } + + SPIFFS_UNLOCK(fs); + return res; +} + #if SPIFFS_TEST_VISUALISATION s32_t SPIFFS_vis(spiffs *fs) { s32_t res = SPIFFS_OK; @@ -826,6 +852,9 @@ s32_t SPIFFS_vis(spiffs *fs) { spiffs_printf("free_blocks: %i\n", fs->free_blocks); spiffs_printf("page_alloc: %i\n", fs->stats_p_allocated); spiffs_printf("page_delet: %i\n", fs->stats_p_deleted); + u32_t total, used; + SPIFFS_info(fs, &total, &used); + spiffs_printf("used: %i of %i\n", used, total); SPIFFS_UNLOCK(fs); return res; diff --git a/src/spiffs_nucleus.c b/src/spiffs_nucleus.c index ad2ae71..2476046 100644 --- a/src/spiffs_nucleus.c +++ b/src/spiffs_nucleus.c @@ -811,7 +811,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { s32_t res = SPIFFS_OK; u32_t written = 0; - res = spiffs_gc_check(fs, len); + res = spiffs_gc_check(fs, len + SPIFFS_DATA_PAGE_SIZE(fs)); // add an extra page of data worth for meta SPIFFS_CHECK_RES(res); spiffs_page_object_ix_header *objix_hdr = (spiffs_page_object_ix_header *)fs->work; @@ -912,7 +912,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &pix); SPIFFS_CHECK_RES(res); } - SPIFFS_DBG("append: %04x found object index at page %04x\n", fd->obj_id, pix); + SPIFFS_DBG("append: %04x found object index at page %04x [fd size %i]\n", fd->obj_id, pix, fd->size); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work); SPIFFS_CHECK_RES(res); @@ -1003,8 +1003,8 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // update size in object header index page res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, offset+written, &new_objix_hdr_page); - SPIFFS_DBG("append: %04x store new size II %i in objix_hdr, %04x:%04x, written %i\n", fd->obj_id - , offset+written, new_objix_hdr_page, 0, written); + SPIFFS_DBG("append: %04x store new size II %i in objix_hdr, %04x:%04x, written %i, res %i\n", fd->obj_id + , offset+written, new_objix_hdr_page, 0, written, res2); SPIFFS_CHECK_RES(res2); } else { // wrote within object index header page @@ -1386,13 +1386,20 @@ s32_t spiffs_object_truncate( ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = SPIFFS_OBJ_ID_FREE; } + SPIFFS_DBG("truncate: got data pix %04x\n", data_pix); + if (cur_size - SPIFFS_DATA_PAGE_SIZE(fs) >= new_size) { // delete full data page res = spiffs_page_data_check(fs, fd, data_pix, data_spix); - if (res != SPIFFS_OK) break; + if (res != SPIFFS_ERR_DELETED && res != SPIFFS_OK) break; + + if (res == SPIFFS_OK) { + res = spiffs_page_delete(fs, data_pix); + if (res != SPIFFS_OK) break; + } else if (res == SPIFFS_ERR_DELETED) { + res = SPIFFS_OK; + } - res = spiffs_page_delete(fs, data_pix); - if (res != SPIFFS_OK) break; // update current size if (cur_size % SPIFFS_DATA_PAGE_SIZE(fs) == 0) { cur_size -= SPIFFS_DATA_PAGE_SIZE(fs); diff --git a/src/test/test_bugreports.c b/src/test/test_bugreports.c new file mode 100644 index 0000000..44f03b8 --- /dev/null +++ b/src/test/test_bugreports.c @@ -0,0 +1,87 @@ +/* + * test_bugreports.c + * + * Created on: Mar 8, 2015 + * Author: petera + */ + + + +#include "testrunner.h" +#include "test_spiffs.h" +#include "spiffs_nucleus.h" +#include "spiffs.h" +#include +#include +#include +#include +#include + + +SUITE(bug_tests) +void setup() { + _setup_test_only(); +} +void teardown() { + _teardown(); +} + +TEST(nodemcu_full_fs) { + fs_reset_specific(0, 65536, 4096, 4096, 256); + + int res; + spiffs_file fd; + + printf(" fill up system by writing one byte a lot\n"); + fd = SPIFFS_open(FS, "test1.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(fd > 0); + int i; + spiffs_stat s; + res = SPIFFS_OK; + for (i = 0; res >= 0 && SPIFFS_errno(FS) == SPIFFS_OK && i < 100*1000; i++) { + u8_t buf = 'x'; + res = SPIFFS_write(FS, fd, &buf, 1); + } + + int errno = SPIFFS_errno(FS); + int res2 = SPIFFS_fstat(FS, fd, &s); + TEST_CHECK(res2 == SPIFFS_OK); + printf(" >>> file %s size: %i\n", s.name, s.size); + + TEST_CHECK(errno == SPIFFS_ERR_FULL); + SPIFFS_close(FS, fd); + + + printf(" remove big file\n"); + res = SPIFFS_remove(FS, "test1.txt"); + + TEST_CHECK(res == SPIFFS_OK); + res2 = SPIFFS_fstat(FS, fd, &s); + TEST_CHECK(res2 == -1); + TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_ERR_FILE_CLOSED); + res2 = SPIFFS_stat(FS, "test1.txt", &s); + TEST_CHECK(res2 == -1); + TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_ERR_NOT_FOUND); + + printf(" create small file\n"); + fd = SPIFFS_open(FS, "test2.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(fd > 0); + res = SPIFFS_OK; + for (i = 0; res >= 0 && i < 1000; i++) { + u8_t buf = 'x'; + res = SPIFFS_write(FS, fd, &buf, 1); + } + TEST_CHECK(res >= SPIFFS_OK); + + res2 = SPIFFS_fstat(FS, fd, &s); + TEST_CHECK(res2 == SPIFFS_OK); + printf(" >>> file %s size: %i\n", s.name, s.size); + + TEST_CHECK(s.size == 1000); + SPIFFS_close(FS, fd); + + return TEST_RES_OK; + +} TEST_END(nodemcu_full_fs) + +SUITE_END(bug_tests) diff --git a/src/test/test_hydrogen.c b/src/test/test_hydrogen.c index 4fd4150..fdba160 100644 --- a/src/test/test_hydrogen.c +++ b/src/test/test_hydrogen.c @@ -24,6 +24,17 @@ void teardown() { _teardown(); } +TEST(info) +{ + u32_t used, total; + int res = SPIFFS_info(FS, &total, &used); + TEST_CHECK(res == SPIFFS_OK); + TEST_CHECK(used == 0); + TEST_CHECK(total < __fs.cfg.phys_size); + return TEST_RES_OK; +} +TEST_END(info) + TEST(missing_file) { diff --git a/src/test/test_spiffs.c b/src/test/test_spiffs.c index 478eea2..2033566 100644 --- a/src/test/test_spiffs.c +++ b/src/test/test_spiffs.c @@ -26,7 +26,7 @@ static unsigned char area[PHYS_FLASH_SIZE]; -static int erases[SPIFFS_FLASH_SIZE/SECTOR_SIZE]; +static int erases[256]; static char _path[256]; static u32_t bytes_rd = 0; static u32_t bytes_wr = 0; @@ -80,11 +80,11 @@ static s32_t _read(u32_t addr, u32_t size, u8_t *dst) { return SPIFFS_ERR_TEST; } } - if (addr < SPIFFS_PHYS_ADDR) { + if (addr < __fs.cfg.phys_addr) { printf("FATAL read addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR); exit(0); } - if (addr + size > SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE) { + if (addr + size > __fs.cfg.phys_addr + __fs.cfg.phys_size) { printf("FATAL read addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE); exit(0); } @@ -106,17 +106,17 @@ static s32_t _write(u32_t addr, u32_t size, u8_t *src) { } } - if (addr < SPIFFS_PHYS_ADDR) { + if (addr < __fs.cfg.phys_addr) { printf("FATAL write addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR); exit(0); } - if (addr + size > SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE) { + if (addr + size > __fs.cfg.phys_addr + __fs.cfg.phys_size) { printf("FATAL write addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE); exit(0); } for (i = 0; i < size; i++) { - if (((addr + i) & (LOG_PAGE-1)) != offsetof(spiffs_page_header, flags)) { + if (((addr + i) & (__fs.cfg.log_page_size-1)) != offsetof(spiffs_page_header, flags)) { if (check_valid_flash && ((area[addr + i] ^ src[i]) & src[i])) { printf("trying to write %02x to %02x at addr %08x\n", src[i], area[addr + i], addr+i); spiffs_page_ix pix = (addr + i) / LOG_PAGE; @@ -130,13 +130,15 @@ static s32_t _write(u32_t addr, u32_t size, u8_t *src) { } static s32_t _erase(u32_t addr, u32_t size) { - if (addr & (SECTOR_SIZE-1)) { + if (addr & (__fs.cfg.phys_erase_block-1)) { + printf("trying to erase at addr %08x, out of boundary\n", addr); return -1; } - if (size & (SECTOR_SIZE-1)) { + if (size & (__fs.cfg.phys_erase_block-1)) { + printf("trying to erase at with size %08x, out of boundary\n", size); return -1; } - erases[(addr-SPIFFS_PHYS_ADDR)/SECTOR_SIZE]++; + erases[(addr-__fs.cfg.phys_addr)/__fs.cfg.phys_erase_block]++; memset(&area[addr], 0xff, size); return 0; } @@ -311,18 +313,22 @@ static void spiffs_check_cb_f(spiffs_check_type type, spiffs_check_report report } } -void fs_reset() { +void fs_reset_specific(u32_t phys_addr, u32_t phys_size, + u32_t phys_sector_size, + u32_t log_block_size, u32_t log_page_size) { memset(area, 0xcc, sizeof(area)); - memset(&area[SPIFFS_PHYS_ADDR], 0xff, SPIFFS_FLASH_SIZE); + memset(&area[phys_addr], 0xff, phys_size); + spiffs_config c; c.hal_erase_f = _erase; c.hal_read_f = _read; c.hal_write_f = _write; - c.log_block_size = LOG_BLOCK; - c.log_page_size = LOG_PAGE; - c.phys_addr = SPIFFS_PHYS_ADDR; - c.phys_erase_block = SECTOR_SIZE; - c.phys_size = SPIFFS_FLASH_SIZE; + c.log_block_size = log_block_size; + c.log_page_size = log_page_size; + c.phys_addr = phys_addr; + c.phys_erase_block = phys_sector_size; + c.phys_size = phys_size; + memset(erases,0,sizeof(erases)); memset(_cache,0,sizeof(_cache)); @@ -333,6 +339,10 @@ void fs_reset() { fs_check_fixes = 0; } +void fs_reset() { + fs_reset_specific(SPIFFS_PHYS_ADDR, SPIFFS_FLASH_SIZE, SECTOR_SIZE, LOG_BLOCK, LOG_PAGE); +} + void set_flash_ops_log(int enable) { log_flash_ops = enable; } @@ -540,12 +550,16 @@ static u32_t cmiss_tot = 0; #endif #endif -void _setup() { - fs_reset(); +void _setup_test_only() { fs_set_validate_flashing(1); test_init(test_on_stop); } +void _setup() { + fs_reset(); + _setup_test_only(); +} + void _teardown() { printf(" free blocks : %i of %i\n", (FS)->free_blocks, (FS)->block_count); printf(" pages allocated : %i\n", (FS)->stats_p_allocated); @@ -606,9 +620,11 @@ int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concu if (tf->state == 0 && cur_config_ix < cfg_count) { // create a new file strcpy(tf->name, name); - if (dbg) printf(" create new %s with cfg %i/%i\n", name, (1+cur_config_ix), cfg_count); tf->state = 1; tf->cfg = cfgs[cur_config_ix]; + int size = tfile_get_size(tf->cfg.tsize); + if (dbg) printf(" create new %s with cfg %i/%i, size %i\n", name, (1+cur_config_ix), cfg_count, size); + if (tf->cfg.tsize == EMPTY) { res = SPIFFS_creat(FS, name, 0); CHECK_RES(res); @@ -625,7 +641,6 @@ int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concu extra_flags = tf->cfg.ttype == APPENDED ? O_APPEND : 0; int pfd = open(make_test_fname(name), extra_flags | O_TRUNC | O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); tf->fd = fd; - int size = tfile_get_size(tf->cfg.tsize); u8_t *buf = malloc(size); memrand(buf, size); res = SPIFFS_write(FS, fd, buf, size); diff --git a/src/test/test_spiffs.h b/src/test/test_spiffs.h index 7519264..14c603f 100644 --- a/src/test/test_spiffs.h +++ b/src/test/test_spiffs.h @@ -57,6 +57,9 @@ typedef struct { void fs_reset(); +void fs_reset_specific(u32_t phys_addr, u32_t phys_size, + u32_t phys_sector_size, + u32_t log_block_size, u32_t log_page_size); int read_and_verify(char *name); int read_and_verify_fd(spiffs_file fd, char *name); void dump_page(spiffs *fs, spiffs_page_ix p); @@ -78,6 +81,7 @@ void memrand(u8_t *b, int len); int test_create_file(char *name); int test_create_and_write_file(char *name, int size, int chunk_size); void _setup(); +void _setup_test_only(); void _teardown(); u32_t tfile_get_size(tfile_size s); int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concurrent_files, int dbg); diff --git a/src/test/testsuites.c b/src/test/testsuites.c index 21c9c88..7627db0 100644 --- a/src/test/testsuites.c +++ b/src/test/testsuites.c @@ -11,4 +11,5 @@ void add_suites() { //ADD_SUITE(dev_tests); ADD_SUITE(check_tests); ADD_SUITE(hydrogen_tests) + ADD_SUITE(bug_tests) }