improved gc checks. fixed an append bug. more robust truncate for very special case.

makefile

@@ -34,6 +34,7 @@ CFILES_TEST = main.c \
 	test_dev.c \
 	test_check.c \
 	test_hydrogen.c \
+	test_bugreports.c \
 	testsuites.c \
 	testrunner.c
 include files.mk

src/default/spiffs_config.h

@@ -22,19 +22,19 @@
 
 // Set generic spiffs debug output call.
 #ifndef SPIFFS_DGB
-#define SPIFFS_DBG(...)
+#define SPIFFS_DBG(...) //printf(__VA_ARGS__)
 #endif
 // Set spiffs debug output call for garbage collecting.
 #ifndef SPIFFS_GC_DGB
-#define SPIFFS_GC_DBG(...)
+#define SPIFFS_GC_DBG(...) //printf(__VA_ARGS__)
 #endif
 // Set spiffs debug output call for caching.
 #ifndef SPIFFS_CACHE_DGB
-#define SPIFFS_CACHE_DBG(...)
+#define SPIFFS_CACHE_DBG(...) //printf("CA: " __VA_ARGS__)
 #endif
 // Set spiffs debug output call for system consistency checks.
 #ifndef SPIFFS_CHECK_DGB
-#define SPIFFS_CHECK_DBG(...)
+#define SPIFFS_CHECK_DBG(...) //printf(__VA_ARGS__)
 #endif
 
 // Enable/disable API functions to determine exact number of bytes
@@ -69,7 +69,7 @@
 
 // Define maximum number of gc runs to perform to reach desired free pages.
 #ifndef SPIFFS_GC_MAX_RUNS
-#define SPIFFS_GC_MAX_RUNS              3
+#define SPIFFS_GC_MAX_RUNS              5
 #endif
 
 // Enable/disable statistics on gc. Debug/test purpose only.

src/spiffs.h

@@ -388,6 +388,12 @@ s32_t SPIFFS_rename(spiffs *fs, char *old, char *newPath);
  */
 s32_t SPIFFS_errno(spiffs *fs);
 
+/**
+ * Clears last error.
+ * @param fs            the file system struct
+ */
+void SPIFFS_clearerr(spiffs *fs);
+
 /**
  * Opens a directory stream corresponding to the given name.
  * The stream is positioned at the first entry in the directory.
@@ -419,6 +425,21 @@ struct spiffs_dirent *SPIFFS_readdir(spiffs_DIR *d, struct spiffs_dirent *e);
  */
 s32_t SPIFFS_check(spiffs *fs);
 
+
+/**
+ * Returns number of total bytes available and number of used bytes.
+ * This is an estimation, and depends on if there a many files with little
+ * data or few files with much data.
+ * NB: If used number of bytes exceeds total bytes, a SPIFFS_check should
+ * run. This indicates a power loss in midst of things. In worst case
+ * (repeated powerlosses in mending or gc) you might have to delete some files.
+ *
+ * @param fs            the file system struct
+ * @param total         total number of bytes in filesystem
+ * @param used          used number of bytes in filesystem
+ */
+s32_t SPIFFS_info(spiffs *fs, u32_t *total, u32_t *used);
+
 #if SPIFFS_TEST_VISUALISATION
 /**
  * Prints out a visualization of the filesystem.

src/spiffs_gc.c

@@ -119,13 +119,13 @@ s32_t spiffs_gc_check(
     spiffs *fs,
     u32_t len) {
   s32_t res;
-  u32_t free_pages =
-      (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count
+  s32_t free_pages =
+      (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count-2)
       - fs->stats_p_allocated - fs->stats_p_deleted;
   int tries = 0;
 
   if (fs->free_blocks > 3 &&
-      len < free_pages * SPIFFS_DATA_PAGE_SIZE(fs)) {
+      (s32_t)len < free_pages * (s32_t)SPIFFS_DATA_PAGE_SIZE(fs)) {
     return SPIFFS_OK;
   }
 
@@ -168,16 +168,22 @@ s32_t spiffs_gc_check(
     SPIFFS_CHECK_RES(res);
 
     free_pages =
-          (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * fs->block_count
+          (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count - 2)
           - fs->stats_p_allocated - fs->stats_p_deleted;
 
   } while (++tries < SPIFFS_GC_MAX_RUNS && (fs->free_blocks <= 2 ||
-      len > free_pages*SPIFFS_DATA_PAGE_SIZE(fs)));
-  SPIFFS_GC_DBG("gc_check: finished\n");
+      (s32_t)len > free_pages*(s32_t)SPIFFS_DATA_PAGE_SIZE(fs)));
 
-  //printf("gcing finished %i dirty, blocks %i free, %i pages free, %i tries, res %i\n",
-  //    fs->stats_p_allocated + fs->stats_p_deleted,
-  //    fs->free_blocks, free_pages, tries, res);
+  free_pages =
+        (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count - 2)
+        - fs->stats_p_allocated - fs->stats_p_deleted;
+  if ((s32_t)len > free_pages*(s32_t)SPIFFS_DATA_PAGE_SIZE(fs)) {
+    res = SPIFFS_ERR_FULL;
+  }
+
+  SPIFFS_GC_DBG("gc_check: finished, %i dirty, blocks %i free, %i pages free, %i tries, res %i\n",
+      fs->stats_p_allocated + fs->stats_p_deleted,
+      fs->free_blocks, free_pages, tries, res);
 
   return res;
 }

src/spiffs_hydrogen.c

@@ -106,6 +106,10 @@ s32_t SPIFFS_errno(spiffs *fs) {
   return fs->err_code;
 }
 
+void SPIFFS_clearerr(spiffs *fs) {
+  fs->err_code = SPIFFS_OK;
+}
+
 s32_t SPIFFS_creat(spiffs *fs, char *path, spiffs_mode mode) {
   (void)mode;
   SPIFFS_API_CHECK_MOUNT(fs);
@@ -314,8 +318,6 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) {
 #endif
   }
 
-  SPIFFS_DBG("SPIFFS_write %i %04x offs:%i len %i\n", fh, fd->obj_id, offset, len);
-
 #if SPIFFS_CACHE_WR
   if ((fd->flags & SPIFFS_DIRECT) == 0) {
     if (len < (s32_t)SPIFFS_CFG_LOG_PAGE_SZ(fs)) {
@@ -334,6 +336,7 @@ s32_t SPIFFS_write(spiffs *fs, spiffs_file fh, void *buf, s32_t len) {
               spiffs_get_cache_page(fs, spiffs_get_cache(fs), fd->cache_page->ix),
               fd->cache_page->offset, fd->cache_page->size);
           spiffs_cache_fd_release(fs, fd->cache_page);
+          SPIFFS_API_CHECK_RES(fs, res);
         } else {
           // writing within cache
           alloc_cpage = 0;
@@ -760,6 +763,29 @@ s32_t SPIFFS_check(spiffs *fs) {
   return res;
 }
 
+s32_t SPIFFS_info(spiffs *fs, u32_t *total, u32_t *used) {
+  s32_t res = SPIFFS_OK;
+  SPIFFS_API_CHECK_MOUNT(fs);
+  SPIFFS_LOCK(fs);
+
+  u32_t pages_per_block = SPIFFS_PAGES_PER_BLOCK(fs);
+  u32_t blocks = fs->block_count;
+  u32_t obj_lu_pages = SPIFFS_OBJ_LOOKUP_PAGES(fs);
+  u32_t data_page_size = SPIFFS_DATA_PAGE_SIZE(fs);
+  u32_t total_data_pages = (blocks - 2) * (pages_per_block - obj_lu_pages) + 1; // -2 for spare blocks, +1 for emergency page
+
+  if (total) {
+    *total = total_data_pages * data_page_size;
+  }
+
+  if (used) {
+    *used = fs->stats_p_allocated * data_page_size;
+  }
+
+  SPIFFS_UNLOCK(fs);
+  return res;
+}
+
 #if SPIFFS_TEST_VISUALISATION
 s32_t SPIFFS_vis(spiffs *fs) {
   s32_t res = SPIFFS_OK;
@@ -826,6 +852,9 @@ s32_t SPIFFS_vis(spiffs *fs) {
   spiffs_printf("free_blocks: %i\n", fs->free_blocks);
   spiffs_printf("page_alloc:  %i\n", fs->stats_p_allocated);
   spiffs_printf("page_delet:  %i\n", fs->stats_p_deleted);
+  u32_t total, used;
+  SPIFFS_info(fs, &total, &used);
+  spiffs_printf("used:        %i of %i\n", used, total);
 
   SPIFFS_UNLOCK(fs);
   return res;

src/spiffs_nucleus.c

@@ -811,7 +811,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
   s32_t res = SPIFFS_OK;
   u32_t written = 0;
 
-  res = spiffs_gc_check(fs, len);
+  res = spiffs_gc_check(fs, len + SPIFFS_DATA_PAGE_SIZE(fs)); // add an extra page of data worth for meta
   SPIFFS_CHECK_RES(res);
 
   spiffs_page_object_ix_header *objix_hdr = (spiffs_page_object_ix_header *)fs->work;
@@ -912,7 +912,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
             res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &pix);
             SPIFFS_CHECK_RES(res);
           }
-          SPIFFS_DBG("append: %04x found object index at page %04x\n", fd->obj_id, pix);
+          SPIFFS_DBG("append: %04x found object index at page %04x [fd size %i]\n", fd->obj_id, pix, fd->size);
           res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ,
               fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
           SPIFFS_CHECK_RES(res);
@@ -1003,8 +1003,8 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
     // update size in object header index page
     res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id,
         fd->objix_hdr_pix, 0, 0, offset+written, &new_objix_hdr_page);
-    SPIFFS_DBG("append: %04x store new size II %i in objix_hdr, %04x:%04x, written %i\n", fd->obj_id
-        , offset+written, new_objix_hdr_page, 0, written);
+    SPIFFS_DBG("append: %04x store new size II %i in objix_hdr, %04x:%04x, written %i, res %i\n", fd->obj_id
+        , offset+written, new_objix_hdr_page, 0, written, res2);
     SPIFFS_CHECK_RES(res2);
   } else {
     // wrote within object index header page
@@ -1386,13 +1386,20 @@ s32_t spiffs_object_truncate(
       ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = SPIFFS_OBJ_ID_FREE;
     }
 
+    SPIFFS_DBG("truncate: got data pix %04x\n", data_pix);
+
     if (cur_size - SPIFFS_DATA_PAGE_SIZE(fs) >= new_size) {
       // delete full data page
       res = spiffs_page_data_check(fs, fd, data_pix, data_spix);
-      if (res != SPIFFS_OK) break;
+      if (res != SPIFFS_ERR_DELETED && res != SPIFFS_OK) break;
+
+      if (res == SPIFFS_OK) {
+        res = spiffs_page_delete(fs, data_pix);
+        if (res != SPIFFS_OK) break;
+      } else if (res == SPIFFS_ERR_DELETED) {
+        res = SPIFFS_OK;
+      }
 
-      res = spiffs_page_delete(fs, data_pix);
-      if (res != SPIFFS_OK) break;
       // update current size
       if (cur_size % SPIFFS_DATA_PAGE_SIZE(fs) == 0) {
         cur_size -= SPIFFS_DATA_PAGE_SIZE(fs);

src/test/test_bugreports.c

@@ -0,0 +1,87 @@
+/*
+ * test_bugreports.c
+ *
+ *  Created on: Mar 8, 2015
+ *      Author: petera
+ */
+
+
+
+#include "testrunner.h"
+#include "test_spiffs.h"
+#include "spiffs_nucleus.h"
+#include "spiffs.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+
+
+SUITE(bug_tests)
+void setup() {
+  _setup_test_only();
+}
+void teardown() {
+  _teardown();
+}
+
+TEST(nodemcu_full_fs) {
+  fs_reset_specific(0, 65536, 4096, 4096, 256);
+
+  int res;
+  spiffs_file fd;
+
+  printf("  fill up system by writing one byte a lot\n");
+  fd = SPIFFS_open(FS, "test1.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+  TEST_CHECK(fd > 0);
+  int i;
+  spiffs_stat s;
+  res = SPIFFS_OK;
+  for (i = 0; res >= 0 && SPIFFS_errno(FS) == SPIFFS_OK && i < 100*1000; i++) {
+    u8_t buf = 'x';
+    res = SPIFFS_write(FS, fd, &buf, 1);
+  }
+
+  int errno = SPIFFS_errno(FS);
+  int res2 = SPIFFS_fstat(FS, fd, &s);
+  TEST_CHECK(res2 == SPIFFS_OK);
+  printf("  >>> file %s size: %i\n", s.name, s.size);
+
+  TEST_CHECK(errno == SPIFFS_ERR_FULL);
+  SPIFFS_close(FS, fd);
+
+
+  printf("  remove big file\n");
+  res = SPIFFS_remove(FS, "test1.txt");
+
+  TEST_CHECK(res == SPIFFS_OK);
+  res2 = SPIFFS_fstat(FS, fd, &s);
+  TEST_CHECK(res2 == -1);
+  TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_ERR_FILE_CLOSED);
+  res2 = SPIFFS_stat(FS, "test1.txt", &s);
+  TEST_CHECK(res2 == -1);
+  TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_ERR_NOT_FOUND);
+
+  printf("  create small file\n");
+  fd = SPIFFS_open(FS, "test2.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+  TEST_CHECK(fd > 0);
+  res = SPIFFS_OK;
+  for (i = 0; res >= 0 && i < 1000; i++) {
+    u8_t buf = 'x';
+    res = SPIFFS_write(FS, fd, &buf, 1);
+  }
+  TEST_CHECK(res >= SPIFFS_OK);
+
+  res2 = SPIFFS_fstat(FS, fd, &s);
+  TEST_CHECK(res2 == SPIFFS_OK);
+  printf("  >>> file %s size: %i\n", s.name, s.size);
+
+  TEST_CHECK(s.size == 1000);
+  SPIFFS_close(FS, fd);
+
+  return TEST_RES_OK;
+
+} TEST_END(nodemcu_full_fs)
+
+SUITE_END(bug_tests)

src/test/test_hydrogen.c

@@ -24,6 +24,17 @@ void teardown() {
   _teardown();
 }
 
+TEST(info)
+{
+  u32_t used, total;
+  int res = SPIFFS_info(FS, &total, &used);
+  TEST_CHECK(res == SPIFFS_OK);
+  TEST_CHECK(used == 0);
+  TEST_CHECK(total < __fs.cfg.phys_size);
+  return TEST_RES_OK;
+}
+TEST_END(info)
+
 
 TEST(missing_file)
 {

src/test/test_spiffs.c

@@ -26,7 +26,7 @@
 
 static unsigned char area[PHYS_FLASH_SIZE];
 
-static int erases[SPIFFS_FLASH_SIZE/SECTOR_SIZE];
+static int erases[256];
 static char _path[256];
 static u32_t bytes_rd = 0;
 static u32_t bytes_wr = 0;
@@ -80,11 +80,11 @@ static s32_t _read(u32_t addr, u32_t size, u8_t *dst) {
       return SPIFFS_ERR_TEST;
     }
   }
-  if (addr < SPIFFS_PHYS_ADDR) {
+  if (addr < __fs.cfg.phys_addr) {
     printf("FATAL read addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR);
     exit(0);
   }
-  if (addr + size > SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE) {
+  if (addr + size > __fs.cfg.phys_addr + __fs.cfg.phys_size) {
     printf("FATAL read addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE);
     exit(0);
   }
@@ -106,17 +106,17 @@ static s32_t _write(u32_t addr, u32_t size, u8_t *src) {
     }
   }
 
-  if (addr < SPIFFS_PHYS_ADDR) {
+  if (addr < __fs.cfg.phys_addr) {
     printf("FATAL write addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR);
     exit(0);
   }
-  if (addr + size > SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE) {
+  if (addr + size > __fs.cfg.phys_addr + __fs.cfg.phys_size) {
     printf("FATAL write addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE);
     exit(0);
   }
 
   for (i = 0; i < size; i++) {
-    if (((addr + i) & (LOG_PAGE-1)) != offsetof(spiffs_page_header, flags)) {
+    if (((addr + i) & (__fs.cfg.log_page_size-1)) != offsetof(spiffs_page_header, flags)) {
       if (check_valid_flash && ((area[addr + i] ^ src[i]) & src[i])) {
         printf("trying to write %02x to %02x at addr %08x\n", src[i], area[addr + i], addr+i);
         spiffs_page_ix pix = (addr + i) / LOG_PAGE;
@@ -130,13 +130,15 @@ static s32_t _write(u32_t addr, u32_t size, u8_t *src) {
 }
 
 static s32_t _erase(u32_t addr, u32_t size) {
-  if (addr & (SECTOR_SIZE-1)) {
+  if (addr & (__fs.cfg.phys_erase_block-1)) {
+    printf("trying to erase at addr %08x, out of boundary\n", addr);
     return -1;
   }
-  if (size & (SECTOR_SIZE-1)) {
+  if (size & (__fs.cfg.phys_erase_block-1)) {
+    printf("trying to erase at with size %08x, out of boundary\n", size);
     return -1;
   }
-  erases[(addr-SPIFFS_PHYS_ADDR)/SECTOR_SIZE]++;
+  erases[(addr-__fs.cfg.phys_addr)/__fs.cfg.phys_erase_block]++;
   memset(&area[addr], 0xff, size);
   return 0;
 }
@@ -311,18 +313,22 @@ static void spiffs_check_cb_f(spiffs_check_type type, spiffs_check_report report
   }
 }
 
-void fs_reset() {
+void fs_reset_specific(u32_t phys_addr, u32_t phys_size,
+    u32_t phys_sector_size,
+    u32_t log_block_size, u32_t log_page_size) {
   memset(area, 0xcc, sizeof(area));
-  memset(&area[SPIFFS_PHYS_ADDR], 0xff, SPIFFS_FLASH_SIZE);
+  memset(&area[phys_addr], 0xff, phys_size);
+
   spiffs_config c;
   c.hal_erase_f = _erase;
   c.hal_read_f = _read;
   c.hal_write_f = _write;
-  c.log_block_size = LOG_BLOCK;
-  c.log_page_size = LOG_PAGE;
-  c.phys_addr = SPIFFS_PHYS_ADDR;
-  c.phys_erase_block = SECTOR_SIZE;
-  c.phys_size = SPIFFS_FLASH_SIZE;
+  c.log_block_size = log_block_size;
+  c.log_page_size = log_page_size;
+  c.phys_addr = phys_addr;
+  c.phys_erase_block = phys_sector_size;
+  c.phys_size = phys_size;
+
   memset(erases,0,sizeof(erases));
   memset(_cache,0,sizeof(_cache));
 
@@ -333,6 +339,10 @@ void fs_reset() {
   fs_check_fixes = 0;
 }
 
+void fs_reset() {
+  fs_reset_specific(SPIFFS_PHYS_ADDR, SPIFFS_FLASH_SIZE, SECTOR_SIZE, LOG_BLOCK, LOG_PAGE);
+}
+
 void set_flash_ops_log(int enable) {
   log_flash_ops = enable;
 }
@@ -540,12 +550,16 @@ static u32_t cmiss_tot = 0;
 #endif
 #endif
 
-void _setup() {
-  fs_reset();
+void _setup_test_only() {
   fs_set_validate_flashing(1);
   test_init(test_on_stop);
 }
 
+void _setup() {
+  fs_reset();
+  _setup_test_only();
+}
+
 void _teardown() {
   printf("  free blocks     : %i of %i\n", (FS)->free_blocks, (FS)->block_count);
   printf("  pages allocated : %i\n", (FS)->stats_p_allocated);
@@ -606,9 +620,11 @@ int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concu
       if (tf->state == 0 && cur_config_ix < cfg_count) {
 // create a new file
         strcpy(tf->name, name);
-        if (dbg) printf("   create new %s with cfg %i/%i\n", name, (1+cur_config_ix), cfg_count);
         tf->state = 1;
         tf->cfg = cfgs[cur_config_ix];
+        int size = tfile_get_size(tf->cfg.tsize);
+        if (dbg) printf("   create new %s with cfg %i/%i, size %i\n", name, (1+cur_config_ix), cfg_count, size);
+
         if (tf->cfg.tsize == EMPTY) {
           res = SPIFFS_creat(FS, name, 0);
           CHECK_RES(res);
@@ -625,7 +641,6 @@ int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concu
           extra_flags = tf->cfg.ttype == APPENDED ? O_APPEND : 0;
           int pfd = open(make_test_fname(name), extra_flags | O_TRUNC | O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
           tf->fd = fd;
-          int size = tfile_get_size(tf->cfg.tsize);
           u8_t *buf = malloc(size);
           memrand(buf, size);
           res = SPIFFS_write(FS, fd, buf, size);

src/test/test_spiffs.h

@@ -57,6 +57,9 @@ typedef struct  {
 
 
 void fs_reset();
+void fs_reset_specific(u32_t phys_addr, u32_t phys_size,
+    u32_t phys_sector_size,
+    u32_t log_block_size, u32_t log_page_size);
 int read_and_verify(char *name);
 int read_and_verify_fd(spiffs_file fd, char *name);
 void dump_page(spiffs *fs, spiffs_page_ix p);
@@ -78,6 +81,7 @@ void memrand(u8_t *b, int len);
 int test_create_file(char *name);
 int test_create_and_write_file(char *name, int size, int chunk_size);
 void _setup();
+void _setup_test_only();
 void _teardown();
 u32_t tfile_get_size(tfile_size s);
 int run_file_config(int cfg_count, tfile_conf* cfgs, int max_runs, int max_concurrent_files, int dbg);

src/test/testsuites.c

@@ -11,4 +11,5 @@ void add_suites() {
   //ADD_SUITE(dev_tests);
   ADD_SUITE(check_tests);
   ADD_SUITE(hydrogen_tests)
+  ADD_SUITE(bug_tests)
 }