GC might be suboptimal #11

GC check more promiscuous
Early abort on crammed systems
Panic gc when full, ignoring erase count
Testcase added

src/spiffs_gc.c

@@ -110,12 +110,15 @@ s32_t spiffs_gc_check(
   }
 
   u32_t needed_pages = (len + SPIFFS_DATA_PAGE_SIZE(fs) - 1) / SPIFFS_DATA_PAGE_SIZE(fs);
-  if (fs->free_blocks <= 2 && (s32_t)needed_pages > free_pages) {
+//  if (fs->free_blocks <= 2 && (s32_t)needed_pages > free_pages) {
+//    SPIFFS_GC_DBG("gc: full freeblk:%i needed:%i free:%i dele:%i\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted);
+//    return SPIFFS_ERR_FULL;
+//  }
+  if ((s32_t)needed_pages > (s32_t)(free_pages + fs->stats_p_deleted)) {
+    SPIFFS_GC_DBG("gc_check: full freeblk:%i needed:%i free:%i dele:%i\n", fs->free_blocks, needed_pages, free_pages, fs->stats_p_deleted);
     return SPIFFS_ERR_FULL;
   }
 
-  //printf("gcing started  %i dirty, blocks %i free, want %i bytes\n", fs->stats_p_allocated + fs->stats_p_deleted, fs->free_blocks, len);
-
   do {
     SPIFFS_GC_DBG("\ngc_check #%i: run gc free_blocks:%i pfree:%i pallo:%i pdele:%i [%i] len:%i of %i\n",
         tries,
@@ -125,11 +128,13 @@ s32_t spiffs_gc_check(
     spiffs_block_ix *cands;
     int count;
     spiffs_block_ix cand;
-    res = spiffs_gc_find_candidate(fs, &cands, &count);
+    s32_t prev_free_pages = free_pages;
+    // if the fs is crammed, ignore block age when selecting candidate - kind of a bad state
+    res = spiffs_gc_find_candidate(fs, &cands, &count, free_pages <= 0);
     SPIFFS_CHECK_RES(res);
     if (count == 0) {
       SPIFFS_GC_DBG("gc_check: no candidates, return\n");
-      return res;
+      return (s32_t)needed_pages < free_pages ? SPIFFS_OK : SPIFFS_ERR_FULL;
     }
 #if SPIFFS_GC_STATS
     fs->stats_gc_runs++;
@@ -156,6 +161,12 @@ s32_t spiffs_gc_check(
           (SPIFFS_PAGES_PER_BLOCK(fs) - SPIFFS_OBJ_LOOKUP_PAGES(fs)) * (fs->block_count - 2)
           - fs->stats_p_allocated - fs->stats_p_deleted;
 
+    if (prev_free_pages <= 0 && prev_free_pages == free_pages) {
+      // abort early to reduce wear, at least tried once
+      SPIFFS_GC_DBG("gc_check: early abort, no result on gc when fs crammed\n");
+      break;
+    }
+
   } while (++tries < SPIFFS_GC_MAX_RUNS && (fs->free_blocks <= 2 ||
       (s32_t)len > free_pages*(s32_t)SPIFFS_DATA_PAGE_SIZE(fs)));
 
@@ -214,7 +225,8 @@ s32_t spiffs_gc_erase_page_stats(
 s32_t spiffs_gc_find_candidate(
     spiffs *fs,
     spiffs_block_ix **block_candidates,
-    int *candidate_count) {
+    int *candidate_count,
+    char fs_crammed) {
   s32_t res = SPIFFS_OK;
   u32_t blocks = fs->block_count;
   spiffs_block_ix cur_block = 0;
@@ -287,7 +299,7 @@ s32_t spiffs_gc_find_candidate(
       s32_t score =
           deleted_pages_in_block * SPIFFS_GC_HEUR_W_DELET +
           used_pages_in_block * SPIFFS_GC_HEUR_W_USED +
-          erase_age * SPIFFS_GC_HEUR_W_ERASE_AGE;
+          erase_age * (fs_crammed ? 0 : SPIFFS_GC_HEUR_W_ERASE_AGE);
       int cand_ix = 0;
       SPIFFS_GC_DBG("gc_check: bix:%i del:%i use:%i score:%i\n", cur_block, deleted_pages_in_block, used_pages_in_block, score);
       while (cand_ix < max_candidates) {

src/spiffs_nucleus.h

@@ -657,7 +657,8 @@ s32_t spiffs_gc_erase_page_stats(
 s32_t spiffs_gc_find_candidate(
     spiffs *fs,
     spiffs_block_ix **block_candidate,
-    int *candidate_count);
+    int *candidate_count,
+    char fs_crammed);
 
 s32_t spiffs_gc_clean(
     spiffs *fs,

src/test/test_bugreports.c

@@ -116,6 +116,8 @@ TEST(nodemcu_full_fs_2) {
   SPIFFS_clearerr(FS);
   printf("  create small file\n");
   fd = SPIFFS_open(FS, "test2.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+#if 0
+  // before gc in v3.1
   TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_OK);
   TEST_CHECK(fd > 0);
 
@@ -130,12 +132,17 @@ TEST(nodemcu_full_fs_2) {
   printf("  >>> file %s size: %i\n", s.name, s.size);
   TEST_CHECK(s.size == 0);
   SPIFFS_clearerr(FS);
-
+#else
+  TEST_CHECK(SPIFFS_errno(FS) == SPIFFS_ERR_FULL);
+  SPIFFS_clearerr(FS);
+#endif
   printf("  remove files\n");
   res = SPIFFS_remove(FS, "test1.txt");
   TEST_CHECK(res == SPIFFS_OK);
+#if 0
   res = SPIFFS_remove(FS, "test2.txt");
   TEST_CHECK(res == SPIFFS_OK);
+#endif
 
   printf("  create medium file\n");
   fd = SPIFFS_open(FS, "test3.txt", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
@@ -172,4 +179,53 @@ TEST(magic_test) {
 
 } TEST_END(magic_test)
 
+TEST(nodemcu_309) {
+  fs_reset_specific(0, 4096*20, 4096, 4096, 256);
+
+  int res;
+  spiffs_file fd;
+  int j;
+
+  for (j = 1; j <= 3; j++) {
+    char fname[32];
+    sprintf(fname, "20K%i.txt", j);
+    fd = SPIFFS_open(FS, fname, SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_DIRECT, 0);
+    TEST_CHECK(fd > 0);
+    int i;
+    spiffs_stat s;
+    res = SPIFFS_OK;
+    for (i = 1; i <= 1280; i++) {
+      char *buf = "0123456789ABCDE";
+      res = SPIFFS_write(FS, fd, buf, strlen(buf));
+      if (res < 0) printf("err @ %i,%i\n", i, j);
+    }
+  }
+
+  int errno = SPIFFS_errno(FS);
+  TEST_CHECK(errno == SPIFFS_ERR_FULL);
+
+  u32_t total;
+  u32_t used;
+
+  SPIFFS_info(FS, &total, &used);
+  printf("total:%i\nused:%i\nremain:%i\nerrno:%i\n", total, used, total-used, errno);
+  TEST_CHECK(total-used < 10000);
+
+  spiffs_DIR d;
+  struct spiffs_dirent e;
+  struct spiffs_dirent *pe = &e;
+
+  SPIFFS_opendir(FS, "/", &d);
+  int spoon_guard = 0;
+  while ((pe = SPIFFS_readdir(&d, pe))) {
+    printf("%s [%04x] size:%i\n", pe->name, pe->obj_id, pe->size);
+    TEST_CHECK(spoon_guard++ < 3);
+  }
+  TEST_CHECK(spoon_guard == 3);
+  SPIFFS_closedir(&d);
+
+  return TEST_RES_OK;
+
+} TEST_END(nodemcu_309)
+
 SUITE_END(bug_tests)