Merge pull request #100 from pjsg/afl

Adds support for fuzzing via AFL

FUZZING.md

@@ -0,0 +1,47 @@
+# Fuzzing SPIFFS
+
+The SPIFFS test suite includes a test program designed for fuzzing with
+[AFL](http://lcamtuf.coredump.cx/afl/). This automatically exercises the 
+SPIFFS API and verifies that the file system does not crash or interact incorrectly
+with the flash chip. 
+
+There are two steps to fuzzing. The first is to build the test suite with
+the AFL version of gcc. The CC variable should point to your copy of afl-gcc.
+
+```
+make clean test CC=/usr/local/bin/afl-gcc
+```
+
+There is a new test `afl_test` that reads from stdin a list of commands
+and arguments. These are interpreted and executed on the API. The `afltests`
+directory contains a number of test cases that can be fed to the `afl_test` test.
+
+
+The second is to run this test suite under afl as follows (where findings is 
+the output directory):
+
+```
+afl-fuzz -i afltests -o findings ./build/linux_spiffs_test -f afl_test
+```
+
+This run will take hours (or days) and will (hopefully) not find any crashes.
+If a crash (or hang) is found, then the input file that caused the crash is 
+saved. This allows the specific test case to be debugged.
+
+## Reducing the size of the file
+
+AFL comes with `afl-tmin` which can reduce the size of the test input file to
+make it easier to debug.
+
+```
+afl-tmin -i findings/crashes/<somefile> -o smalltest -- build/linux_spiffs_test -f afl_test
+```
+
+This will write a short version of the testcase file to `smalltest`. This can then be
+fed into the test program for debugging:
+
+```
+build/linux_spiffs_test -f afl_test < smalltest
+```
+
+This should still crash, but allows it to be run under a debugger. 

afltests/100

@@ -0,0 +1,15 @@
+‰5S-C4
+d5rh
+OlWkR#C4
+d5rh
+O4W4R4O4W4êC4#d5rh
+O4d5rh
+OlWkRh
+O4Y5rh
+OlWkR4C44R45ŠË
+O4W4ê4C4C4
+O4O4W4R4O4W4êC4#d5rh
+O4d5rh
+W4R45rË
+O4W4ê4#d5rh
+rz

afltests/a

@@ -0,0 +1,18 @@
+ b55
+O4W4R4C4D4
+b45
+d5rh
+O4W4R4f4C4
+baaU
+d5rh
+OaWaRafaCa
+cd5rh
+OaWaRafaCa
+O4S4W4R4C4
+d5rh
+O4W4S4R4C4
+d5rh
+O4W4R4S4C4
+d5rh
+O4W4R4C4
+d5rh

afltests/b

@@ -0,0 +1,15 @@
+b55
+O4
+W?W?W?W?W?f4
+W<W=W>W:W;f4
+C4
+b45
+d5rh
+O4W?R4f4C4
+baa
+d5rh
+OaWaRafaCa
+d5rh
+OaWaRafaCa
+O4W?R4C4
+d5rh

makefile

@@ -16,7 +16,7 @@ builddir = build
 #
 #############
 
-CC = gcc $(COMPILEROPTIONS)
+CC ?= gcc
 LD = ld
 GDB = gdb
 OBJCOPY = objcopy
@@ -48,7 +48,7 @@ include files.mk
 INCLUDE_DIRECTIVES = -I./${sourcedir} -I./${sourcedir}/default -I./${sourcedir}/test 
 COMPILEROPTIONS = $(INCLUDE_DIRECTIVES)
 
-COMPILEROPTIONS_APP = \
+COMPILEROPTIONS_APP = $(INCLUDE_DIRECTIVES) \
 -Wall -Wno-format-y2k -W -Wstrict-prototypes -Wmissing-prototypes \
 -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch \
 -Wshadow -Wcast-align -Wchar-subscripts -Winline -Wnested-externs\
@@ -89,7 +89,7 @@ $(OBJFILES) : ${builddir}/%.o:%.c
 
 $(OBJFILES_TEST) : ${builddir}/%.o:%.c
 		@echo "... compile $@"
-		@${CC} $(CFLAGS) -g -c -o $@ $<
+		@${CC} ${COMPILEROPTIONS} $(CFLAGS) -g -c -o $@ $<
 
 # make dependencies
 #		@echo "... depend $@"; 

src/spiffs.h

@@ -137,7 +137,7 @@ typedef void (*spiffs_file_callback)(struct spiffs_t *fs, spiffs_fileop_type op,
 
 #ifndef SPIFFS_DBG
 #define SPIFFS_DBG(...) \
-    print(__VA_ARGS__)
+    printf(__VA_ARGS__)
 #endif
 #ifndef SPIFFS_GC_DBG
 #define SPIFFS_GC_DBG(...) printf(__VA_ARGS__)

src/test/test_bugreports.c

@@ -591,6 +591,379 @@ TEST(temporal_fd_cache) {
   return TEST_RES_OK;
 } TEST_END
 
+static int run_fuzz_test(FILE *f, int maxfds, int debuglog) {
+  // There are a bunch of arbitrary constants in this test case. Changing them will
+  // almost certainly change the effets of an input file. It *may* be worth
+  // making some of these constants to come from the input file. 
+  int setup = fgetc(f);
+
+  int page_size = 128 << (setup & 3);
+  setup >>= 2;
+  int erase_size = 4096 << (setup & 3);
+  setup >>= 2;
+  int block_size = erase_size << (setup & 1);
+  setup >>= 1;
+  int blocks = 4 + (setup & 7);
+  fs_reset_specific(0, 0, blocks * block_size, erase_size, block_size, page_size);
+  int res;
+  (FS)->fd_count = 4;
+
+  int c;
+
+  spiffs_file fd[4];
+  memset(fd, -1, sizeof(fd));
+  char *filename[8];
+
+  int i;
+
+  for (i = 0; i < 8; i++) {
+    char buff[64];
+    sprintf(buff, "%dfile%d.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxasdasdasdadxxxxxxxxxxxxxxxxxxx", i, i);
+    buff[9 + 2 * i] = 0;
+    filename[i] = strdup(buff);
+  }
+
+  // The list of 8 modes that are chosen. SPIFFS_EXCL is not present -- it probably ought to be.
+  int modes[8] = {SPIFFS_RDONLY, SPIFFS_RDWR, SPIFFS_RDWR|SPIFFS_TRUNC, SPIFFS_RDWR|SPIFFS_CREAT, SPIFFS_RDWR|SPIFFS_CREAT|SPIFFS_TRUNC,
+                  SPIFFS_WRONLY|SPIFFS_CREAT|SPIFFS_TRUNC, SPIFFS_RDWR|SPIFFS_CREAT|SPIFFS_TRUNC|SPIFFS_DIRECT, SPIFFS_WRONLY};
+
+  char buff[2048];
+  for (i = 0; i < sizeof(buff); i++) {
+    buff[i] = i * 19;
+  }
+
+#define LOGOP if (debuglog) printf
+
+  while ((c = fgetc(f)) >= 0) {
+    int add;
+    char rbuff[2048];
+    if (c <= ' ') {
+      continue;
+    }
+    int arg = fgetc(f);
+    if (arg < 0) {
+      break;
+    }
+    int fdn = ((arg >> 6) & 3) % maxfds;
+    switch(c) {
+      case 'O':
+	if (fd[fdn] >= 0) {
+          LOGOP("  close(%d)\n", fd[fdn]);
+	  SPIFFS_close(FS, fd[fdn]);
+	}
+        LOGOP("  open(\"%s\", 0x%x)", filename[(arg>>3) & 7], modes[arg & 7]);
+	fd[fdn] = SPIFFS_open(FS, filename[(arg>>3) & 7], modes[arg & 7], 0);
+        LOGOP(" -> %d\n", fd[fdn]);
+	break;
+
+      case 'S':
+	if (fd[fdn] >= 0) {
+	  int offset = (14 << (arg & 7)) + arg;
+	  if (arg & 16) {
+	    offset = -offset;
+	  }
+	  int whence = (arg & 63) % 3;
+          LOGOP("  lseek(%d, %d, %d)\n", fd[fdn], offset, whence);
+	  SPIFFS_lseek(FS, fd[fdn], offset, whence);
+	}
+	break;
+
+      case 'R':
+	if (fd[fdn] >= 0) {
+          LOGOP("  read(%d, , %d)", fd[fdn], (15 << (arg & 7)) + (arg & 127));
+	  int rlen = SPIFFS_read(FS, fd[fdn], rbuff, (15 << (arg & 7)) + (arg & 127));
+          LOGOP(" -> %d\n", rlen);
+	}
+	break;
+
+      case 'W':
+	if (fd[fdn] >= 0) {
+          LOGOP("  write(%d, , %d)", fd[fdn], (15 << (arg & 7)) + (arg & 127));
+	  int rc = SPIFFS_write(FS, fd[fdn], buff, (15 << (arg & 7)) + (arg & 127));
+          LOGOP(" -> %d\n", rc);
+	}
+	break;
+
+      case 'C':
+	if (fd[fdn] >= 0) {
+          LOGOP("  close(%d)\n", fd[fdn]);
+	  SPIFFS_close(FS, fd[fdn]);
+	}
+	fd[fdn] = -1;
+	break;
+
+      case 'b':
+        add = fgetc(f);
+	for (i = 0; i < sizeof(buff); i++) {
+	  buff[i] = add + i * arg;
+	}
+	break;
+
+      case 'f':
+	if (fd[fdn] >= 0) {
+          LOGOP("  fflush(%d)\n", fd[fdn]);
+	  SPIFFS_fflush(FS, fd[fdn]);
+	}
+	break;
+
+      case 'D':
+	if (fd[fdn] >= 0) {
+          LOGOP("  fremove(%d)\n", fd[fdn]);
+	  SPIFFS_fremove(FS, fd[fdn]);
+	}
+	break;
+
+      case 'd':
+        LOGOP("  remove(\"%s\")\n", filename[arg & 7]);
+        SPIFFS_remove(FS, filename[arg & 7]);
+	break;
+
+      case 'r':
+        LOGOP("  rename(\"%s\", \"%s\")\n", filename[arg & 7], filename[(arg >> 3) & 7]);
+        SPIFFS_rename(FS, filename[arg & 7], filename[(arg >> 3) & 7]);
+	break;
+
+      case 'U':
+	ungetc(arg, f);
+	for (i = 0; i < 4; i++) {
+	  fd[i] = -1;
+	}
+	{
+	  char *tmpfile = strdup("/tmp/fsdump.XXXXXX");
+          LOGOP("  unmount and remount\n");
+	  close(mkstemp(tmpfile));
+	  fs_store_dump(tmpfile);
+	  fs_mount_dump(tmpfile, 0, 0, blocks * block_size, erase_size, block_size, page_size);
+	  unlink(tmpfile);
+	  free(tmpfile);
+	}
+	break;
+
+      case 'c':
+        LOGOP("  check()\n");
+        SPIFFS_check(FS);
+	ungetc(arg, f);
+	break;
+
+      default:
+	ungetc(arg, f);
+	break;
+    }
+  }
+
+  for (i = 0; i < 4; i++) {
+    if (fd[i] >= 0) {
+      LOGOP("  close(%d)\n", fd[i]);
+      SPIFFS_close(FS, fd[i]);
+    }
+  }
+
+  return TEST_RES_OK;
+}
+
+#define FMEMARGS(x)	x, sizeof(x) - 1
+
+TEST(fuzzer_found_1) {
+  return run_fuzz_test(fmemopen(FMEMARGS("\021OlWkd5O4W4W0O5OlWkO5OlW0O5O4W0"), "r"), 4, 1);
+} TEST_END
+
+TEST(fuzzer_found_2) {
+  return run_fuzz_test(fmemopen(FMEMARGS("bO4W6W0d\036O4W6"), "r"), 4, 1);
+} TEST_END
+
+TEST(fuzzer_found_3) {
+  return run_fuzz_test(fmemopen(FMEMARGS("\264O4OqWeWWWWW@O4WWW\027"), "r"), 4, 1);
+} TEST_END
+
+TEST(fuzzer_found_single_1) {
+  return run_fuzz_test(fmemopen(FMEMARGS("\000O\004Odr4d\356Okr0WWUO;WWWWd\035W4"), "r"), 1, 1);
+} TEST_END
+
+TEST(afl_test) {
+  u32_t old_val = set_abort_on_error(1);
+  int rc = run_fuzz_test(stdin, 4, 0);
+  set_abort_on_error(old_val);
+  return rc;
+} TEST_END
+
+TEST(afl_single) {
+  u32_t old_val = set_abort_on_error(1);
+  int rc = run_fuzz_test(stdin, 1, 0);
+  set_abort_on_error(old_val);
+  return rc;
+} TEST_END
+
+TEST(small_free_space) {
+  fs_reset_specific(0, 0, 400*1024, 4096, 2*4096, 256);
+  spiffs_file fd;
+  int res;
+  (FS)->fd_count = 4;
+
+  int tfd = SPIFFS_open(FS, "testfile", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+  TEST_CHECK(tfd > 0);
+  char *tbuf = "some data";
+  res = SPIFFS_write(FS, tfd, tbuf, strlen(tbuf));
+
+  TEST_CHECK(res == strlen(tbuf));
+
+  res = SPIFFS_fflush(FS, tfd);
+  TEST_CHECK(res >= SPIFFS_OK);
+
+  SPIFFS_close(FS, tfd);
+
+  const int runs = 1000;
+
+  int fileCurrNumber = 0;
+  int fileDelNumber = 1;
+
+  int run = 0;
+  do {
+    u8_t buf[1000];
+
+    sprintf(buf, "%d", fileCurrNumber);
+    int i;
+    for (i = 0; i < 100; i++) {
+      strcat(buf, " azzaaax");
+    }
+
+    int maxFileNr = 500;
+    char *filename = "awyn";
+    char *fileext = ".dat";
+
+    u32_t total;
+    u32_t used;
+
+    SPIFFS_info(FS, &total, &used);
+
+    if (total - used < 20000) {
+      maxFileNr = 1;
+    }
+
+    fileCurrNumber++;
+    int fileCntr = fileCurrNumber + 1 - fileDelNumber;
+
+    char fileCurrName[64];
+    sprintf(fileCurrName, "%s%d%s", filename, fileCurrNumber, fileext);
+
+    fd = SPIFFS_open(FS, fileCurrName, SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+    TEST_CHECK(fd > 0);
+
+    //printf("About to write to %s\n", fileCurrName);
+    res = SPIFFS_write(FS, fd, buf, strlen(buf));
+
+    TEST_CHECK(res == strlen(buf));
+
+    res = SPIFFS_fflush(FS, fd);
+    TEST_CHECK_EQ(res, SPIFFS_OK);
+
+    SPIFFS_close(FS, fd);
+
+    if (fileCntr > maxFileNr) {
+      char fileDelName[64];
+      sprintf(fileDelName, "%s%d%s", filename, fileDelNumber, fileext);
+      //printf("Deleting %s (free space %d)\n", fileDelName, total - used);
+  
+      res = SPIFFS_remove(FS, fileDelName);
+
+      TEST_CHECK(res == SPIFFS_OK);
+      fileDelNumber++;
+    }
+  } while (run ++ < runs);
+
+  tfd = SPIFFS_open(FS, "testfile", SPIFFS_RDONLY, 0);
+  TEST_CHECK(tfd > 0);
+  char rbuf[32];
+  res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf));
+
+  TEST_CHECK(res == strlen(tbuf));
+
+  SPIFFS_close(FS, tfd);
+
+  TEST_CHECK(memcmp(rbuf, tbuf, strlen(tbuf)) == 0);
+
+  return TEST_RES_OK;
+} TEST_END
+
+TEST(lots_of_overwrite) {
+  fs_reset_specific(0, 0, 3000*1024, 4096, 2*4096, 256);
+  spiffs_file fd;
+  int res;
+  (FS)->fd_count = 4;
+
+  int i;
+
+  for (i = 0; i < 5; i++) {
+
+    char filename[64];
+    sprintf(filename, "%d-tstfile", i);
+    int tfd = SPIFFS_open(FS, filename, SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+    TEST_CHECK(tfd > 0);
+    char tbuf[1024];
+    memset(tbuf, 'a', 700);
+    tbuf[700] = 0;
+    res = SPIFFS_write(FS, tfd, tbuf, strlen(tbuf));
+
+    TEST_CHECK(res == strlen(tbuf));
+
+    res = SPIFFS_fflush(FS, tfd);
+    TEST_CHECK(res >= SPIFFS_OK);
+
+    SPIFFS_close(FS, tfd);
+  }
+
+  const int runs = 100000;
+
+  int run = 0;
+  for (run = 0; run < runs; run++) {
+    u8_t buf[2000];
+
+    sprintf(buf, "%d", run);
+    int i;
+    for (i = 0; i < 100 + (run % 100); i++) {
+      strcat(buf, " azzaaax");
+    }
+
+    int tfd = SPIFFS_open(FS, "file.dat", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0);
+    TEST_CHECK(tfd > 0);
+    res = SPIFFS_write(FS, tfd, buf, strlen(buf));
+
+    TEST_CHECK(res == strlen(buf));
+
+    res = SPIFFS_fflush(FS, tfd);
+    TEST_CHECK(res >= SPIFFS_OK);
+
+    SPIFFS_close(FS, tfd);
+
+    tfd = SPIFFS_open(FS, "file.dat", SPIFFS_RDONLY, 0);
+    TEST_CHECK(tfd > 0);
+    char rbuf[2000];
+    res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf));
+
+    TEST_CHECK(res == strlen(buf));
+
+    SPIFFS_close(FS, tfd);
+
+    TEST_CHECK(memcmp(rbuf, buf, strlen(buf)) == 0);
+
+    char filename[64];
+    sprintf(filename, "%d-tstfile", run % 5);
+    tfd = SPIFFS_open(FS, filename, SPIFFS_RDONLY, 0);
+    TEST_CHECK(tfd > 0);
+    char tbuf[1024];
+    memset(tbuf, 'a', 700);
+    tbuf[700] = 0;
+    res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf));
+
+    TEST_CHECK(res == strlen(tbuf));
+
+    SPIFFS_close(FS, tfd);
+    TEST_CHECK(memcmp(rbuf, tbuf, strlen(tbuf)) == 0);
+  }
+
+  return TEST_RES_OK;
+} TEST_END
+
+
 #if 0
 TEST(spiffs_hidden_file_90) {
   fs_mount_dump("imgs/90.hidden_file.spiffs", 0, 0, 1*1024*1024, 4096, 4096, 128);
@@ -665,6 +1038,14 @@ SUITE_TESTS(bug_tests)
   ADD_TEST(eof_tell_72)
   ADD_TEST(spiffs_dup_file_74)
   ADD_TEST(temporal_fd_cache)
+  //ADD_TEST(small_free_space)
+  ADD_TEST(lots_of_overwrite)
+  ADD_TEST(fuzzer_found_1)
+  ADD_TEST(fuzzer_found_2)
+  ADD_TEST(fuzzer_found_3)
+  ADD_TEST(fuzzer_found_single_1)
+  ADD_TEST_NON_DEFAULT(afl_test)
+  ADD_TEST_NON_DEFAULT(afl_single)
 #if 0
   ADD_TEST(spiffs_hidden_file_90)
 #endif

src/test/test_spiffs.c

@@ -27,10 +27,15 @@
 
 #define AREA(x) _area[(x) - addr_offset]
 
+#define ERREXIT() if (abort_on_error) abort(); else error_count++
+
 static u32_t _area_sz;
 static unsigned char *_area = NULL;
 static u32_t addr_offset = 0;
 
+static u32_t abort_on_error = 0;
+static int error_count = 0;
+
 static int *_erases;
 static char _path[256];
 static u32_t bytes_rd = 0;
@@ -110,6 +115,18 @@ static int mkpath(const char *path, mode_t mode) {
 }
 
 // end take
+//
+//
+int get_error_count() {
+  return error_count;
+}
+
+u32_t set_abort_on_error(u32_t val) {
+  u32_t old_val = abort_on_error;
+  abort_on_error = val;
+
+  return old_val;
+}
 
 char *make_test_fname(const char *name) {
   sprintf(_path, "%s/%s", TEST_PATH, name);
@@ -157,10 +174,12 @@ static s32_t _read(
   }
   if (addr < SPIFFS_CFG_PHYS_ADDR(&__fs)) {
     printf("FATAL read addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR);
+    ERREXIT();
     exit(0);
   }
   if (addr + size > SPIFFS_CFG_PHYS_ADDR(&__fs) + SPIFFS_CFG_PHYS_SZ(&__fs)) {
     printf("FATAL read addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE);
+    ERREXIT();
     exit(0);
   }
   memcpy(dst, &AREA(addr), size);
@@ -187,10 +206,12 @@ static s32_t _write(
 
   if (addr < SPIFFS_CFG_PHYS_ADDR(&__fs)) {
     printf("FATAL write addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR);
+    ERREXIT();
     exit(0);
   }
   if (addr + size > SPIFFS_CFG_PHYS_ADDR(&__fs) + SPIFFS_CFG_PHYS_SZ(&__fs)) {
     printf("FATAL write addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE);
+    ERREXIT();
     exit(0);
   }
 
@@ -200,6 +221,7 @@ static s32_t _write(
         printf("trying to write %02x to %02x at addr %08x\n", src[i], AREA(addr + i), addr+i);
         spiffs_page_ix pix = (addr + i) / LOG_PAGE;
         dump_page(&__fs, pix);
+	ERREXIT();
         return -1;
       }
     }
@@ -214,10 +236,12 @@ static s32_t _erase(
     u32_t addr, u32_t size) {
   if (addr & (SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)-1)) {
     printf("trying to erase at addr %08x, out of boundary\n", addr);
+    ERREXIT();
     return -1;
   }
   if (size & (SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)-1)) {
     printf("trying to erase at with size %08x, out of boundary\n", size);
+    ERREXIT();
     return -1;
   }
   _erases[(addr-SPIFFS_CFG_PHYS_ADDR(&__fs))/SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)]++;
@@ -354,6 +378,7 @@ void dump_flash_access_stats() {
 }
 
 
+static int check_cb_count;
 // static u32_t old_perc = 999;
 static void spiffs_check_cb_f(
 #if SPIFFS_HAL_CALLBACK_EXTRA
@@ -375,6 +400,7 @@ static void spiffs_check_cb_f(
     printf("%i%%\n", arg1 * 100 / 256);
   }*/
   if (report != SPIFFS_CHECK_PROGRESS) {
+    check_cb_count++;
     if (report != SPIFFS_CHECK_ERROR) fs_check_fixes++;
     printf("   check: ");
     switch (type) {
@@ -413,6 +439,7 @@ void fs_set_addr_offset(u32_t offset) {
 void test_lock(spiffs *fs) {
   if (_fs_locks != 0) {
     printf("FATAL: reentrant locks. Abort.\n");
+    ERREXIT();
     exit(-1);
   }
   _fs_locks++;
@@ -421,6 +448,7 @@ void test_lock(spiffs *fs) {
 void test_unlock(spiffs *fs) {
   if (_fs_locks != 1) {
     printf("FATAL: unlocking unlocked. Abort.\n");
+    ERREXIT();
     exit(-1);
   }
   _fs_locks--;
@@ -905,14 +933,19 @@ void _teardown() {
 #endif
     dump_erase_counts(FS);
     printf("  fs consistency check output begin\n");
+    check_cb_count = 0;
     SPIFFS_check(FS);
     printf("  fs consistency check output end\n");
+    if (check_cb_count) {
+      ERREXIT();
+    }
   }
   clear_test_path();
   fs_free();
   printf("  locks : %i\n", _fs_locks);
   if (_fs_locks != 0) {
     printf("FATAL: lock asymmetry. Abort.\n");
+    ERREXIT();
     exit(-1);
   }
 }

src/test/test_spiffs.h

@@ -55,6 +55,9 @@ typedef struct  {
   char name[32];
 } tfile;
 
+u32_t set_abort_on_error(u32_t val);
+int get_error_count();
+
 void fs_reset();
 void fs_reset_specific(u32_t addr_offset, u32_t phys_addr, u32_t phys_size,
     u32_t phys_sector_size,

src/test/testrunner.c

@@ -57,7 +57,7 @@ static char check_spec(char *name) {
 
 static char check_incl_filter(char *name) {
   if (strlen(test_main.incl_filter)== 0) return 1;
-  return strstr(name, test_main.incl_filter) == 0 ? 0 : 1;
+  return strstr(name, test_main.incl_filter) == 0 ? 0 : 2;
 }
 
 static char check_excl_filter(char *name) {
@@ -65,10 +65,10 @@ static char check_excl_filter(char *name) {
   return strstr(name, test_main.excl_filter) == 0 ? 1 : 0;
 }
 
-void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t)) {
+void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t), int non_default) {
   if (f == 0) return;
   if (!check_spec(name)) return;
-  if (!check_incl_filter(name)) return;
+  if (check_incl_filter(name) <= non_default) return;
   if (!check_excl_filter(name)) return;
   DBGT("adding test %s\n", name);
   test *t = malloc(sizeof(test));
@@ -166,7 +166,11 @@ int run_tests(int argc, char **args) {
     test *next_test = cur_t->_next;
     DBGT("TEST %i/%i : running test %s\n", i, test_main.test_count, cur_t->name);
     i++;
+    int start_error_count = get_error_count();
     int res = cur_t->f(cur_t);
+    if (res == TEST_RES_OK && get_error_count() != start_error_count) {
+      res = TEST_RES_FAIL;
+    }
     cur_t->test_result = res;
     int fd = res == TEST_RES_OK ? fd_success : fd_bad;
     write(fd, cur_t->name, strlen(cur_t->name));

src/test/testrunner.h

@@ -130,7 +130,10 @@ typedef struct test_res_s {
   }
 
 #define ADD_TEST(tf) \
-  _add_test(__test_##tf, str(tf), setup, teardown);
+  _add_test(__test_##tf, str(tf), setup, teardown, 0);
+
+#define ADD_TEST_NON_DEFAULT(tf) \
+  _add_test(__test_##tf, str(tf), setup, teardown, 1);
 
 #define ADD_SUITE(sui) \
   extern void _add_suite_tests_##sui(void); \
@@ -150,6 +153,6 @@ void test_init(void (*on_stop)(test *t));
 // returns 0 if all tests ok, -1 if any test failed, -2 on badness
 int run_tests(int argc, char **args);
 void _add_suite(const char *suite_name);
-void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t));
+void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t), int non_default);
 
 #endif /* TESTRUNNER_H_ */