diff --git a/FUZZING.md b/FUZZING.md new file mode 100644 index 0000000..f517800 --- /dev/null +++ b/FUZZING.md @@ -0,0 +1,47 @@ +# Fuzzing SPIFFS + +The SPIFFS test suite includes a test program designed for fuzzing with +[AFL](http://lcamtuf.coredump.cx/afl/). This automatically exercises the +SPIFFS API and verifies that the file system does not crash or interact incorrectly +with the flash chip. + +There are two steps to fuzzing. The first is to build the test suite with +the AFL version of gcc. The CC variable should point to your copy of afl-gcc. + +``` +make clean test CC=/usr/local/bin/afl-gcc +``` + +There is a new test `afl_test` that reads from stdin a list of commands +and arguments. These are interpreted and executed on the API. The `afltests` +directory contains a number of test cases that can be fed to the `afl_test` test. + + +The second is to run this test suite under afl as follows (where findings is +the output directory): + +``` +afl-fuzz -i afltests -o findings ./build/linux_spiffs_test -f afl_test +``` + +This run will take hours (or days) and will (hopefully) not find any crashes. +If a crash (or hang) is found, then the input file that caused the crash is +saved. This allows the specific test case to be debugged. + +## Reducing the size of the file + +AFL comes with `afl-tmin` which can reduce the size of the test input file to +make it easier to debug. + +``` +afl-tmin -i findings/crashes/ -o smalltest -- build/linux_spiffs_test -f afl_test +``` + +This will write a short version of the testcase file to `smalltest`. This can then be +fed into the test program for debugging: + +``` +build/linux_spiffs_test -f afl_test < smalltest +``` + +This should still crash, but allows it to be run under a debugger. diff --git a/afltests/100 b/afltests/100 new file mode 100644 index 0000000..6bb2239 --- /dev/null +++ b/afltests/100 @@ -0,0 +1,15 @@ +‰5S-C4 +d5rh +OlWkR#C4 +d5rh +O4W4R4O4W4êC4#d5rh +O4d5rh +OlWkRh +O4Y5rh +OlWkR4C44R45ŠË +O4W4ê4C4C4 +O4O4W4R4O4W4êC4#d5rh +O4d5rh +W4R45rË +O4W4ê4#d5rh +rz diff --git a/afltests/200 b/afltests/200 new file mode 100644 index 0000000..9014312 Binary files /dev/null and b/afltests/200 differ diff --git a/afltests/a b/afltests/a new file mode 100644 index 0000000..24e3a21 --- /dev/null +++ b/afltests/a @@ -0,0 +1,18 @@ + b55 +O4W4R4C4D4 +b45 +d5rh +O4W4R4f4C4 +baaU +d5rh +OaWaRafaCa +cd5rh +OaWaRafaCa +O4S4W4R4C4 +d5rh +O4W4S4R4C4 +d5rh +O4W4R4S4C4 +d5rh +O4W4R4C4 +d5rh diff --git a/afltests/b b/afltests/b new file mode 100644 index 0000000..1f95774 --- /dev/null +++ b/afltests/b @@ -0,0 +1,15 @@ +b55 +O4 +W?W?W?W?W?f4 +WW:W;f4 +C4 +b45 +d5rh +O4W?R4f4C4 +baa +d5rh +OaWaRafaCa +d5rh +OaWaRafaCa +O4W?R4C4 +d5rh diff --git a/makefile b/makefile index 13ebd96..b1755fc 100644 --- a/makefile +++ b/makefile @@ -16,7 +16,7 @@ builddir = build # ############# -CC = gcc $(COMPILEROPTIONS) +CC ?= gcc LD = ld GDB = gdb OBJCOPY = objcopy @@ -48,7 +48,7 @@ include files.mk INCLUDE_DIRECTIVES = -I./${sourcedir} -I./${sourcedir}/default -I./${sourcedir}/test COMPILEROPTIONS = $(INCLUDE_DIRECTIVES) -COMPILEROPTIONS_APP = \ +COMPILEROPTIONS_APP = $(INCLUDE_DIRECTIVES) \ -Wall -Wno-format-y2k -W -Wstrict-prototypes -Wmissing-prototypes \ -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch \ -Wshadow -Wcast-align -Wchar-subscripts -Winline -Wnested-externs\ @@ -89,7 +89,7 @@ $(OBJFILES) : ${builddir}/%.o:%.c $(OBJFILES_TEST) : ${builddir}/%.o:%.c @echo "... compile $@" - @${CC} $(CFLAGS) -g -c -o $@ $< + @${CC} ${COMPILEROPTIONS} $(CFLAGS) -g -c -o $@ $< # make dependencies # @echo "... depend $@"; diff --git a/src/spiffs.h b/src/spiffs.h index b66dc32..d87422d 100644 --- a/src/spiffs.h +++ b/src/spiffs.h @@ -137,7 +137,7 @@ typedef void (*spiffs_file_callback)(struct spiffs_t *fs, spiffs_fileop_type op, #ifndef SPIFFS_DBG #define SPIFFS_DBG(...) \ - print(__VA_ARGS__) + printf(__VA_ARGS__) #endif #ifndef SPIFFS_GC_DBG #define SPIFFS_GC_DBG(...) printf(__VA_ARGS__) diff --git a/src/test/test_bugreports.c b/src/test/test_bugreports.c index 2484f86..a6f07d8 100644 --- a/src/test/test_bugreports.c +++ b/src/test/test_bugreports.c @@ -591,6 +591,379 @@ TEST(temporal_fd_cache) { return TEST_RES_OK; } TEST_END +static int run_fuzz_test(FILE *f, int maxfds, int debuglog) { + // There are a bunch of arbitrary constants in this test case. Changing them will + // almost certainly change the effets of an input file. It *may* be worth + // making some of these constants to come from the input file. + int setup = fgetc(f); + + int page_size = 128 << (setup & 3); + setup >>= 2; + int erase_size = 4096 << (setup & 3); + setup >>= 2; + int block_size = erase_size << (setup & 1); + setup >>= 1; + int blocks = 4 + (setup & 7); + fs_reset_specific(0, 0, blocks * block_size, erase_size, block_size, page_size); + int res; + (FS)->fd_count = 4; + + int c; + + spiffs_file fd[4]; + memset(fd, -1, sizeof(fd)); + char *filename[8]; + + int i; + + for (i = 0; i < 8; i++) { + char buff[64]; + sprintf(buff, "%dfile%d.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxasdasdasdadxxxxxxxxxxxxxxxxxxx", i, i); + buff[9 + 2 * i] = 0; + filename[i] = strdup(buff); + } + + // The list of 8 modes that are chosen. SPIFFS_EXCL is not present -- it probably ought to be. + int modes[8] = {SPIFFS_RDONLY, SPIFFS_RDWR, SPIFFS_RDWR|SPIFFS_TRUNC, SPIFFS_RDWR|SPIFFS_CREAT, SPIFFS_RDWR|SPIFFS_CREAT|SPIFFS_TRUNC, + SPIFFS_WRONLY|SPIFFS_CREAT|SPIFFS_TRUNC, SPIFFS_RDWR|SPIFFS_CREAT|SPIFFS_TRUNC|SPIFFS_DIRECT, SPIFFS_WRONLY}; + + char buff[2048]; + for (i = 0; i < sizeof(buff); i++) { + buff[i] = i * 19; + } + +#define LOGOP if (debuglog) printf + + while ((c = fgetc(f)) >= 0) { + int add; + char rbuff[2048]; + if (c <= ' ') { + continue; + } + int arg = fgetc(f); + if (arg < 0) { + break; + } + int fdn = ((arg >> 6) & 3) % maxfds; + switch(c) { + case 'O': + if (fd[fdn] >= 0) { + LOGOP(" close(%d)\n", fd[fdn]); + SPIFFS_close(FS, fd[fdn]); + } + LOGOP(" open(\"%s\", 0x%x)", filename[(arg>>3) & 7], modes[arg & 7]); + fd[fdn] = SPIFFS_open(FS, filename[(arg>>3) & 7], modes[arg & 7], 0); + LOGOP(" -> %d\n", fd[fdn]); + break; + + case 'S': + if (fd[fdn] >= 0) { + int offset = (14 << (arg & 7)) + arg; + if (arg & 16) { + offset = -offset; + } + int whence = (arg & 63) % 3; + LOGOP(" lseek(%d, %d, %d)\n", fd[fdn], offset, whence); + SPIFFS_lseek(FS, fd[fdn], offset, whence); + } + break; + + case 'R': + if (fd[fdn] >= 0) { + LOGOP(" read(%d, , %d)", fd[fdn], (15 << (arg & 7)) + (arg & 127)); + int rlen = SPIFFS_read(FS, fd[fdn], rbuff, (15 << (arg & 7)) + (arg & 127)); + LOGOP(" -> %d\n", rlen); + } + break; + + case 'W': + if (fd[fdn] >= 0) { + LOGOP(" write(%d, , %d)", fd[fdn], (15 << (arg & 7)) + (arg & 127)); + int rc = SPIFFS_write(FS, fd[fdn], buff, (15 << (arg & 7)) + (arg & 127)); + LOGOP(" -> %d\n", rc); + } + break; + + case 'C': + if (fd[fdn] >= 0) { + LOGOP(" close(%d)\n", fd[fdn]); + SPIFFS_close(FS, fd[fdn]); + } + fd[fdn] = -1; + break; + + case 'b': + add = fgetc(f); + for (i = 0; i < sizeof(buff); i++) { + buff[i] = add + i * arg; + } + break; + + case 'f': + if (fd[fdn] >= 0) { + LOGOP(" fflush(%d)\n", fd[fdn]); + SPIFFS_fflush(FS, fd[fdn]); + } + break; + + case 'D': + if (fd[fdn] >= 0) { + LOGOP(" fremove(%d)\n", fd[fdn]); + SPIFFS_fremove(FS, fd[fdn]); + } + break; + + case 'd': + LOGOP(" remove(\"%s\")\n", filename[arg & 7]); + SPIFFS_remove(FS, filename[arg & 7]); + break; + + case 'r': + LOGOP(" rename(\"%s\", \"%s\")\n", filename[arg & 7], filename[(arg >> 3) & 7]); + SPIFFS_rename(FS, filename[arg & 7], filename[(arg >> 3) & 7]); + break; + + case 'U': + ungetc(arg, f); + for (i = 0; i < 4; i++) { + fd[i] = -1; + } + { + char *tmpfile = strdup("/tmp/fsdump.XXXXXX"); + LOGOP(" unmount and remount\n"); + close(mkstemp(tmpfile)); + fs_store_dump(tmpfile); + fs_mount_dump(tmpfile, 0, 0, blocks * block_size, erase_size, block_size, page_size); + unlink(tmpfile); + free(tmpfile); + } + break; + + case 'c': + LOGOP(" check()\n"); + SPIFFS_check(FS); + ungetc(arg, f); + break; + + default: + ungetc(arg, f); + break; + } + } + + for (i = 0; i < 4; i++) { + if (fd[i] >= 0) { + LOGOP(" close(%d)\n", fd[i]); + SPIFFS_close(FS, fd[i]); + } + } + + return TEST_RES_OK; +} + +#define FMEMARGS(x) x, sizeof(x) - 1 + +TEST(fuzzer_found_1) { + return run_fuzz_test(fmemopen(FMEMARGS("\021OlWkd5O4W4W0O5OlWkO5OlW0O5O4W0"), "r"), 4, 1); +} TEST_END + +TEST(fuzzer_found_2) { + return run_fuzz_test(fmemopen(FMEMARGS("bO4W6W0d\036O4W6"), "r"), 4, 1); +} TEST_END + +TEST(fuzzer_found_3) { + return run_fuzz_test(fmemopen(FMEMARGS("\264O4OqWeWWWWW@O4WWW\027"), "r"), 4, 1); +} TEST_END + +TEST(fuzzer_found_single_1) { + return run_fuzz_test(fmemopen(FMEMARGS("\000O\004Odr4d\356Okr0WWUO;WWWWd\035W4"), "r"), 1, 1); +} TEST_END + +TEST(afl_test) { + u32_t old_val = set_abort_on_error(1); + int rc = run_fuzz_test(stdin, 4, 0); + set_abort_on_error(old_val); + return rc; +} TEST_END + +TEST(afl_single) { + u32_t old_val = set_abort_on_error(1); + int rc = run_fuzz_test(stdin, 1, 0); + set_abort_on_error(old_val); + return rc; +} TEST_END + +TEST(small_free_space) { + fs_reset_specific(0, 0, 400*1024, 4096, 2*4096, 256); + spiffs_file fd; + int res; + (FS)->fd_count = 4; + + int tfd = SPIFFS_open(FS, "testfile", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(tfd > 0); + char *tbuf = "some data"; + res = SPIFFS_write(FS, tfd, tbuf, strlen(tbuf)); + + TEST_CHECK(res == strlen(tbuf)); + + res = SPIFFS_fflush(FS, tfd); + TEST_CHECK(res >= SPIFFS_OK); + + SPIFFS_close(FS, tfd); + + const int runs = 1000; + + int fileCurrNumber = 0; + int fileDelNumber = 1; + + int run = 0; + do { + u8_t buf[1000]; + + sprintf(buf, "%d", fileCurrNumber); + int i; + for (i = 0; i < 100; i++) { + strcat(buf, " azzaaax"); + } + + int maxFileNr = 500; + char *filename = "awyn"; + char *fileext = ".dat"; + + u32_t total; + u32_t used; + + SPIFFS_info(FS, &total, &used); + + if (total - used < 20000) { + maxFileNr = 1; + } + + fileCurrNumber++; + int fileCntr = fileCurrNumber + 1 - fileDelNumber; + + char fileCurrName[64]; + sprintf(fileCurrName, "%s%d%s", filename, fileCurrNumber, fileext); + + fd = SPIFFS_open(FS, fileCurrName, SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(fd > 0); + + //printf("About to write to %s\n", fileCurrName); + res = SPIFFS_write(FS, fd, buf, strlen(buf)); + + TEST_CHECK(res == strlen(buf)); + + res = SPIFFS_fflush(FS, fd); + TEST_CHECK_EQ(res, SPIFFS_OK); + + SPIFFS_close(FS, fd); + + if (fileCntr > maxFileNr) { + char fileDelName[64]; + sprintf(fileDelName, "%s%d%s", filename, fileDelNumber, fileext); + //printf("Deleting %s (free space %d)\n", fileDelName, total - used); + + res = SPIFFS_remove(FS, fileDelName); + + TEST_CHECK(res == SPIFFS_OK); + fileDelNumber++; + } + } while (run ++ < runs); + + tfd = SPIFFS_open(FS, "testfile", SPIFFS_RDONLY, 0); + TEST_CHECK(tfd > 0); + char rbuf[32]; + res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf)); + + TEST_CHECK(res == strlen(tbuf)); + + SPIFFS_close(FS, tfd); + + TEST_CHECK(memcmp(rbuf, tbuf, strlen(tbuf)) == 0); + + return TEST_RES_OK; +} TEST_END + +TEST(lots_of_overwrite) { + fs_reset_specific(0, 0, 3000*1024, 4096, 2*4096, 256); + spiffs_file fd; + int res; + (FS)->fd_count = 4; + + int i; + + for (i = 0; i < 5; i++) { + + char filename[64]; + sprintf(filename, "%d-tstfile", i); + int tfd = SPIFFS_open(FS, filename, SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(tfd > 0); + char tbuf[1024]; + memset(tbuf, 'a', 700); + tbuf[700] = 0; + res = SPIFFS_write(FS, tfd, tbuf, strlen(tbuf)); + + TEST_CHECK(res == strlen(tbuf)); + + res = SPIFFS_fflush(FS, tfd); + TEST_CHECK(res >= SPIFFS_OK); + + SPIFFS_close(FS, tfd); + } + + const int runs = 100000; + + int run = 0; + for (run = 0; run < runs; run++) { + u8_t buf[2000]; + + sprintf(buf, "%d", run); + int i; + for (i = 0; i < 100 + (run % 100); i++) { + strcat(buf, " azzaaax"); + } + + int tfd = SPIFFS_open(FS, "file.dat", SPIFFS_RDWR | SPIFFS_CREAT | SPIFFS_TRUNC, 0); + TEST_CHECK(tfd > 0); + res = SPIFFS_write(FS, tfd, buf, strlen(buf)); + + TEST_CHECK(res == strlen(buf)); + + res = SPIFFS_fflush(FS, tfd); + TEST_CHECK(res >= SPIFFS_OK); + + SPIFFS_close(FS, tfd); + + tfd = SPIFFS_open(FS, "file.dat", SPIFFS_RDONLY, 0); + TEST_CHECK(tfd > 0); + char rbuf[2000]; + res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf)); + + TEST_CHECK(res == strlen(buf)); + + SPIFFS_close(FS, tfd); + + TEST_CHECK(memcmp(rbuf, buf, strlen(buf)) == 0); + + char filename[64]; + sprintf(filename, "%d-tstfile", run % 5); + tfd = SPIFFS_open(FS, filename, SPIFFS_RDONLY, 0); + TEST_CHECK(tfd > 0); + char tbuf[1024]; + memset(tbuf, 'a', 700); + tbuf[700] = 0; + res = SPIFFS_read(FS, tfd, rbuf, sizeof(rbuf)); + + TEST_CHECK(res == strlen(tbuf)); + + SPIFFS_close(FS, tfd); + TEST_CHECK(memcmp(rbuf, tbuf, strlen(tbuf)) == 0); + } + + return TEST_RES_OK; +} TEST_END + + #if 0 TEST(spiffs_hidden_file_90) { fs_mount_dump("imgs/90.hidden_file.spiffs", 0, 0, 1*1024*1024, 4096, 4096, 128); @@ -665,6 +1038,14 @@ SUITE_TESTS(bug_tests) ADD_TEST(eof_tell_72) ADD_TEST(spiffs_dup_file_74) ADD_TEST(temporal_fd_cache) + //ADD_TEST(small_free_space) + ADD_TEST(lots_of_overwrite) + ADD_TEST(fuzzer_found_1) + ADD_TEST(fuzzer_found_2) + ADD_TEST(fuzzer_found_3) + ADD_TEST(fuzzer_found_single_1) + ADD_TEST_NON_DEFAULT(afl_test) + ADD_TEST_NON_DEFAULT(afl_single) #if 0 ADD_TEST(spiffs_hidden_file_90) #endif diff --git a/src/test/test_spiffs.c b/src/test/test_spiffs.c index 8d72463..56be261 100644 --- a/src/test/test_spiffs.c +++ b/src/test/test_spiffs.c @@ -27,10 +27,15 @@ #define AREA(x) _area[(x) - addr_offset] +#define ERREXIT() if (abort_on_error) abort(); else error_count++ + static u32_t _area_sz; static unsigned char *_area = NULL; static u32_t addr_offset = 0; +static u32_t abort_on_error = 0; +static int error_count = 0; + static int *_erases; static char _path[256]; static u32_t bytes_rd = 0; @@ -110,6 +115,18 @@ static int mkpath(const char *path, mode_t mode) { } // end take +// +// +int get_error_count() { + return error_count; +} + +u32_t set_abort_on_error(u32_t val) { + u32_t old_val = abort_on_error; + abort_on_error = val; + + return old_val; +} char *make_test_fname(const char *name) { sprintf(_path, "%s/%s", TEST_PATH, name); @@ -157,10 +174,12 @@ static s32_t _read( } if (addr < SPIFFS_CFG_PHYS_ADDR(&__fs)) { printf("FATAL read addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR); + ERREXIT(); exit(0); } if (addr + size > SPIFFS_CFG_PHYS_ADDR(&__fs) + SPIFFS_CFG_PHYS_SZ(&__fs)) { printf("FATAL read addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE); + ERREXIT(); exit(0); } memcpy(dst, &AREA(addr), size); @@ -187,10 +206,12 @@ static s32_t _write( if (addr < SPIFFS_CFG_PHYS_ADDR(&__fs)) { printf("FATAL write addr too low %08x < %08x\n", addr, SPIFFS_PHYS_ADDR); + ERREXIT(); exit(0); } if (addr + size > SPIFFS_CFG_PHYS_ADDR(&__fs) + SPIFFS_CFG_PHYS_SZ(&__fs)) { printf("FATAL write addr too high %08x + %08x > %08x\n", addr, size, SPIFFS_PHYS_ADDR + SPIFFS_FLASH_SIZE); + ERREXIT(); exit(0); } @@ -200,6 +221,7 @@ static s32_t _write( printf("trying to write %02x to %02x at addr %08x\n", src[i], AREA(addr + i), addr+i); spiffs_page_ix pix = (addr + i) / LOG_PAGE; dump_page(&__fs, pix); + ERREXIT(); return -1; } } @@ -214,10 +236,12 @@ static s32_t _erase( u32_t addr, u32_t size) { if (addr & (SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)-1)) { printf("trying to erase at addr %08x, out of boundary\n", addr); + ERREXIT(); return -1; } if (size & (SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)-1)) { printf("trying to erase at with size %08x, out of boundary\n", size); + ERREXIT(); return -1; } _erases[(addr-SPIFFS_CFG_PHYS_ADDR(&__fs))/SPIFFS_CFG_PHYS_ERASE_SZ(&__fs)]++; @@ -354,6 +378,7 @@ void dump_flash_access_stats() { } +static int check_cb_count; // static u32_t old_perc = 999; static void spiffs_check_cb_f( #if SPIFFS_HAL_CALLBACK_EXTRA @@ -375,6 +400,7 @@ static void spiffs_check_cb_f( printf("%i%%\n", arg1 * 100 / 256); }*/ if (report != SPIFFS_CHECK_PROGRESS) { + check_cb_count++; if (report != SPIFFS_CHECK_ERROR) fs_check_fixes++; printf(" check: "); switch (type) { @@ -413,6 +439,7 @@ void fs_set_addr_offset(u32_t offset) { void test_lock(spiffs *fs) { if (_fs_locks != 0) { printf("FATAL: reentrant locks. Abort.\n"); + ERREXIT(); exit(-1); } _fs_locks++; @@ -421,6 +448,7 @@ void test_lock(spiffs *fs) { void test_unlock(spiffs *fs) { if (_fs_locks != 1) { printf("FATAL: unlocking unlocked. Abort.\n"); + ERREXIT(); exit(-1); } _fs_locks--; @@ -905,14 +933,19 @@ void _teardown() { #endif dump_erase_counts(FS); printf(" fs consistency check output begin\n"); + check_cb_count = 0; SPIFFS_check(FS); printf(" fs consistency check output end\n"); + if (check_cb_count) { + ERREXIT(); + } } clear_test_path(); fs_free(); printf(" locks : %i\n", _fs_locks); if (_fs_locks != 0) { printf("FATAL: lock asymmetry. Abort.\n"); + ERREXIT(); exit(-1); } } diff --git a/src/test/test_spiffs.h b/src/test/test_spiffs.h index 1377729..8986f03 100644 --- a/src/test/test_spiffs.h +++ b/src/test/test_spiffs.h @@ -55,6 +55,9 @@ typedef struct { char name[32]; } tfile; +u32_t set_abort_on_error(u32_t val); +int get_error_count(); + void fs_reset(); void fs_reset_specific(u32_t addr_offset, u32_t phys_addr, u32_t phys_size, u32_t phys_sector_size, diff --git a/src/test/testrunner.c b/src/test/testrunner.c index 96fa1a3..6b80115 100644 --- a/src/test/testrunner.c +++ b/src/test/testrunner.c @@ -57,7 +57,7 @@ static char check_spec(char *name) { static char check_incl_filter(char *name) { if (strlen(test_main.incl_filter)== 0) return 1; - return strstr(name, test_main.incl_filter) == 0 ? 0 : 1; + return strstr(name, test_main.incl_filter) == 0 ? 0 : 2; } static char check_excl_filter(char *name) { @@ -65,10 +65,10 @@ static char check_excl_filter(char *name) { return strstr(name, test_main.excl_filter) == 0 ? 1 : 0; } -void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t)) { +void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t), int non_default) { if (f == 0) return; if (!check_spec(name)) return; - if (!check_incl_filter(name)) return; + if (check_incl_filter(name) <= non_default) return; if (!check_excl_filter(name)) return; DBGT("adding test %s\n", name); test *t = malloc(sizeof(test)); @@ -166,7 +166,11 @@ int run_tests(int argc, char **args) { test *next_test = cur_t->_next; DBGT("TEST %i/%i : running test %s\n", i, test_main.test_count, cur_t->name); i++; + int start_error_count = get_error_count(); int res = cur_t->f(cur_t); + if (res == TEST_RES_OK && get_error_count() != start_error_count) { + res = TEST_RES_FAIL; + } cur_t->test_result = res; int fd = res == TEST_RES_OK ? fd_success : fd_bad; write(fd, cur_t->name, strlen(cur_t->name)); diff --git a/src/test/testrunner.h b/src/test/testrunner.h index c64c920..5b1df51 100644 --- a/src/test/testrunner.h +++ b/src/test/testrunner.h @@ -130,7 +130,10 @@ typedef struct test_res_s { } #define ADD_TEST(tf) \ - _add_test(__test_##tf, str(tf), setup, teardown); + _add_test(__test_##tf, str(tf), setup, teardown, 0); + +#define ADD_TEST_NON_DEFAULT(tf) \ + _add_test(__test_##tf, str(tf), setup, teardown, 1); #define ADD_SUITE(sui) \ extern void _add_suite_tests_##sui(void); \ @@ -150,6 +153,6 @@ void test_init(void (*on_stop)(test *t)); // returns 0 if all tests ok, -1 if any test failed, -2 on badness int run_tests(int argc, char **args); void _add_suite(const char *suite_name); -void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t)); +void _add_test(test_f f, char *name, void (*setup)(test *t), void (*teardown)(test *t), int non_default); #endif /* TESTRUNNER_H_ */