pellepl/spiffs
1
0
Fork 0
mirror of https://github.com/pellepl/spiffs.git synced 2026-06-06 05:25:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add index memory mapping feature #97

Browse Source
This commit is contained in:
Peter Andersson
2016-08-08 19:07:05 +02:00
parent e04a67bb80
commit 87856826e1
10 changed files with 982 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
makefile
+24 -21
View File
@@ -33,7 +33,7 @@ NO_TEST ?= 0
CFLAGS = $(FLAGS)
ifeq (1, $(strip $(NO_TEST)))
CFILES_TEST = main.c
CFLAGS += -DNO_TEST
CFLAGS += -DNO_TEST -Werror
else
CFILES_TEST = main.c \
test_spiffs.c \
@@ -123,7 +123,6 @@ clean:
@rm -f ${builddir}/*.d
@rm -f ${builddir}/*.elf
BUILD_ALL_FLAGS = NO_TEST=1 -Werror
ONOFF = 1 0
OFFON = 0 1
build-all:
@@ -133,27 +132,31 @@ build-all:
for cache in $(OFFON); do \
for magic in $(OFFON); do \
for temporal_cache in $(OFFON); do \
echo; \
echo ============================================================; \
echo SPIFFS_READ_ONLY=$$rdonly; \
echo SPIFFS_SINGLETON=$$singleton; \
echo SPIFFS_HAL_CALLBACK_EXTRA=$$hal_cb_xtra; \
echo SPIFFS_CACHE, SPIFFS_CACHE_WR=$$cache; \
echo SPIFFS_USE_MAGIC, SPIFFS_USE_MAGIC_LENGTH=$$magic; \
echo SPIFFS_TEMPORAL_FD_CACHE=$$temporal_cache; \
$(MAKE) clean && $(MAKE) FLAGS="\
-DSPIFFS_HAL_CALLBACK_EXTRA=$$hal_cb_xtra \
-DSPIFFS_SINGLETON=$$singleton \
-DSPIFFS_CACHE=$$cache \
-DSPIFFS_CACHE_WR=$$cache \
-DSPIFFS_READ_ONLY=$$rdonly \
-DSPIFFS_USE_MAGIC=$$magic \
-DSPIFFS_USE_MAGIC_LENGTH=$$magic \
-DSPIFFS_TEMPORAL_FD_CACHE=$$temporal_cache \
" $(BUILD_ALL_FLAGS); \
for ix_map in $(OFFON); do \
echo; \
echo ============================================================; \
echo SPIFFS_READ_ONLY=$$rdonly; \
echo SPIFFS_SINGLETON=$$singleton; \
echo SPIFFS_HAL_CALLBACK_EXTRA=$$hal_cb_xtra; \
echo SPIFFS_CACHE, SPIFFS_CACHE_WR=$$cache; \
echo SPIFFS_USE_MAGIC, SPIFFS_USE_MAGIC_LENGTH=$$magic; \
echo SPIFFS_TEMPORAL_FD_CACHE=$$temporal_cache; \
echo SPIFFS_IX_MAP=$$ix_map; \
$(MAKE) clean && $(MAKE) FLAGS="\
-DSPIFFS_HAL_CALLBACK_EXTRA=$$hal_cb_xtra \
-DSPIFFS_SINGLETON=$$singleton \
-DSPIFFS_CACHE=$$cache \
-DSPIFFS_CACHE_WR=$$cache \
-DSPIFFS_READ_ONLY=$$rdonly \
-DSPIFFS_USE_MAGIC=$$magic \
-DSPIFFS_USE_MAGIC_LENGTH=$$magic \
-DSPIFFS_TEMPORAL_FD_CACHE=$$temporal_cache \
-DSPIFFS_IX_MAP=$$ix_map \
" NO_TEST=1; \
done || exit 1; \
done \
done \
done \
done \
done \
done
done
src/default/spiffs_config.h
+16
View File
@@ -237,6 +237,22 @@
#define SPIFFS_TEMPORAL_CACHE_HIT_SCORE 4
#endif
// Enable to be able to map object indices to memory.
// This allows for faster and more deterministic reading if cases of reading
// large files and when changing file offset by seeking around a lot.
// When mapping a file's index, the file system will be scanned for index pages
// and the info will be put in memory provided by user. When reading, the
// memory map can be looked up instead of searching for index pages on the
// medium. This way, user can trade memory against performance.
// Whole, parts of, or future parts not being written yet can be mapped. The
// memory array will be owned by spiffs and updated accordingly during garbage
// collecting or when modifying the indices. The latter is invoked by when the
// file is modified in some way. The index buffer is tied to the file
// descriptor.
#ifndef SPIFFS_IX_MAP
#define SPIFFS_IX_MAP 1
#endif
// Set SPIFFS_TEST_VISUALISATION to non-zero to enable SPIFFS_vis function
// in the api. This function will visualize all filesystem using given printf
// function.
src/spiffs.h
+98
View File
@@ -56,6 +56,10 @@ extern "C" {
#define SPIFFS_ERR_PROBE_NOT_A_FS -10035
#define SPIFFS_ERR_NAME_TOO_LONG -10036
#define SPIFFS_ERR_IX_MAP_UNMAPPED -10037
#define SPIFFS_ERR_IX_MAP_MAPPED -10038
#define SPIFFS_ERR_IX_MAP_BAD_RANGE -10039
#define SPIFFS_ERR_INTERNAL -10050
#define SPIFFS_ERR_TEST -10100
@@ -309,6 +313,21 @@ typedef struct {
int entry;
} spiffs_DIR;
#if SPIFFS_IX_MAP
typedef struct {
// buffer with looked up data pixes
spiffs_page_ix *map_buf;
// precise file byte offset
u32_t offset;
// start data span index of lookup buffer
spiffs_span_ix start_spix;
// end data span index of lookup buffer
spiffs_span_ix end_spix;
} spiffs_ix_map;
#endif
// functions
#if SPIFFS_USE_MAGIC && SPIFFS_USE_MAGIC_LENGTH && SPIFFS_SINGLETON==0
@@ -658,6 +677,85 @@ s32_t SPIFFS_tell(spiffs *fs, spiffs_file fh);
*/
s32_t SPIFFS_set_file_callback_func(spiffs *fs, spiffs_file_callback cb_func);
#if SPIFFS_IX_MAP
/**
* Maps the first level index lookup to a given memory map.
* This will make reading big files faster, as the memory map will be used for
* looking up data pages instead of searching for the indices on the physical
* medium. When mapping, all affected indicies are found and the information is
* copied to the array.
* Whole file or only parts of it may be mapped. The index map will cover file
* contents from argument offset until and including arguments (offset+len).
* It is valid to map a longer range than the current file size. The map will
* then be populated when the file grows.
* On garbage collections and file data page movements, the map array will be
* automatically updated. Do not tamper with the map array, as this contains
* the references to the data pages. Modifying it from outside will corrupt any
* future readings using this file descriptor.
* The map will no longer be used when the file descriptor closed or the file
* is unmapped.
* This can be useful to get faster and more deterministic timing when reading
* large files, or when seeking and reading a lot within a file.
* @param fs the file system struct
* @param fh the file handle of the file to map
* @param map a spiffs_ix_map struct, describing the index map
* @param offset absolute file offset where to start the index map
* @param len length of the mapping in actual file bytes
* @param map_buf the array buffer for the look up data - number of required
* elements in the array can be derived from function
* SPIFFS_bytes_to_ix_map_entries given the length
*/
s32_t SPIFFS_ix_map(spiffs *fs, spiffs_file fh, spiffs_ix_map *map,
u32_t offset, u32_t len, spiffs_page_ix *map_buf);
/**
* Unmaps the index lookup from this filehandle. All future readings will
* proceed as normal, requiring reading of the first level indices from
* physical media.
* The map and map buffer given in function SPIFFS_ix_map will no longer be
* referenced by spiffs.
* It is not strictly necessary to unmap a file before closing it, as closing
* a file will automatically unmap it.
* @param fs the file system struct
* @param fh the file handle of the file to unmap
*/
s32_t SPIFFS_ix_unmap(spiffs *fs, spiffs_file fh);
/**
* Moves the offset for the index map given in function SPIFFS_ix_map. Parts or
* all of the map buffer will repopulated.
* @param fs the file system struct
* @param fh the mapped file handle of the file to remap
* @param offset new absolute file offset where to start the index map
*/
s32_t SPIFFS_ix_remap(spiffs *fs, spiffs_file fh, u32_t offs);
/**
* Utility function to get number of spiffs_page_ix entries a map buffer must
* contain on order to map given amount of file data in bytes.
* See function SPIFFS_ix_map and SPIFFS_ix_map_entries_to_bytes.
* @param fs the file system struct
* @param bytes number of file data bytes to map
* @return needed number of elements in a spiffs_page_ix array needed to
* map given amount of bytes in a file
*/
s32_t SPIFFS_bytes_to_ix_map_entries(spiffs *fs, u32_t bytes);
/**
* Utility function to amount of file data bytes that can be mapped when
* mapping a file with buffer having given number of spiffs_page_ix entries.
* See function SPIFFS_ix_map and SPIFFS_bytes_to_ix_map_entries.
* @param fs the file system struct
* @param map_page_ix_entries number of entries in a spiffs_page_ix array
* @return amount of file data in bytes that can be mapped given a map
* buffer having given amount of spiffs_page_ix entries
*/
s32_t SPIFFS_ix_map_entries_to_bytes(spiffs *fs, u32_t map_page_ix_entries);
#endif // SPIFFS_IX_MAP
#if SPIFFS_TEST_VISUALISATION
/**
* Prints out a visualization of the filesystem.
src/spiffs_gc.c
+6 -3
View File
@@ -491,7 +491,8 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) {
res = spiffs_page_move(fs, 0, 0, obj_id, &p_hdr, cur_pix, &new_pix);
SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX move objix %04x:%04x page %04x to %04x\n", obj_id, p_hdr.span_ix, cur_pix, new_pix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, obj_id, p_hdr.span_ix, new_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)&p_hdr,
SPIFFS_EV_IX_MOV, obj_id, p_hdr.span_ix, new_pix, 0);
// move wipes obj_lu, reload it
res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
0, bix * SPIFFS_CFG_LOG_BLOCK_SZ(fs) + SPIFFS_PAGE_TO_PADDR(fs, obj_lookup_page),
@@ -504,7 +505,8 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) {
SPIFFS_GC_DBG("gc_clean: MOVE_OBJIX wipe objix %04x:%04x page %04x\n", obj_id, p_hdr.span_ix, cur_pix);
res = spiffs_page_delete(fs, cur_pix);
if (res == SPIFFS_OK) {
spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_DEL, obj_id, p_hdr.span_ix, cur_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0,
SPIFFS_EV_IX_DEL, obj_id, p_hdr.span_ix, cur_pix, 0);
}
}
SPIFFS_CHECK_RES(res);
@@ -581,7 +583,8 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) {
res = spiffs_page_move(fs, 0, fs->work, gc.cur_obj_id | SPIFFS_OBJ_ID_IX_FLAG, 0, gc.cur_objix_pix, &new_objix_pix);
SPIFFS_GC_DBG("gc_clean: MOVE_DATA store modified objix page, %04x:%04x\n", new_objix_pix, objix->p_hdr.span_ix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_UPD, gc.cur_obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work,
SPIFFS_EV_IX_UPD, gc.cur_obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
}
}
break;
src/spiffs_hydrogen.c
+125
View File
@@ -1106,6 +1106,131 @@ s32_t SPIFFS_set_file_callback_func(spiffs *fs, spiffs_file_callback cb_func) {
return 0;
}
#if SPIFFS_IX_MAP
s32_t SPIFFS_ix_map(spiffs *fs, spiffs_file fh, spiffs_ix_map *map,
u32_t offset, u32_t len, spiffs_page_ix *map_buf) {
s32_t res;
SPIFFS_API_CHECK_CFG(fs);
SPIFFS_API_CHECK_MOUNT(fs);
SPIFFS_LOCK(fs);
fh = SPIFFS_FH_UNOFFS(fs, fh);
spiffs_fd *fd;
res = spiffs_fd_get(fs, fh, &fd);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
if (fd->ix_map) {
SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_MAPPED);
}
map->map_buf = map_buf;
map->offset = offset;
map->start_spix = offset / SPIFFS_DATA_PAGE_SIZE(fs);
map->end_spix = (offset + len) / SPIFFS_DATA_PAGE_SIZE(fs);
memset(map_buf, 0, sizeof(spiffs_page_ix) * (map->end_spix - map->start_spix + 1));
fd->ix_map = map;
// scan for pixes
res = spiffs_populate_ix_map(fs, fd, 0, map->end_spix - map->start_spix + 1);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
SPIFFS_UNLOCK(fs);
return res;
}
s32_t SPIFFS_ix_unmap(spiffs *fs, spiffs_file fh) {
s32_t res;
SPIFFS_API_CHECK_CFG(fs);
SPIFFS_API_CHECK_MOUNT(fs);
SPIFFS_LOCK(fs);
fh = SPIFFS_FH_UNOFFS(fs, fh);
spiffs_fd *fd;
res = spiffs_fd_get(fs, fh, &fd);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
if (fd->ix_map == 0) {
SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_UNMAPPED);
}
fd->ix_map = 0;
SPIFFS_UNLOCK(fs);
return res;
}
s32_t SPIFFS_ix_remap(spiffs *fs, spiffs_file fh, u32_t offset) {
s32_t res = SPIFFS_OK;
SPIFFS_API_CHECK_CFG(fs);
SPIFFS_API_CHECK_MOUNT(fs);
SPIFFS_LOCK(fs);
fh = SPIFFS_FH_UNOFFS(fs, fh);
spiffs_fd *fd;
res = spiffs_fd_get(fs, fh, &fd);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
if (fd->ix_map == 0) {
SPIFFS_API_CHECK_RES_UNLOCK(fs, SPIFFS_ERR_IX_MAP_UNMAPPED);
}
spiffs_ix_map *map = fd->ix_map;
s32_t spix_diff = offset / SPIFFS_DATA_PAGE_SIZE(fs) - map->start_spix;
map->offset = offset;
// move existing pixes if within map offs
if (spix_diff != 0) {
// move vector
int i;
const s32_t vec_len = map->end_spix - map->start_spix + 1;
map->start_spix += spix_diff;
map->end_spix += spix_diff;
if (spix_diff >= vec_len) {
// moving beyond range
memset(&map->map_buf, 0, vec_len * sizeof(spiffs_page_ix));
res = spiffs_populate_ix_map(fs, fd, 0, vec_len);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
} else if (spix_diff > 0) {
// diff positive
for (i = 0; i < vec_len - spix_diff - 1; i++) {
map->map_buf[i] = map->map_buf[i + spix_diff];
}
memset(&map->map_buf[vec_len - spix_diff - 1], 0, spix_diff * sizeof(spiffs_page_ix));
res = spiffs_populate_ix_map(fs, fd, vec_len - spix_diff, vec_len);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
} else {
// diff negative
for (i = vec_len - 1; i >= -spix_diff; i--) {
map->map_buf[i] = map->map_buf[i + spix_diff];
}
memset(&map->map_buf[0], 0, -spix_diff * sizeof(spiffs_page_ix));
res = spiffs_populate_ix_map(fs, fd, 0, spix_diff);
SPIFFS_API_CHECK_RES_UNLOCK(fs, res);
}
}
SPIFFS_UNLOCK(fs);
return res;
}
s32_t SPIFFS_bytes_to_ix_map_entries(spiffs *fs, u32_t bytes) {
SPIFFS_API_CHECK_CFG(fs);
return (bytes + (SPIFFS_DATA_PAGE_SIZE(fs) -1) ) / SPIFFS_DATA_PAGE_SIZE(fs);
}
s32_t SPIFFS_ix_map_entries_to_bytes(spiffs *fs, u32_t map_page_ix_entries) {
SPIFFS_API_CHECK_CFG(fs);
return map_page_ix_entries * SPIFFS_DATA_PAGE_SIZE(fs);
}
#endif // SPIFFS_IX_MAP
#if SPIFFS_TEST_VISUALISATION
s32_t SPIFFS_vis(spiffs *fs) {
s32_t res = SPIFFS_OK;
src/spiffs_nucleus.c
+241 -50
View File
@@ -600,6 +600,151 @@ s32_t spiffs_obj_lu_find_id_and_span_by_phdr(
return res;
}
#if SPIFFS_IX_MAP
// update index map of given fd with given object index data
static void spiffs_update_ix_map(spiffs *fs,
spiffs_fd *fd, spiffs_span_ix objix_spix, spiffs_page_object_ix *objix) {
#if SPIFFS_SINGLETON
(void)fs;
#endif
spiffs_ix_map *map = fd->ix_map;
spiffs_span_ix map_objix_start_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix);
spiffs_span_ix map_objix_end_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->end_spix);
// check if updated ix is within map range
if (objix_spix < map_objix_start_spix || objix_spix > map_objix_end_spix) {
return;
}
// update memory mapped page index buffer to new pages
// get range of updated object index map data span indices
spiffs_span_ix objix_data_spix_start =
SPIFFS_DATA_SPAN_IX_FOR_OBJ_IX_SPAN_IX(fs, objix_spix);
spiffs_span_ix objix_data_spix_end = objix_data_spix_start +
(objix_spix == 0 ? SPIFFS_OBJ_HDR_IX_LEN(fs) : SPIFFS_OBJ_IX_LEN(fs));
// calc union of object index range and index map range array
spiffs_span_ix map_spix = MAX(map->start_spix, objix_data_spix_start);
spiffs_span_ix map_spix_end = MIN(map->end_spix + 1, objix_data_spix_end);
while (map_spix < map_spix_end) {
spiffs_page_ix objix_data_pix;
if (objix_spix == 0) {
// get data page from object index header page
objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix_header)))[map_spix];
} else {
// get data page from object index page
objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, map_spix)];
}
if (objix_data_pix == (spiffs_page_ix)-1) {
// reached end of object, abort
break;
}
map->map_buf[map_spix - map->start_spix] = objix_data_pix;
SPIFFS_DBG("map %04x:%04x (%04x--%04x) objix.spix:%04x to pix %04x\n",
fd->obj_id, map_spix - map->start_spix,
map->start_spix, map->end_spix,
objix->p_hdr.span_ix,
objix_data_pix);
map_spix++;
}
}
typedef struct {
spiffs_fd *fd;
u32_t remaining_objix_pages_to_visit;
spiffs_span_ix map_objix_start_spix;
spiffs_span_ix map_objix_end_spix;
} spiffs_ix_map_populate_state;
static s32_t spiffs_populate_ix_map_v(
spiffs *fs,
spiffs_obj_id obj_id,
spiffs_block_ix bix,
int ix_entry,
const void *user_const_p,
void *user_var_p) {
(void)user_const_p;
s32_t res;
spiffs_ix_map_populate_state *state = (spiffs_ix_map_populate_state *)user_var_p;
spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry);
// load header to check it
spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work;
res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix), (u8_t *)objix);
SPIFFS_CHECK_RES(res);
SPIFFS_VALIDATE_OBJIX(objix->p_hdr, obj_id, objix->p_hdr.span_ix);
// check if hdr is ok, and if objix range overlap with ix map range
if ((objix->p_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) ==
(SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_IXDELE) &&
objix->p_hdr.span_ix >= state->map_objix_start_spix &&
objix->p_hdr.span_ix <= state->map_objix_end_spix) {
// ok, load rest of object index
res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ,
0, SPIFFS_PAGE_TO_PADDR(fs, pix) + sizeof(spiffs_page_object_ix),
SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix),
(u8_t *)objix + sizeof(spiffs_page_object_ix));
SPIFFS_CHECK_RES(res);
spiffs_update_ix_map(fs, state->fd, objix->p_hdr.span_ix, objix);
state->remaining_objix_pages_to_visit--;
SPIFFS_DBG("map %04x (%04x--%04x) remaining objix pages %i\n",
state->fd->obj_id,
state->fd->ix_map->start_spix, state->fd->ix_map->end_spix,
state->remaining_objix_pages_to_visit);
}
if (res == SPIFFS_OK) {
res = state->remaining_objix_pages_to_visit ? SPIFFS_VIS_COUNTINUE : SPIFFS_VIS_END;
}
return res;
}
s32_t spiffs_populate_ix_map(spiffs *fs, spiffs_fd *fd, u32_t vec_entry_start, u32_t vec_entry_end) {
s32_t res;
spiffs_ix_map *map = fd->ix_map;
spiffs_ix_map_populate_state state;
vec_entry_start = MIN((map->end_spix - map->start_spix + 1) - 1, (s32_t)vec_entry_start);
vec_entry_end = MAX((map->end_spix - map->start_spix + 1) - 1, (s32_t)vec_entry_end);
if (vec_entry_start > vec_entry_end) {
return SPIFFS_ERR_IX_MAP_BAD_RANGE;
}
state.map_objix_start_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix + vec_entry_start);
state.map_objix_end_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, map->start_spix + vec_entry_end);
state.remaining_objix_pages_to_visit =
state.map_objix_end_spix - state.map_objix_start_spix + 1;
state.fd = fd;
res = spiffs_obj_lu_find_entry_visitor(
fs,
SPIFFS_BLOCK_FOR_PAGE(fs, fd->objix_hdr_pix),
SPIFFS_OBJ_LOOKUP_ENTRY_FOR_PAGE(fs, fd->objix_hdr_pix),
SPIFFS_VIS_CHECK_ID,
fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG,
spiffs_populate_ix_map_v,
0,
&state,
0,
0);
if (res == SPIFFS_VIS_END) {
res = SPIFFS_OK;
}
return res;
}
#endif
#if !SPIFFS_READ_ONLY
// Allocates a free defined page with given obj_id
// Occupies object lookup entry and page
@@ -800,7 +945,8 @@ s32_t spiffs_object_create(
0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, entry), sizeof(spiffs_page_object_ix_header), (u8_t*)&oix_hdr);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, 0, SPIFFS_EV_IX_NEW, obj_id, 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), SPIFFS_UNDEFINED_LEN);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)&oix_hdr,
SPIFFS_EV_IX_NEW, obj_id, 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry), SPIFFS_UNDEFINED_LEN);
if (objix_hdr_pix) {
*objix_hdr_pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, entry);
@@ -859,7 +1005,9 @@ s32_t spiffs_object_update_index_hdr(
*new_pix = new_objix_hdr_pix;
}
// callback on object index update
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, obj_id, objix_hdr->p_hdr.span_ix, new_objix_hdr_pix, objix_hdr->size);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix_hdr,
new_objix_hdr_data ? SPIFFS_EV_IX_UPD : SPIFFS_EV_IX_UPD_HDR,
obj_id, objix_hdr->p_hdr.span_ix, new_objix_hdr_pix, objix_hdr->size);
if (fd) fd->objix_hdr_pix = new_objix_hdr_pix; // if this is not in the registered cluster
}
@@ -869,13 +1017,15 @@ s32_t spiffs_object_update_index_hdr(
void spiffs_cb_object_event(
spiffs *fs,
spiffs_fd *fd,
spiffs_page_object_ix *objix,
int ev,
spiffs_obj_id obj_id_raw,
spiffs_span_ix spix,
spiffs_page_ix new_pix,
u32_t new_size) {
(void)fd;
#if SPIFFS_IX_MAP == 0
(void)objix;
#endif
// update index caches in all file descriptors
spiffs_obj_id obj_id = obj_id_raw & ~SPIFFS_OBJ_ID_IX_FLAG;
u32_t i;
@@ -888,19 +1038,19 @@ void spiffs_cb_object_event(
if (cur_fd->file_nbr == 0 || (cur_fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) != obj_id) continue;
#endif
if (spix == 0) {
if (ev == SPIFFS_EV_IX_NEW || ev == SPIFFS_EV_IX_UPD) {
if (ev != SPIFFS_EV_IX_DEL) {
SPIFFS_DBG(" callback: setting fd %i:%04x objix_hdr_pix to %04x, size:%i\n", cur_fd->file_nbr, cur_fd->obj_id, new_pix, new_size);
cur_fd->objix_hdr_pix = new_pix;
if (new_size != 0) {
cur_fd->size = new_size;
}
} else if (ev == SPIFFS_EV_IX_DEL) {
} else {
cur_fd->file_nbr = 0;
cur_fd->obj_id = SPIFFS_OBJ_ID_DELETED;
}
}
if (cur_fd->cursor_objix_spix == spix) {
if (ev == SPIFFS_EV_IX_NEW || ev == SPIFFS_EV_IX_UPD) {
if (ev != SPIFFS_EV_IX_DEL) {
SPIFFS_DBG(" callback: setting fd %i:%04x span:%04x objix_pix to %04x\n", cur_fd->file_nbr, cur_fd->obj_id, spix, new_pix);
cur_fd->cursor_objix_pix = new_pix;
} else {
@@ -909,12 +1059,31 @@ void spiffs_cb_object_event(
}
}
#if SPIFFS_IX_MAP
// update index maps
if (ev == SPIFFS_EV_IX_UPD || ev == SPIFFS_EV_IX_NEW) {
for (i = 0; i < fs->fd_count; i++) {
spiffs_fd *cur_fd = &fds[i];
// check fd opened, having ix map, match obj id
if (cur_fd->file_nbr == 0 ||
cur_fd->ix_map == 0 ||
(cur_fd->obj_id & ~SPIFFS_OBJ_ID_IX_FLAG) != obj_id) continue;
SPIFFS_DBG(" callback: map ix update fd %i:%04x span:%04x\n", cur_fd->file_nbr, cur_fd->obj_id, spix, new_pix);
spiffs_update_ix_map(fs, cur_fd, spix, objix);
}
}
#endif
// callback to user if object index header
if (fs->file_cb_f && spix == 0 && (obj_id_raw & SPIFFS_OBJ_ID_IX_FLAG)) {
spiffs_fileop_type op;
if (ev == SPIFFS_EV_IX_NEW) {
op = SPIFFS_CB_CREATED;
} else if (ev == SPIFFS_EV_IX_UPD) {
} else if (ev == SPIFFS_EV_IX_UPD ||
ev == SPIFFS_EV_IX_MOV ||
ev == SPIFFS_EV_IX_UPD_HDR) {
op = SPIFFS_CB_UPDATED;
} else if (ev == SPIFFS_EV_IX_DEL) {
op = SPIFFS_CB_DELETED;
@@ -1055,7 +1224,8 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_UPDT,
fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD,fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work,
SPIFFS_EV_IX_UPD,fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0);
// update length in object index header page
res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id,
fd->objix_hdr_pix, 0, 0, offset+written, &new_objix_hdr_page);
@@ -1085,10 +1255,11 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
res = spiffs_page_allocate_data(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG,
&p_hdr, 0, 0, 0, 1, &cur_objix_pix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_NEW, fd->obj_id, cur_objix_spix, cur_objix_pix, 0);
// quick "load" of new object index page
memset(fs->work, 0xff, SPIFFS_CFG_LOG_PAGE_SZ(fs));
memcpy(fs->work, &p_hdr, sizeof(spiffs_page_header));
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work,
SPIFFS_EV_IX_NEW, fd->obj_id, cur_objix_spix, cur_objix_pix, 0);
SPIFFS_DBG("append: %04x create objix page, %04x:%04x, written %i\n", fd->obj_id
, cur_objix_pix, cur_objix_spix, written);
} else {
@@ -1187,7 +1358,8 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
res2 = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_UPDT,
fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
SPIFFS_CHECK_RES(res2);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work,
SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, cur_objix_pix, 0);
// update size in object header index page
res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id,
@@ -1210,7 +1382,8 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, cur_objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
SPIFFS_CHECK_RES(res2);
// callback on object index update
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix_hdr->p_hdr.span_ix, cur_objix_pix, objix_hdr->size);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)fs->work,
SPIFFS_EV_IX_UPD_HDR, fd->obj_id, objix_hdr->p_hdr.span_ix, cur_objix_pix, objix_hdr->size);
} else {
// modifying object index header page, update size and make new copy
res2 = spiffs_object_update_index_hdr(fs, fd, fd->obj_id,
@@ -1277,7 +1450,8 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
res = spiffs_page_move(fs, fd->file_nbr, (u8_t*)objix, fd->obj_id, 0, cur_objix_pix, &new_objix_pix);
SPIFFS_DBG("modify: store previous modified objix page, %04x:%04x, written %i\n", new_objix_pix, objix->p_hdr.span_ix, written);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix,
SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
}
}
@@ -1414,7 +1588,8 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) {
fd->cursor_objix_pix = new_objix_pix;
fd->cursor_objix_spix = cur_objix_spix;
SPIFFS_CHECK_RES(res2);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix,
SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
} else {
// wrote within object index header page
@@ -1547,7 +1722,8 @@ s32_t spiffs_object_truncate(
res = spiffs_page_delete(fs, objix_pix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_DEL, fd->obj_id, objix->p_hdr.span_ix, objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0,
SPIFFS_EV_IX_DEL, fd->obj_id, objix->p_hdr.span_ix, objix_pix, 0);
if (prev_objix_spix > 0) {
// Update object index header page, unless we totally want to remove the file.
// If fully removing, we're not keeping consistency as good as when storing the header between chunks,
@@ -1687,7 +1863,8 @@ s32_t spiffs_object_truncate(
res = spiffs_page_delete(fs, objix_pix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_DEL, fd->obj_id, 0, objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)0,
SPIFFS_EV_IX_DEL, fd->obj_id, 0, objix_pix, 0);
} else {
// make uninitialized object
SPIFFS_DBG("truncate: reset objix_hdr page %04x\n", objix_pix);
@@ -1714,7 +1891,8 @@ s32_t spiffs_object_truncate(
// move and update object index page
res = spiffs_page_move(fs, fd->file_nbr, (u8_t*)objix_hdr, fd->obj_id, 0, objix_pix, &new_objix_pix);
SPIFFS_CHECK_RES(res);
spiffs_cb_object_event(fs, fd, SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
spiffs_cb_object_event(fs, (spiffs_page_object_ix *)objix_hdr,
SPIFFS_EV_IX_UPD, fd->obj_id, objix->p_hdr.span_ix, new_objix_pix, 0);
SPIFFS_DBG("truncate: store modified objix page, %04x:%04x\n", new_objix_pix, cur_objix_spix);
fd->cursor_objix_pix = new_objix_pix;
fd->cursor_objix_spix = cur_objix_spix;
@@ -1747,41 +1925,51 @@ s32_t spiffs_object_read(
spiffs_page_object_ix *objix = (spiffs_page_object_ix *)fs->work;
while (cur_offset < offset + len) {
cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix);
if (prev_objix_spix != cur_objix_spix) {
// load current object index (header) page
if (cur_objix_spix == 0) {
objix_pix = fd->objix_hdr_pix;
} else {
SPIFFS_DBG("read: find objix %04x:%04x\n", fd->obj_id, cur_objix_spix);
if (fd->cursor_objix_spix == cur_objix_spix) {
objix_pix = fd->cursor_objix_pix;
} else {
res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &objix_pix);
SPIFFS_CHECK_RES(res);
}
}
SPIFFS_DBG("read: load objix page %04x:%04x for data spix:%04x\n", objix_pix, cur_objix_spix, data_spix);
res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ,
fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
SPIFFS_CHECK_RES(res);
SPIFFS_VALIDATE_OBJIX(objix->p_hdr, fd->obj_id, cur_objix_spix);
fd->offset = cur_offset;
fd->cursor_objix_pix = objix_pix;
fd->cursor_objix_spix = cur_objix_spix;
prev_objix_spix = cur_objix_spix;
}
if (cur_objix_spix == 0) {
// get data page from object index header page
data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix];
#if SPIFFS_IX_MAP
// check if we have a memory, index map and if so, if we're within index map's range
// and if so, if the entry is populated
if (fd->ix_map && data_spix >= fd->ix_map->start_spix && data_spix <= fd->ix_map->end_spix
&& fd->ix_map->map_buf[data_spix - fd->ix_map->start_spix]) {
data_pix = fd->ix_map->map_buf[data_spix - fd->ix_map->start_spix];
} else {
// get data page from object index page
data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)];
}
#endif
cur_objix_spix = SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, data_spix);
if (prev_objix_spix != cur_objix_spix) {
// load current object index (header) page
if (cur_objix_spix == 0) {
objix_pix = fd->objix_hdr_pix;
} else {
SPIFFS_DBG("read: find objix %04x:%04x\n", fd->obj_id, cur_objix_spix);
if (fd->cursor_objix_spix == cur_objix_spix) {
objix_pix = fd->cursor_objix_pix;
} else {
res = spiffs_obj_lu_find_id_and_span(fs, fd->obj_id | SPIFFS_OBJ_ID_IX_FLAG, cur_objix_spix, 0, &objix_pix);
SPIFFS_CHECK_RES(res);
}
}
SPIFFS_DBG("read: load objix page %04x:%04x for data spix:%04x\n", objix_pix, cur_objix_spix, data_spix);
res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ,
fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, objix_pix), SPIFFS_CFG_LOG_PAGE_SZ(fs), fs->work);
SPIFFS_CHECK_RES(res);
SPIFFS_VALIDATE_OBJIX(objix->p_hdr, fd->obj_id, cur_objix_spix);
fd->offset = cur_offset;
fd->cursor_objix_pix = objix_pix;
fd->cursor_objix_spix = cur_objix_spix;
prev_objix_spix = cur_objix_spix;
}
if (cur_objix_spix == 0) {
// get data page from object index header page
data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix];
} else {
// get data page from object index page
data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)];
}
#if SPIFFS_IX_MAP
}
#endif
// all remaining data
u32_t len_to_read = offset + len - cur_offset;
// remaining data in page
@@ -2084,6 +2272,9 @@ s32_t spiffs_fd_return(spiffs *fs, spiffs_file f) {
return SPIFFS_ERR_FILE_CLOSED;
}
fd->file_nbr = 0;
#if SPIFFS_IX_MAP
fd->ix_map = 0;
#endif
return SPIFFS_OK;
}
src/spiffs_nucleus.h
+32 -6
View File
@@ -116,13 +116,23 @@
#define SPIFFS_ERR_CHECK_FLAGS_BAD (SPIFFS_ERR_INTERNAL - 3)
#define _SPIFFS_ERR_CHECK_LAST (SPIFFS_ERR_INTERNAL - 4)
// visitor result, continue searching
#define SPIFFS_VIS_COUNTINUE (SPIFFS_ERR_INTERNAL - 20)
// visitor result, continue searching after reloading lu buffer
#define SPIFFS_VIS_COUNTINUE_RELOAD (SPIFFS_ERR_INTERNAL - 21)
// visitor result, stop searching
#define SPIFFS_VIS_END (SPIFFS_ERR_INTERNAL - 22)
#define SPIFFS_EV_IX_UPD 0
#define SPIFFS_EV_IX_NEW 1
#define SPIFFS_EV_IX_DEL 2
// updating an object index contents
#define SPIFFS_EV_IX_UPD (0)
// creating a new object index
#define SPIFFS_EV_IX_NEW (1)
// deleting an object index
#define SPIFFS_EV_IX_DEL (2)
// moving an object index without updating contents
#define SPIFFS_EV_IX_MOV (3)
// updating an object index header data only, not the table itself
#define SPIFFS_EV_IX_UPD_HDR (4)
#define SPIFFS_OBJ_ID_IX_FLAG ((spiffs_obj_id)(1<<(8*sizeof(spiffs_obj_id)-1)))
@@ -228,7 +238,9 @@
// object index span index number for given data span index or entry
#define SPIFFS_OBJ_IX_ENTRY_SPAN_IX(fs, spix) \
((spix) < SPIFFS_OBJ_HDR_IX_LEN(fs) ? 0 : (1+((spix)-SPIFFS_OBJ_HDR_IX_LEN(fs))/SPIFFS_OBJ_IX_LEN(fs)))
// get data span index for object index span index
#define SPIFFS_DATA_SPAN_IX_FOR_OBJ_IX_SPAN_IX(fs, spix) \
( (spix) == 0 ? 0 : (SPIFFS_OBJ_HDR_IX_LEN(fs) + (((spix)-1) * SPIFFS_OBJ_IX_LEN(fs))) )
#define SPIFFS_OP_T_OBJ_LU (0<<0)
#define SPIFFS_OP_T_OBJ_LU2 (1<<0)
@@ -312,7 +324,7 @@
if ((ph).span_ix != (spix)) return SPIFFS_ERR_DATA_SPAN_MISMATCH;
// check id
// check id, only visit matching objec ids
#define SPIFFS_VIS_CHECK_ID (1<<0)
// report argument object id to visitor - else object lookup id is reported
#define SPIFFS_VIS_CHECK_PH (1<<1)
@@ -431,6 +443,10 @@ typedef struct {
// hit score (score == 0 indicates never used fd)
u16_t score;
#endif
#if SPIFFS_IX_MAP
// spiffs index map, if 0 it means unmapped
spiffs_ix_map *ix_map;
#endif
} spiffs_fd;
@@ -632,9 +648,19 @@ s32_t spiffs_object_update_index_hdr(
u32_t size,
spiffs_page_ix *new_pix);
void spiffs_cb_object_event(
#if SPIFFS_IX_MAP
s32_t spiffs_populate_ix_map(
spiffs *fs,
spiffs_fd *fd,
u32_t vec_entry_start,
u32_t vec_entry_end);
#endif
void spiffs_cb_object_event(
spiffs *fs,
spiffs_page_object_ix *objix,
int ev,
spiffs_obj_id obj_id,
spiffs_span_ix spix,
src/test/test_hydrogen.c
+353
View File
@@ -1894,6 +1894,353 @@ TEST(long_run)
}
TEST_END
#if SPIFFS_IX_MAP
TEST(ix_map_basic)
{
// create a scattered file
s32_t res;
spiffs_file fd1, fd2;
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
TEST_CHECK_GT(fd1, 0);
fd2 = SPIFFS_open(FS, "2", SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
TEST_CHECK_GT(fd2, 0);
u8_t buf[SPIFFS_DATA_PAGE_SIZE(FS)];
int i;
for (i = 0; i < SPIFFS_CFG_PHYS_SZ(FS) / 4 / SPIFFS_DATA_PAGE_SIZE(FS); i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd1, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd2, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
res = SPIFFS_close(FS, fd1);
TEST_CHECK_GE(res, SPIFFS_OK);
res = SPIFFS_close(FS, fd2);
TEST_CHECK_GE(res, SPIFFS_OK);
res = SPIFFS_remove(FS, "2");
TEST_CHECK_GE(res, SPIFFS_OK);
spiffs_stat s;
res = SPIFFS_stat(FS, "1", &s);
TEST_CHECK_GE(res, SPIFFS_OK);
u32_t size = s.size;
printf("file created, size: %i..\n", size);
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_RDONLY, 0);
TEST_CHECK_GT(fd1, 0);
printf(".. corresponding pix entries: %i\n", SPIFFS_bytes_to_ix_map_entries(FS, size));
u8_t rd_buf[SPIFFS_CFG_LOG_PAGE_SZ(FS)];
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_RDONLY, 0);
TEST_CHECK_GT(fd1, 0);
clear_flash_ops_log();
printf("reading file without memory mapped index\n");
while ((res = SPIFFS_read(FS, fd1, rd_buf, sizeof(rd_buf))) == sizeof(rd_buf));
TEST_CHECK_GT(res, SPIFFS_OK);
res = SPIFFS_OK;
u32_t reads_without_ixmap = get_flash_ops_log_read_bytes();
dump_flash_access_stats();
u32_t crc_non_map_ix = get_spiffs_file_crc_by_fd(fd1);
TEST_CHECK_EQ(SPIFFS_close(FS, fd1), SPIFFS_OK);
printf("reading file with memory mapped index\n");
spiffs_ix_map map;
spiffs_page_ix ixbuf[SPIFFS_bytes_to_ix_map_entries(FS, size)];
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_RDONLY, 0);
TEST_CHECK_GT(fd1, 0);
// map index to memory
res = SPIFFS_ix_map(FS, fd1, &map, 0, size, ixbuf);
TEST_CHECK_GE(res, SPIFFS_OK);
clear_flash_ops_log();
while ((res = SPIFFS_read(FS, fd1, rd_buf, sizeof(rd_buf))) == sizeof(rd_buf));
TEST_CHECK_GT(res, SPIFFS_OK);
u32_t reads_with_ixmap_pass1 = get_flash_ops_log_read_bytes();
dump_flash_access_stats();
u32_t crc_map_ix_pass1 = get_spiffs_file_crc_by_fd(fd1);
TEST_CHECK_LT(reads_with_ixmap_pass1, reads_without_ixmap);
TEST_CHECK_EQ(crc_non_map_ix, crc_map_ix_pass1);
spiffs_page_ix ref_ixbuf[SPIFFS_bytes_to_ix_map_entries(FS, size)];
memcpy(ref_ixbuf, ixbuf, sizeof(ixbuf));
// force a gc by creating small files until full, reordering the index
printf("forcing gc, error ERR_FULL %i expected\n", SPIFFS_ERR_FULL);
res = SPIFFS_OK;
u32_t ix = 10;
while (res == SPIFFS_OK) {
char name[32];
sprintf(name, "%i", ix);
res = test_create_and_write_file(name, SPIFFS_CFG_LOG_BLOCK_SZ(FS), SPIFFS_CFG_LOG_BLOCK_SZ(FS));
ix++;
}
TEST_CHECK_EQ(SPIFFS_errno(FS), SPIFFS_ERR_FULL);
// make sure the map array was altered
TEST_CHECK_NEQ(0, memcmp(ref_ixbuf, ixbuf, sizeof(ixbuf)));
TEST_CHECK_GE(SPIFFS_lseek(FS, fd1, 0, SPIFFS_SEEK_SET), SPIFFS_OK);
clear_flash_ops_log();
while ((res = SPIFFS_read(FS, fd1, rd_buf, sizeof(rd_buf))) == sizeof(rd_buf));
TEST_CHECK_GT(res, SPIFFS_OK);
u32_t reads_with_ixmap_pass2 = get_flash_ops_log_read_bytes();
TEST_CHECK_EQ(reads_with_ixmap_pass1, reads_with_ixmap_pass2);
u32_t crc_map_ix_pass2 = get_spiffs_file_crc_by_fd(fd1);
TEST_CHECK_EQ(crc_map_ix_pass1, crc_map_ix_pass2);
TEST_CHECK_EQ(SPIFFS_close(FS, fd1), SPIFFS_OK);
return TEST_RES_OK;
}
TEST_END
TEST(ix_map_remap)
{
// create a file, 10 data pages long
s32_t res;
spiffs_file fd1, fd2;
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
TEST_CHECK_GT(fd1, 0);
const int size_pages = 10;
u8_t buf[SPIFFS_DATA_PAGE_SIZE(FS)];
int i;
for (i = 0; i < size_pages; i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd1, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
res = SPIFFS_close(FS, fd1);
TEST_CHECK_GE(res, SPIFFS_OK);
spiffs_stat s;
res = SPIFFS_stat(FS, "1", &s);
TEST_CHECK_GE(res, SPIFFS_OK);
u32_t size = s.size;
printf("file created, size: %i..\n", size);
fd1 = SPIFFS_open(FS, "1", SPIFFS_O_RDONLY, 0);
TEST_CHECK_GT(fd1, 0);
printf(".. corresponding pix entries: %i\n", SPIFFS_bytes_to_ix_map_entries(FS, size));
TEST_CHECK_EQ(SPIFFS_bytes_to_ix_map_entries(FS, size), size_pages);
// map index to memory
// move around, check validity
const int entries = SPIFFS_bytes_to_ix_map_entries(FS, size/2);
spiffs_ix_map map;
spiffs_page_ix ixbuf[entries];
spiffs_page_ix ixbuf_ref[entries];
res = SPIFFS_ix_map(FS, fd1, &map, 0, size/2, ixbuf);
TEST_CHECK_GE(res, SPIFFS_OK);
memcpy(ixbuf_ref, ixbuf, sizeof(ixbuf));
TEST_CHECK_EQ(SPIFFS_ix_remap(FS, fd1, 0), SPIFFS_OK);
TEST_CHECK_EQ(0, memcmp(ixbuf_ref, ixbuf, sizeof(ixbuf)));
TEST_CHECK_EQ(SPIFFS_ix_remap(FS, fd1, SPIFFS_DATA_PAGE_SIZE(FS)), SPIFFS_OK);
TEST_CHECK_EQ(0, memcmp(&ixbuf_ref[1], ixbuf, sizeof(spiffs_page_ix) * (entries-1)));
TEST_CHECK_EQ(SPIFFS_ix_remap(FS, fd1, 0), SPIFFS_OK);
TEST_CHECK_EQ(0, memcmp(ixbuf_ref, ixbuf, sizeof(ixbuf)));
TEST_CHECK_EQ(SPIFFS_ix_remap(FS, fd1, size/2), SPIFFS_OK);
for (i = 0; i < entries; i++) {
int j;
for (j = 0; j < entries; j++) {
TEST_CHECK_NEQ(ixbuf_ref[i], ixbuf[j]);
}
}
return TEST_RES_OK;
}
TEST_END
TEST(ix_map_partial)
{
// create a file, 10 data pages long
s32_t res;
spiffs_file fd, fd2;
fd = SPIFFS_open(FS, "1", SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
TEST_CHECK_GT(fd, 0);
const int size_pages = 10;
u8_t buf[SPIFFS_DATA_PAGE_SIZE(FS)];
int i;
for (i = 0; i < size_pages; i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
res = SPIFFS_close(FS, fd);
TEST_CHECK_GE(res, SPIFFS_OK);
spiffs_stat s;
res = SPIFFS_stat(FS, "1", &s);
TEST_CHECK_GE(res, SPIFFS_OK);
u32_t size = s.size;
printf("file created, size: %i..\n", size);
const u32_t crc_unmapped = get_spiffs_file_crc("1");
fd = SPIFFS_open(FS, "1", SPIFFS_O_RDONLY, 0);
TEST_CHECK_GT(fd, 0);
// map index to memory
const int entries = SPIFFS_bytes_to_ix_map_entries(FS, size/2);
spiffs_ix_map map;
spiffs_page_ix ixbuf[entries];
spiffs_page_ix ixbuf_ref[entries];
printf("map 0-50%%\n");
res = SPIFFS_ix_map(FS, fd, &map, 0, size/2, ixbuf);
TEST_CHECK_GE(res, SPIFFS_OK);
const u32_t crc_mapped_beginning = get_spiffs_file_crc_by_fd(fd);
TEST_CHECK_EQ(crc_mapped_beginning, crc_unmapped);
printf("map 25-75%%\n");
res = SPIFFS_ix_remap(FS, fd, size/4);
TEST_CHECK_GE(res, SPIFFS_OK);
const u32_t crc_mapped_middle = get_spiffs_file_crc_by_fd(fd);
TEST_CHECK_EQ(crc_mapped_middle, crc_unmapped);
printf("map 50-100%%\n");
res = SPIFFS_ix_remap(FS, fd, size/2);
TEST_CHECK_GE(res, SPIFFS_OK);
const u32_t crc_mapped_end = get_spiffs_file_crc_by_fd(fd);
TEST_CHECK_EQ(crc_mapped_end, crc_unmapped);
return TEST_RES_OK;
}
TEST_END
TEST(ix_map_beyond)
{
// create a file, 10 data pages long
s32_t res;
spiffs_file fd;
fd = SPIFFS_open(FS, "1", SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
TEST_CHECK_GT(fd, 0);
const int size_pages = 10;
u8_t buf[SPIFFS_DATA_PAGE_SIZE(FS)];
int i;
for (i = 0; i < size_pages; i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
res = SPIFFS_close(FS, fd);
TEST_CHECK_GE(res, SPIFFS_OK);
spiffs_stat s;
res = SPIFFS_stat(FS, "1", &s);
TEST_CHECK_GE(res, SPIFFS_OK);
u32_t size = s.size;
printf("file created, size: %i..\n", size);
// map index to memory
fd = SPIFFS_open(FS, "1", SPIFFS_O_RDWR | SPIFFS_O_APPEND, 0);
TEST_CHECK_GT(fd, 0);
const int entries = SPIFFS_bytes_to_ix_map_entries(FS, size);
spiffs_ix_map map;
spiffs_page_ix ixbuf[entries];
printf("map 100-200%%\n");
res = SPIFFS_ix_map(FS, fd, &map, size, size*2, ixbuf);
TEST_CHECK_GE(res, SPIFFS_OK);
// make sure map is empty
for (i = 0; i < entries; i++) {
TEST_CHECK_EQ(ixbuf[i], 0);
}
printf("elongate by 100%%\n");
for (i = 0; i < size_pages; i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
TEST_CHECK_GE(SPIFFS_fflush(FS, fd), SPIFFS_OK);
res = SPIFFS_stat(FS, "1", &s);
TEST_CHECK_GE(res, SPIFFS_OK);
size = s.size;
printf("file elongated, size: %i..\n", size);
// make sure map is non-empty
for (i = 0; i < entries; i++) {
TEST_CHECK_NEQ(ixbuf[i], 0);
}
printf("remap till end\n");
TEST_CHECK_EQ(SPIFFS_ix_remap(FS, fd, size), SPIFFS_OK);
// make sure map is empty but for one element
int nonzero = 0;
for (i = 0; i < entries; i++) {
if (ixbuf[i]) nonzero++;
}
TEST_CHECK_LE(1, nonzero);
printf("elongate again, by other fd\n");
spiffs_file fd2 = SPIFFS_open(FS, "1", SPIFFS_O_WRONLY | SPIFFS_O_APPEND, 0);
TEST_CHECK_GT(fd2, 0);
for (i = 0; i < size_pages; i++) {
memrand(buf, sizeof(buf));
res = SPIFFS_write(FS, fd2, buf, sizeof(buf));
TEST_CHECK_GE(res, SPIFFS_OK);
}
TEST_CHECK_GE(SPIFFS_close(FS, fd2), SPIFFS_OK);
// make sure map is non-empty
for (i = 0; i < entries; i++) {
TEST_CHECK_NEQ(ixbuf[i], 0);
}
return TEST_RES_OK;
}
TEST_END
#endif // SPIFFS_IX_MAP
SUITE_TESTS(hydrogen_tests)
ADD_TEST(info)
#if SPIFFS_USE_MAGIC
@@ -1956,6 +2303,12 @@ SUITE_TESTS(hydrogen_tests)
ADD_TEST(long_run_config_many_medium)
ADD_TEST(long_run_config_many_small)
ADD_TEST(long_run)
#if SPIFFS_IX_MAP
ADD_TEST(ix_map_basic)
ADD_TEST(ix_map_remap)
ADD_TEST(ix_map_partial)
ADD_TEST(ix_map_beyond)
#endif
SUITE_END(hydrogen_tests)
src/test/test_spiffs.c
+85
View File
@@ -750,6 +750,91 @@ int test_create_and_write_file(char *name, int size, int chunk_size) {
return 0;
}
static u32_t crc32_tab[] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
};
static u32_t crc32(u32_t crc, const void *buf, size_t size)
{
const u8_t *p;
p = buf;
crc = crc ^ ~0U;
while (size--)
crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
return crc ^ ~0U;
}
u32_t get_spiffs_file_crc_by_fd(spiffs_file fd) {
s32_t res;
u32_t crc = 0;
u8_t buf[256];
ASSERT(SPIFFS_lseek(FS, fd, 0, SPIFFS_SEEK_SET) >= 0, "could not seek to start of file");
while ((res = SPIFFS_read(FS, fd, buf, sizeof(buf))) >= SPIFFS_OK) {
crc = crc32(crc, buf, res);
}
ASSERT(res == SPIFFS_ERR_END_OF_OBJECT || res == SPIFFS_OK, "failed reading file");
return crc;
}
u32_t get_spiffs_file_crc(char *name) {
s32_t res;
spiffs_file fd;
fd = SPIFFS_open(FS, name, SPIFFS_O_RDONLY, 0);
ASSERT(fd >= 0, "Could not open file");
u32_t crc = get_spiffs_file_crc_by_fd(fd);
res = SPIFFS_close(FS, fd);
ASSERT(res >= SPIFFS_OK, "failing closing file");
return crc;
}
#if SPIFFS_CACHE
#if SPIFFS_CACHE_STATS
static u32_t chits_tot = 0;
src/test/test_spiffs.h
+2 -1
View File
@@ -55,7 +55,6 @@ typedef struct {
char name[32];
} tfile;
void fs_reset();
void fs_reset_specific(u32_t addr_offset, u32_t phys_addr, u32_t phys_size,
u32_t phys_sector_size,
@@ -94,6 +93,8 @@ void fs_set_validate_flashing(int i);
void memrand(u8_t *b, int len);
int test_create_file(char *name);
int test_create_and_write_file(char *name, int size, int chunk_size);
u32_t get_spiffs_file_crc_by_fd(spiffs_file fd);
u32_t get_spiffs_file_crc(char *name);
void _setup();
void _setup_test_only();
void _teardown();