Gitweb links: ...log http://git.netsurf-browser.org/netsurf.git/shortlog/ac28190fd677fd6a8364b... ...commit http://git.netsurf-browser.org/netsurf.git/commit/ac28190fd677fd6a8364beb... ...tree http://git.netsurf-browser.org/netsurf.git/tree/ac28190fd677fd6a8364bebba... The branch, vince/prstllcache has been updated via ac28190fd677fd6a8364bebbaf5dd9a4126c580c (commit) via 77b704be9966f3969838d2d14abfeafd10aff75c (commit) via 7bae620e9e51618aa0eada5dce7835b30ce4bea8 (commit) via da0925041385197ca4c3ba6b1a10edbe0cf9e6f8 (commit) from 8b617a0dde2c50c28f3bfa2d61877cfe747127e0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=ac28190fd677fd6a836... commit ac28190fd677fd6a8364bebbaf5dd9a4126c580c Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; add initial writeout ordering generator diff --git a/content/llcache.c b/content/llcache.c index 6025a16..a4d17cd 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -2103,17 +2103,54 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) * LLCACHE_MIN_DISC_LIFETIME time are simply not considered, they will * become stale before pushing to backing store is worth the cost. * - * The list is sorted by object size, largest first - * * \todo calculate useful cost metrics to improve sorting * */ -#if 0 -static struct llcache_object** -build_candidate_list() +static nserror +build_candidate_list(struct llcache_object **lst_out, int *lst_len_out) { + llcache_object *object, *next; + struct llcache_object **lst; + int lst_len = 0; + + lst = calloc(512, sizeof(struct llcache_object *)); + if (lst == NULL) + return NSERROR_NOMEM; + + for (object = llcache->cached_objects; object != NULL; object = next) { + next = object->next; + + remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache); + + /* cacehable objects with no pending fetches, not + * already on disc and with sufficient lifetime to + * make disc cache worthwile + */ + if ((object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false) && + (object->store_state == LLCACHE_STATE_RAM) && + (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + lst[lst_len] = object; + lst_len++; + if (lst_len == 512) + break; + } + } + + /* sort list here */ + + *lst_len_out = lst_len; + + if (lst_len == 0) { + free(lst); + *lst_out = NULL; + } else { + *lst_out = lst; + } + + return NSERROR_OK; } -#endif /** * possibly push objects data to persiatant storage. commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=77b704be9966f396983... commit 77b704be9966f3969838d2d14abfeafd10aff75c Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; make backing store invalidate work diff --git a/gtk/llcache.c b/gtk/llcache.c index 0d761fe..9c5b78a 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -275,6 +275,15 @@ fetch(nsurl *url, static nserror invalidate(nsurl *url) { + char *fname; + fname = store_name(url, LLCACHE_STORE_META); + unlink(fname); + free(fname); + + fname = store_name(url, LLCACHE_STORE_NONE); + unlink(fname); + free(fname); + return NSERROR_OK; } commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=7bae620e9e51618aa0e... commit 7bae620e9e51618aa0eada5dce7835b30ce4bea8 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; make backing store use new API diff --git a/gtk/llcache.c b/gtk/llcache.c index e4572cc..0d761fe 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -30,14 +30,16 @@ #include "gtk/llcache.h" -static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', - 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_'}; +static uint8_t encoding_table[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_' +}; static unsigned int mod_table[] = {0, 2, 1}; @@ -103,7 +105,9 @@ initialise(const struct llcache_store_parameters *parameters) return NSERROR_NOMEM; } - memcpy(&storestate->params, parameters, sizeof(struct llcache_store_parameters)); + memcpy(&storestate->params, + parameters, + sizeof(struct llcache_store_parameters)); return NSERROR_OK; } @@ -123,17 +127,17 @@ finalise(void) return NSERROR_OK; } -static char *store_name(nsurl *url, bool metadata) +static char *store_name(nsurl *url, enum llcache_store_flags flags) { char *fname; uint8_t *b64u; size_t b64ulen; char sep; - if (metadata) { - sep ='i'; - } else { + if ((flags & LLCACHE_STORE_META) == 0) { sep ='d'; + } else { + sep ='i'; } switch (storestate->params.hashsize) { @@ -162,22 +166,22 @@ static char *store_name(nsurl *url, bool metadata) * Place a source object and its metadata in the backing store. * * @param url The url is used as the unique primary key for the data. - * @param metadata The metadata associated with the object. - * @param metadatalen The length of the \a metadata. + * @param flags The flags to control how the object is stored. * @param data The objects source data. * @param datalen The length of the \a data. * @return NSERROR_OK on success or error code on faliure. */ static nserror store(nsurl *url, - const uint8_t *metadata, const size_t metadatalen, - const uint8_t *data, const size_t datalen) + enum llcache_store_flags flags, + const uint8_t *data, + const size_t datalen) { char *fname; FILE *file; - LOG(("Writing cache file for url:%s", nsurl_access(url))); - fname = store_name(url, false); + LOG(("Writing object for url:%s", nsurl_access(url))); + fname = store_name(url, flags); LOG(("Opening data file %s", fname)); file = fopen(fname, "wb"); @@ -195,30 +199,23 @@ store(nsurl *url, fclose(file); - - fname = store_name(url, true); - LOG(("Opening info file %s", fname)); - - file = fopen(fname, "wb"); - free(fname); - if (file == NULL) { - return NSERROR_SAVE_FAILED; - } - - LOG(("Writing %d bytes from %p", metadatalen, metadata)); - if (fwrite(metadata, metadatalen, 1, file) != 1) { - LOG(("did not return 1")); - fclose(file); - return NSERROR_SAVE_FAILED; - } - - fclose(file); - return NSERROR_OK; } +/** + * Retrive an object from the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param flags The flags to control how the object is stored. + * @param data The objects data. + * @param datalen The length of the \a data retrieved. + * @return NSERROR_OK on success or error code on faliure. + */ static nserror -fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) +fetch(nsurl *url, + enum llcache_store_flags *flags, + uint8_t **data_out, + size_t *datalen_out) { char *fname; FILE *file; @@ -230,7 +227,7 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) LOG(("retriving cache file for url:%s", nsurl_access(url))); - fname = store_name(url, false); + fname = store_name(url, *flags); LOG(("Opening file %s",fname)); @@ -274,63 +271,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) return NSERROR_OK; } -static nserror -fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) -{ - char *fname; - FILE *file; - uint8_t *data; - size_t datalen; - - data = *metadata_out; - datalen = *metadatalen_out; - - LOG(("retriving cache file for url:%s", nsurl_access(url))); - - fname = store_name(url, true); - - LOG(("Opening file %s",fname)); - - file = fopen(fname, "rb"); - free(fname); - if (file == NULL) { - return NSERROR_NOT_FOUND; - } - - /* need to deal with allocating buffers */ - if (data == NULL) { - if (datalen == 0) { - /* caller did not know the files length */ - fseek(file, 0L, SEEK_END); - datalen = ftell(file); - fseek(file, 0L, SEEK_SET); - } - - data = malloc(datalen); - if (data == NULL) { - fclose(file); - return NSERROR_NOMEM; - } - } - - LOG(("Reading %d bytes into %p from file", datalen, data)); - if (fread(data, datalen, 1, file) != 1) { - LOG(("did not return 1")); - - fclose(file); - if ((*metadata_out) == NULL) { - free(data); - } - return NSERROR_NOT_FOUND; - } - - fclose(file); - - *metadata_out = data; - *metadatalen_out = datalen; - - return NSERROR_OK; -} static nserror invalidate(nsurl *url) @@ -344,7 +284,6 @@ static struct gui_llcache_table llcache_table = { .finalise = finalise, .store = store, .fetch = fetch, - .meta = fetchmeta, .invalidate = invalidate, }; commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=da0925041385197ca4c... commit da0925041385197ca4c3ba6b1a10edbe0cf9e6f8 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; improve backing store API diff --git a/content/llcache.c b/content/llcache.c index e60c0c6..6025a16 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -162,9 +162,9 @@ typedef struct { /** Current status of objects data */ typedef enum { - LLCACHE_STORE_RAM = 0, /**< sourec data is stored in RAM only */ - LLCACHE_STORE_MMAP, /**< source data is mmaped (implies on disc too) */ - LLCACHE_STORE_DISC, /**< source data is stored on disc */ + LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */ + LLCACHE_STATE_MMAP, /**< source data is mmaped (implies on disc too) */ + LLCACHE_STATE_DISC, /**< source data is stored on disc */ } llcache_store_state; /** Low-level cache object */ @@ -1075,9 +1075,11 @@ llcache_object_remove_from_list(llcache_object *object, llcache_object **list) */ static nserror llcache_persist_retrieve(llcache_object *object) { + enum llcache_store_flags flags = LLCACHE_STORE_NONE; + /* ensure the source data is present if necessary */ if ((object->source_data != NULL) || - (object->store_state != LLCACHE_STORE_DISC)) { + (object->store_state != LLCACHE_STATE_DISC)) { /* source data does not require retriving from * persistant store. */ @@ -1086,6 +1088,7 @@ static nserror llcache_persist_retrieve(llcache_object *object) /* Source data for the object may be in the persiatant store */ return guit->llcache->fetch(object->url, + &flags, &object->source_data, &object->source_len); } @@ -1222,9 +1225,13 @@ llcache_process_metadata(llcache_object *object) size_t num_headers; size_t hloop; struct tm ltm; + enum llcache_store_flags flags = LLCACHE_STORE_META; /* attempt to retrieve object metadata from the cache */ - res = guit->llcache->meta(object->url, &metadata, &metadatalen); + res = guit->llcache->fetch(object->url, + &flags, + &metadata, + &metadatalen); if (res != NSERROR_OK) return res; @@ -1272,7 +1279,7 @@ llcache_process_metadata(llcache_object *object) if ((lnsize < 1) || (sscanf(ln, "%zu", &object->source_len) != 1)) goto format_error; - object->source_alloc = object->source_len; + object->source_alloc = metadatalen; /* metadata line 3 is the time of request */ line = 3; @@ -1331,7 +1338,7 @@ llcache_process_metadata(llcache_object *object) free(metadata); /* object stored in backing store */ - object->store_state = LLCACHE_STORE_DISC; + object->store_state = LLCACHE_STATE_DISC; return NSERROR_OK; @@ -2130,35 +2137,53 @@ static void llcache_persist(void *p) if ((object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_RAM) && + (object->store_state == LLCACHE_STATE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { uint8_t *metadata; size_t metadatasize; - ret = llcache_serialise_metadata(object, &metadata, &metadatasize); + /* ok found an object to write */ + ret = guit->llcache->store(object->url, + LLCACHE_STORE_NONE, + object->source_data, + object->source_len); if (ret != NSERROR_OK) { - /* as there has been a serialisation - * error, give up on making any more + /* unable to put source data in backing + * store, give up on making any more * objects persistant for now. */ return; } - /* ok found an object to write */ + ret = llcache_serialise_metadata(object, + &metadata, + &metadatasize); + if (ret != NSERROR_OK) { + /* There has been a metadata + * serialisation error. ensure we + * invalidate the already written data + * object and give up on making any + * more objects persistant. + */ + guit->llcache->invalidate(object->url); + return; + } + ret = guit->llcache->store(object->url, + LLCACHE_STORE_META, metadata, - metadatasize, - object->source_data, - object->source_len); + metadatasize); free(metadata); if (ret != NSERROR_OK) { - /* as there has been a serialisation - * error, give up on making any more - * objects persistant for now. + /* There has been an error putting the + * metadata in the backing store. Ensure + * the data object is removed and give up + * on making any more objects persistant. */ + guit->llcache->invalidate(object->url); return; } - object->store_state = LLCACHE_STORE_DISC; + object->store_state = LLCACHE_STATE_DISC; size_written += object->source_len + metadatasize; if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { @@ -2761,7 +2786,7 @@ void llcache_clean(void) llcache_object_remove_from_list(object, &llcache->cached_objects); - if (object->store_state == LLCACHE_STORE_DISC) { + if (object->store_state == LLCACHE_STATE_DISC) { guit->llcache->invalidate(object->url); } @@ -2793,7 +2818,7 @@ void llcache_clean(void) (object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_DISC)) { + (object->store_state == LLCACHE_STATE_DISC)) { free(object->source_data); object->source_data = NULL; @@ -2817,7 +2842,7 @@ void llcache_clean(void) (object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_RAM)) { + (object->store_state == LLCACHE_STATE_RAM)) { LLCACHE_LOG(("discarding object:%p len:%d age:%d", object, object->source_len, diff --git a/content/llcache_private.h b/content/llcache_private.h index 7791b98..a946249 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,6 +25,17 @@ #include "content/llcache.h" +/** storage control flags */ +enum llcache_store_flags { + LLCACHE_STORE_NONE = 0, /**< no special processing */ + LLCACHE_STORE_META = 1, /**< data is metadata */ + LLCACHE_STORE_MMAP = 2, /**< when data is retrived this indicates the + * returned buffer may be memory mapped, + * flag must be cleared if the storage is + * allocated and is not memory mapped. + */ +}; + /** low level cache backing store operation table * * The low level cache (source objects) has the capability to make @@ -48,24 +59,28 @@ struct gui_llcache_table { nserror (*finalise)(void); /** - * Place a source object and its metadata in the backing store. + * Place an object in the backing store. * * @param url The url is used as the unique primary key for the data. - * @param metadata The metadata associated with the object. - * @param metadatalen The length of the \a metadata. - * @param data The objects source data. + * @param flags The flags to control how the obejct is stored. + * @param data The objects data. * @param datalen The length of the \a data. * @return NSERROR_OK on success or error code on faliure. */ - nserror (*store)(struct nsurl *url, - const uint8_t *metadata, const size_t metadatalen, + nserror (*store)(struct nsurl *url, enum llcache_store_flags flags, const uint8_t *data, const size_t datalen); - /** retrive source object data from backing store. */ - nserror (*fetch)(struct nsurl *url, uint8_t **data, size_t *datalen); - - /** retrive source object metadata from backing store. */ - nserror (*meta)(struct nsurl *url, uint8_t **data, size_t *datalen); + /** + * Retrive an object from the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param flags The flags to control how the object is retrived. + * @param data The objects data. + * @param datalen The length of the \a data retrieved. + * @return NSERROR_OK on success or error code on faliure. + */ + nserror (*fetch)(struct nsurl *url, enum llcache_store_flags *flags, + uint8_t **data, size_t *datalen); /** * invalidate a source object from the backing store. diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index a3772f6..079e33d 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -359,20 +359,18 @@ gui_default_finalise(void) static nserror gui_default_store(nsurl *url, - const uint8_t *metadata, const size_t metadatalen, - const uint8_t *data, const size_t datalen) + enum llcache_store_flags flags, + const uint8_t *data, + const size_t datalen) { return NSERROR_SAVE_FAILED; } static nserror -gui_default_fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) -{ - return NSERROR_NOT_FOUND; -} - -static nserror -gui_default_meta(nsurl *url, uint8_t **data_out, size_t *datalen_out) +gui_default_fetch(nsurl *url, + enum llcache_store_flags *flags, + uint8_t **data_out, + size_t *datalen_out) { return NSERROR_NOT_FOUND; } @@ -389,7 +387,6 @@ static struct gui_llcache_table default_llcache_table = { .finalise = gui_default_finalise, .store = gui_default_store, .fetch = gui_default_fetch, - .meta = gui_default_meta, .invalidate = gui_default_invalidate, }; @@ -408,9 +405,6 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) if (glt->fetch == NULL) { return NSERROR_BAD_PARAMETER; } - if (glt->meta == NULL) { - return NSERROR_BAD_PARAMETER; - } if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } ----------------------------------------------------------------------- Summary of changes: content/llcache.c | 118 +++++++++++++++++++++++++++++--------- content/llcache_private.h | 37 ++++++++---- desktop/gui_factory.c | 20 ++---- gtk/llcache.c | 140 ++++++++++++++------------------------------- 4 files changed, 167 insertions(+), 148 deletions(-) diff --git a/content/llcache.c b/content/llcache.c index e60c0c6..a4d17cd 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -162,9 +162,9 @@ typedef struct { /** Current status of objects data */ typedef enum { - LLCACHE_STORE_RAM = 0, /**< sourec data is stored in RAM only */ - LLCACHE_STORE_MMAP, /**< source data is mmaped (implies on disc too) */ - LLCACHE_STORE_DISC, /**< source data is stored on disc */ + LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */ + LLCACHE_STATE_MMAP, /**< source data is mmaped (implies on disc too) */ + LLCACHE_STATE_DISC, /**< source data is stored on disc */ } llcache_store_state; /** Low-level cache object */ @@ -1075,9 +1075,11 @@ llcache_object_remove_from_list(llcache_object *object, llcache_object **list) */ static nserror llcache_persist_retrieve(llcache_object *object) { + enum llcache_store_flags flags = LLCACHE_STORE_NONE; + /* ensure the source data is present if necessary */ if ((object->source_data != NULL) || - (object->store_state != LLCACHE_STORE_DISC)) { + (object->store_state != LLCACHE_STATE_DISC)) { /* source data does not require retriving from * persistant store. */ @@ -1086,6 +1088,7 @@ static nserror llcache_persist_retrieve(llcache_object *object) /* Source data for the object may be in the persiatant store */ return guit->llcache->fetch(object->url, + &flags, &object->source_data, &object->source_len); } @@ -1222,9 +1225,13 @@ llcache_process_metadata(llcache_object *object) size_t num_headers; size_t hloop; struct tm ltm; + enum llcache_store_flags flags = LLCACHE_STORE_META; /* attempt to retrieve object metadata from the cache */ - res = guit->llcache->meta(object->url, &metadata, &metadatalen); + res = guit->llcache->fetch(object->url, + &flags, + &metadata, + &metadatalen); if (res != NSERROR_OK) return res; @@ -1272,7 +1279,7 @@ llcache_process_metadata(llcache_object *object) if ((lnsize < 1) || (sscanf(ln, "%zu", &object->source_len) != 1)) goto format_error; - object->source_alloc = object->source_len; + object->source_alloc = metadatalen; /* metadata line 3 is the time of request */ line = 3; @@ -1331,7 +1338,7 @@ llcache_process_metadata(llcache_object *object) free(metadata); /* object stored in backing store */ - object->store_state = LLCACHE_STORE_DISC; + object->store_state = LLCACHE_STATE_DISC; return NSERROR_OK; @@ -2096,17 +2103,54 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) * LLCACHE_MIN_DISC_LIFETIME time are simply not considered, they will * become stale before pushing to backing store is worth the cost. * - * The list is sorted by object size, largest first - * * \todo calculate useful cost metrics to improve sorting * */ -#if 0 -static struct llcache_object** -build_candidate_list() +static nserror +build_candidate_list(struct llcache_object **lst_out, int *lst_len_out) { + llcache_object *object, *next; + struct llcache_object **lst; + int lst_len = 0; + + lst = calloc(512, sizeof(struct llcache_object *)); + if (lst == NULL) + return NSERROR_NOMEM; + + for (object = llcache->cached_objects; object != NULL; object = next) { + next = object->next; + + remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache); + + /* cacehable objects with no pending fetches, not + * already on disc and with sufficient lifetime to + * make disc cache worthwile + */ + if ((object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false) && + (object->store_state == LLCACHE_STATE_RAM) && + (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + lst[lst_len] = object; + lst_len++; + if (lst_len == 512) + break; + } + } + + /* sort list here */ + + *lst_len_out = lst_len; + + if (lst_len == 0) { + free(lst); + *lst_out = NULL; + } else { + *lst_out = lst; + } + + return NSERROR_OK; } -#endif /** * possibly push objects data to persiatant storage. @@ -2130,35 +2174,53 @@ static void llcache_persist(void *p) if ((object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_RAM) && + (object->store_state == LLCACHE_STATE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { uint8_t *metadata; size_t metadatasize; - ret = llcache_serialise_metadata(object, &metadata, &metadatasize); + /* ok found an object to write */ + ret = guit->llcache->store(object->url, + LLCACHE_STORE_NONE, + object->source_data, + object->source_len); if (ret != NSERROR_OK) { - /* as there has been a serialisation - * error, give up on making any more + /* unable to put source data in backing + * store, give up on making any more * objects persistant for now. */ return; } - /* ok found an object to write */ + ret = llcache_serialise_metadata(object, + &metadata, + &metadatasize); + if (ret != NSERROR_OK) { + /* There has been a metadata + * serialisation error. ensure we + * invalidate the already written data + * object and give up on making any + * more objects persistant. + */ + guit->llcache->invalidate(object->url); + return; + } + ret = guit->llcache->store(object->url, + LLCACHE_STORE_META, metadata, - metadatasize, - object->source_data, - object->source_len); + metadatasize); free(metadata); if (ret != NSERROR_OK) { - /* as there has been a serialisation - * error, give up on making any more - * objects persistant for now. + /* There has been an error putting the + * metadata in the backing store. Ensure + * the data object is removed and give up + * on making any more objects persistant. */ + guit->llcache->invalidate(object->url); return; } - object->store_state = LLCACHE_STORE_DISC; + object->store_state = LLCACHE_STATE_DISC; size_written += object->source_len + metadatasize; if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { @@ -2761,7 +2823,7 @@ void llcache_clean(void) llcache_object_remove_from_list(object, &llcache->cached_objects); - if (object->store_state == LLCACHE_STORE_DISC) { + if (object->store_state == LLCACHE_STATE_DISC) { guit->llcache->invalidate(object->url); } @@ -2793,7 +2855,7 @@ void llcache_clean(void) (object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_DISC)) { + (object->store_state == LLCACHE_STATE_DISC)) { free(object->source_data); object->source_data = NULL; @@ -2817,7 +2879,7 @@ void llcache_clean(void) (object->candidate_count == 0) && (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && - (object->store_state == LLCACHE_STORE_RAM)) { + (object->store_state == LLCACHE_STATE_RAM)) { LLCACHE_LOG(("discarding object:%p len:%d age:%d", object, object->source_len, diff --git a/content/llcache_private.h b/content/llcache_private.h index 7791b98..a946249 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,6 +25,17 @@ #include "content/llcache.h" +/** storage control flags */ +enum llcache_store_flags { + LLCACHE_STORE_NONE = 0, /**< no special processing */ + LLCACHE_STORE_META = 1, /**< data is metadata */ + LLCACHE_STORE_MMAP = 2, /**< when data is retrived this indicates the + * returned buffer may be memory mapped, + * flag must be cleared if the storage is + * allocated and is not memory mapped. + */ +}; + /** low level cache backing store operation table * * The low level cache (source objects) has the capability to make @@ -48,24 +59,28 @@ struct gui_llcache_table { nserror (*finalise)(void); /** - * Place a source object and its metadata in the backing store. + * Place an object in the backing store. * * @param url The url is used as the unique primary key for the data. - * @param metadata The metadata associated with the object. - * @param metadatalen The length of the \a metadata. - * @param data The objects source data. + * @param flags The flags to control how the obejct is stored. + * @param data The objects data. * @param datalen The length of the \a data. * @return NSERROR_OK on success or error code on faliure. */ - nserror (*store)(struct nsurl *url, - const uint8_t *metadata, const size_t metadatalen, + nserror (*store)(struct nsurl *url, enum llcache_store_flags flags, const uint8_t *data, const size_t datalen); - /** retrive source object data from backing store. */ - nserror (*fetch)(struct nsurl *url, uint8_t **data, size_t *datalen); - - /** retrive source object metadata from backing store. */ - nserror (*meta)(struct nsurl *url, uint8_t **data, size_t *datalen); + /** + * Retrive an object from the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param flags The flags to control how the object is retrived. + * @param data The objects data. + * @param datalen The length of the \a data retrieved. + * @return NSERROR_OK on success or error code on faliure. + */ + nserror (*fetch)(struct nsurl *url, enum llcache_store_flags *flags, + uint8_t **data, size_t *datalen); /** * invalidate a source object from the backing store. diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index a3772f6..079e33d 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -359,20 +359,18 @@ gui_default_finalise(void) static nserror gui_default_store(nsurl *url, - const uint8_t *metadata, const size_t metadatalen, - const uint8_t *data, const size_t datalen) + enum llcache_store_flags flags, + const uint8_t *data, + const size_t datalen) { return NSERROR_SAVE_FAILED; } static nserror -gui_default_fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) -{ - return NSERROR_NOT_FOUND; -} - -static nserror -gui_default_meta(nsurl *url, uint8_t **data_out, size_t *datalen_out) +gui_default_fetch(nsurl *url, + enum llcache_store_flags *flags, + uint8_t **data_out, + size_t *datalen_out) { return NSERROR_NOT_FOUND; } @@ -389,7 +387,6 @@ static struct gui_llcache_table default_llcache_table = { .finalise = gui_default_finalise, .store = gui_default_store, .fetch = gui_default_fetch, - .meta = gui_default_meta, .invalidate = gui_default_invalidate, }; @@ -408,9 +405,6 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) if (glt->fetch == NULL) { return NSERROR_BAD_PARAMETER; } - if (glt->meta == NULL) { - return NSERROR_BAD_PARAMETER; - } if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } diff --git a/gtk/llcache.c b/gtk/llcache.c index e4572cc..9c5b78a 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -30,14 +30,16 @@ #include "gtk/llcache.h" -static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', - 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_'}; +static uint8_t encoding_table[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_' +}; static unsigned int mod_table[] = {0, 2, 1}; @@ -103,7 +105,9 @@ initialise(const struct llcache_store_parameters *parameters) return NSERROR_NOMEM; } - memcpy(&storestate->params, parameters, sizeof(struct llcache_store_parameters)); + memcpy(&storestate->params, + parameters, + sizeof(struct llcache_store_parameters)); return NSERROR_OK; } @@ -123,17 +127,17 @@ finalise(void) return NSERROR_OK; } -static char *store_name(nsurl *url, bool metadata) +static char *store_name(nsurl *url, enum llcache_store_flags flags) { char *fname; uint8_t *b64u; size_t b64ulen; char sep; - if (metadata) { - sep ='i'; - } else { + if ((flags & LLCACHE_STORE_META) == 0) { sep ='d'; + } else { + sep ='i'; } switch (storestate->params.hashsize) { @@ -162,22 +166,22 @@ static char *store_name(nsurl *url, bool metadata) * Place a source object and its metadata in the backing store. * * @param url The url is used as the unique primary key for the data. - * @param metadata The metadata associated with the object. - * @param metadatalen The length of the \a metadata. + * @param flags The flags to control how the object is stored. * @param data The objects source data. * @param datalen The length of the \a data. * @return NSERROR_OK on success or error code on faliure. */ static nserror store(nsurl *url, - const uint8_t *metadata, const size_t metadatalen, - const uint8_t *data, const size_t datalen) + enum llcache_store_flags flags, + const uint8_t *data, + const size_t datalen) { char *fname; FILE *file; - LOG(("Writing cache file for url:%s", nsurl_access(url))); - fname = store_name(url, false); + LOG(("Writing object for url:%s", nsurl_access(url))); + fname = store_name(url, flags); LOG(("Opening data file %s", fname)); file = fopen(fname, "wb"); @@ -195,30 +199,23 @@ store(nsurl *url, fclose(file); - - fname = store_name(url, true); - LOG(("Opening info file %s", fname)); - - file = fopen(fname, "wb"); - free(fname); - if (file == NULL) { - return NSERROR_SAVE_FAILED; - } - - LOG(("Writing %d bytes from %p", metadatalen, metadata)); - if (fwrite(metadata, metadatalen, 1, file) != 1) { - LOG(("did not return 1")); - fclose(file); - return NSERROR_SAVE_FAILED; - } - - fclose(file); - return NSERROR_OK; } +/** + * Retrive an object from the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param flags The flags to control how the object is stored. + * @param data The objects data. + * @param datalen The length of the \a data retrieved. + * @return NSERROR_OK on success or error code on faliure. + */ static nserror -fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) +fetch(nsurl *url, + enum llcache_store_flags *flags, + uint8_t **data_out, + size_t *datalen_out) { char *fname; FILE *file; @@ -230,7 +227,7 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) LOG(("retriving cache file for url:%s", nsurl_access(url))); - fname = store_name(url, false); + fname = store_name(url, *flags); LOG(("Opening file %s",fname)); @@ -274,67 +271,19 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) return NSERROR_OK; } + static nserror -fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) +invalidate(nsurl *url) { char *fname; - FILE *file; - uint8_t *data; - size_t datalen; - - data = *metadata_out; - datalen = *metadatalen_out; - - LOG(("retriving cache file for url:%s", nsurl_access(url))); - - fname = store_name(url, true); - - LOG(("Opening file %s",fname)); - - file = fopen(fname, "rb"); + fname = store_name(url, LLCACHE_STORE_META); + unlink(fname); free(fname); - if (file == NULL) { - return NSERROR_NOT_FOUND; - } - /* need to deal with allocating buffers */ - if (data == NULL) { - if (datalen == 0) { - /* caller did not know the files length */ - fseek(file, 0L, SEEK_END); - datalen = ftell(file); - fseek(file, 0L, SEEK_SET); - } - - data = malloc(datalen); - if (data == NULL) { - fclose(file); - return NSERROR_NOMEM; - } - } - - LOG(("Reading %d bytes into %p from file", datalen, data)); - if (fread(data, datalen, 1, file) != 1) { - LOG(("did not return 1")); - - fclose(file); - if ((*metadata_out) == NULL) { - free(data); - } - return NSERROR_NOT_FOUND; - } - - fclose(file); - - *metadata_out = data; - *metadatalen_out = datalen; - - return NSERROR_OK; -} + fname = store_name(url, LLCACHE_STORE_NONE); + unlink(fname); + free(fname); -static nserror -invalidate(nsurl *url) -{ return NSERROR_OK; } @@ -344,7 +293,6 @@ static struct gui_llcache_table llcache_table = { .finalise = finalise, .store = store, .fetch = fetch, - .meta = fetchmeta, .invalidate = invalidate, }; -- NetSurf Browser