Gitweb links: ...log http://git.netsurf-browser.org/netsurf.git/shortlog/8b617a0dde2c50c28f3bf... ...commit http://git.netsurf-browser.org/netsurf.git/commit/8b617a0dde2c50c28f3bfa2... ...tree http://git.netsurf-browser.org/netsurf.git/tree/8b617a0dde2c50c28f3bfa2d6... The branch, vince/prstllcache has been updated via 8b617a0dde2c50c28f3bfa2d61877cfe747127e0 (commit) via c82249a95d8a8c1183f38b8b5330aaf9ca0cee14 (commit) via 39bda3deb0daccb4348b5e1742e3ed2cee3d4faf (commit) via ab4d68a95f266fe3366a69dc6c61d9686cf46526 (commit) from 5b29a5efcd996f2b218faf30433da587041ed991 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=8b617a0dde2c50c28f3... commit 8b617a0dde2c50c28f3bfa2d61877cfe747127e0 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; make backing store file names based on hash selection parameter diff --git a/content/llcache.h b/content/llcache.h index 6f12c48..854f54f 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -164,6 +164,12 @@ struct llcache_store_parameters { size_t limit; /**< The backing store upper bound target size */ size_t hysteresis; /**< The hysteresis around the target size */ + + /** length of hash to use for filename mapping. 0 indicates + * use no hash. 23 is a reasonable compromise for RISC OS. 160 + * uses sha1 in a manner similar to git. + */ + int hashsize; }; /** diff --git a/gtk/llcache.c b/gtk/llcache.c index 4fa4e1e..e4572cc 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -79,28 +79,110 @@ base64url_encode(const char *data, return encoded_data; } +struct llcache_store_state { + /** parameters controlling the backing store */ + struct llcache_store_parameters params; +}; + +struct llcache_store_state *storestate; + +/** + * Initialise the backing store. + * + * @param parameters to configure backing store. + * @return NSERROR_OK on success or error code on faliure. + */ +static nserror +initialise(const struct llcache_store_parameters *parameters) +{ + if (storestate != NULL) { + return NSERROR_INIT_FAILED; + } + storestate = calloc(1, sizeof(struct llcache_store_state)); + if (storestate == NULL) { + return NSERROR_NOMEM; + } + + memcpy(&storestate->params, parameters, sizeof(struct llcache_store_parameters)); + + return NSERROR_OK; +} + +/** + * Finalise the backing store. + * + * @return NSERROR_OK on success. + */ +static nserror +finalise(void) +{ + if (storestate != NULL) { + free(storestate); + storestate = NULL; + } + return NSERROR_OK; +} + +static char *store_name(nsurl *url, bool metadata) +{ + char *fname; + uint8_t *b64u; + size_t b64ulen; + char sep; + + if (metadata) { + sep ='i'; + } else { + sep ='d'; + } + + switch (storestate->params.hashsize) { + default: + /* no hashing, just base64 encode the url */ + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(strlen(storestate->params.path) + + SLEN("/x/") + b64ulen + 1); + + snprintf(fname, + (strlen(storestate->params.path) + + SLEN("/x/") + b64ulen + 1), + "%s/%c/%s", + storestate->params.path, + sep, + b64u); + free(b64u); + } + return fname; +} + +/** + * Place a source object and its metadata in the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param metadata The metadata associated with the object. + * @param metadatalen The length of the \a metadata. + * @param data The objects source data. + * @param datalen The length of the \a data. + * @return NSERROR_OK on success or error code on faliure. + */ static nserror store(nsurl *url, const uint8_t *metadata, const size_t metadatalen, const uint8_t *data, const size_t datalen) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; LOG(("Writing cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), strlen(nsurl_access(url)), &b64ulen); - - fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fname = store_name(url, false); LOG(("Opening data file %s", fname)); file = fopen(fname, "wb"); - + free(fname); if (file == NULL) { - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } @@ -108,21 +190,18 @@ store(nsurl *url, if (fwrite(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } fclose(file); - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + fname = store_name(url, true); LOG(("Opening info file %s", fname)); - file = fopen(fname, "wb"); + file = fopen(fname, "wb"); + free(fname); if (file == NULL) { - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } @@ -130,24 +209,17 @@ store(nsurl *url, if (fwrite(metadata, metadatalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } fclose(file); - free(fname); - free(b64u); - return NSERROR_OK; } static nserror fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; uint8_t *data; @@ -157,18 +229,14 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) datalen = *datalen_out; LOG(("retriving cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), - strlen(nsurl_access(url)), - &b64ulen); - fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fname = store_name(url, false); LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); free(fname); if (file == NULL) { - free(b64u); return NSERROR_NOT_FOUND; } @@ -184,7 +252,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) data = malloc(datalen); if (data == NULL) { fclose(file); - free(b64u); return NSERROR_NOMEM; } } @@ -193,7 +260,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) if (fread(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(b64u); if ((*data_out) == NULL) { free(data); } @@ -201,7 +267,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) } fclose(file); - free(b64u); *data_out = data; *datalen_out = datalen; @@ -212,8 +277,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) static nserror fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; uint8_t *data; @@ -223,18 +286,14 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) datalen = *metadatalen_out; LOG(("retriving cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), - strlen(nsurl_access(url)), - &b64ulen); - fname = malloc(SLEN("/tmp/ns/i/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + fname = store_name(url, true); LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); free(fname); if (file == NULL) { - free(b64u); return NSERROR_NOT_FOUND; } @@ -250,7 +309,6 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) data = malloc(datalen); if (data == NULL) { fclose(file); - free(b64u); return NSERROR_NOMEM; } } @@ -258,8 +316,8 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) LOG(("Reading %d bytes into %p from file", datalen, data)); if (fread(data, datalen, 1, file) != 1) { LOG(("did not return 1")); + fclose(file); - free(b64u); if ((*metadata_out) == NULL) { free(data); } @@ -267,7 +325,6 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) } fclose(file); - free(b64u); *metadata_out = data; *metadatalen_out = datalen; @@ -283,6 +340,8 @@ invalidate(nsurl *url) static struct gui_llcache_table llcache_table = { + .initialise = initialise, + .finalise = finalise, .store = store, .fetch = fetch, .meta = fetchmeta, commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=c82249a95d8a8c1183f... commit c82249a95d8a8c1183f38b8b5330aaf9ca0cee14 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; fix hard coded default backing store path diff --git a/desktop/netsurf.c b/desktop/netsurf.c index 582c119..a4b4da2 100644 --- a/desktop/netsurf.c +++ b/desktop/netsurf.c @@ -191,7 +191,7 @@ nserror netsurf_init(const char *messages, struct gui_table *gt) /* set the path to the backing store */ /** \todo set the backing store path properly */ - hlcache_parameters.llcache.store.path = "/tmp/ns/"; + hlcache_parameters.llcache.store.path = "/tmp/ns"; /* image handler bitmap cache */ ret = image_cache_init(&image_cache_parameters); commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=39bda3deb0daccb4348... commit 39bda3deb0daccb4348b5e1742e3ed2cee3d4faf Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; improve comments diff --git a/content/llcache.c b/content/llcache.c index 8564bfe..e60c0c6 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -23,16 +23,29 @@ * stores source objects in memory and may use a persistant backing * store to extend their lifetime. * - * \todo fix writeout conditions and ordering - * \todo implement expiry from backing store of objects not currently in cache lists - * \todo make the reference backing store have an index to improve performance - * \todo make the reference backing store have a more efficient metadata store. - * \todo make the reference backing store cope with filesystems that have filename length limits and directory entry count limits - * \todo make backing store size bounded by configuration + * \todo fix writeout conditions and ordering. + * * \todo support mmaped retrieve - * \todo make metadata buffer owned by backing store (avoid allocation and freeing) + * + * \todo make metadata buffer owned by backing store (avoid allocation + * and freeing). + * * \todo instrument and (auto)tune + * * \todo turn llcache debugging off + * + * \todo implement expiry from backing store of objects not currently + * in cache lists. + * + * \todo make backing store size bounded by configuration + * + * \todo make backing store have an index to improve performance + * + * \todo make backing store have a more efficient metadata store. + * + * \todo make backing store cope with filesystems that have filename + * length limits and directory entry count limits. + * */ #include <stdlib.h> diff --git a/content/llcache_private.h b/content/llcache_private.h index dc6cb86..7791b98 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -32,13 +32,31 @@ * backing store using these operations. */ struct gui_llcache_table { - /** Initialise the backing store. */ + /** + * Initialise the backing store. + * + * @param parameters to configure backing store. + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*initialise)(const struct llcache_store_parameters *parameters); - /** Finalise the backing store. */ + /** + * Finalise the backing store. + * + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*finalise)(void); - /** Place a source object and its metadata in the backing store. */ + /** + * Place a source object and its metadata in the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param metadata The metadata associated with the object. + * @param metadatalen The length of the \a metadata. + * @param data The objects source data. + * @param datalen The length of the \a data. + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*store)(struct nsurl *url, const uint8_t *metadata, const size_t metadatalen, const uint8_t *data, const size_t datalen); commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=ab4d68a95f266fe3366... commit ab4d68a95f266fe3366a69dc6c61d9686cf46526 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; ensure metadata contains all the information we require to determine the objects freshness diff --git a/content/llcache.c b/content/llcache.c index 944e0a5..8564bfe 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -1077,6 +1077,123 @@ static nserror llcache_persist_retrieve(llcache_object *object) &object->source_len); } +/** + * Generate a serialised version of an objects metadata + * + * metadata includes object headers + */ +static nserror +llcache_serialise_metadata(llcache_object *object, + uint8_t **data_out, + size_t *datasize_out) +{ + size_t allocsize; + int datasize; + uint8_t *data; + char *op; + unsigned int hloop; + int use; + struct tm *ltm; + + allocsize = 10 + 1; /* object length */ + + allocsize += 10 + 1; /* request time */ + + allocsize += 10 + 1; /* response time */ + + allocsize += 10 + 1; /* completion time */ + + allocsize += 10 + 1; /* space for number of header entries */ + + allocsize += nsurl_length(object->url) + 1; + + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + allocsize += strlen(object->headers[hloop].name) + 1; + allocsize += strlen(object->headers[hloop].value) + 1; + } + + data = malloc(allocsize); + if (data == NULL) { + return NSERROR_NOMEM; + } + + op = (char *)data; + datasize = allocsize; + + /* the url, used for checking for collisions */ + use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* object size */ + use = snprintf(op, datasize, "%zu%c", object->source_len, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* Time of request */ + ltm = localtime(&object->cache.req_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* Time of response */ + ltm = localtime(&object->cache.res_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* Time of completion */ + ltm = localtime(&object->cache.fin_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* number of headers */ + use = snprintf(op, datasize, "%zu%c", object->num_headers, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* headers */ + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + use = snprintf(op, datasize, + "%s:%s%c", + object->headers[hloop].name, + object->headers[hloop].value, + 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + } + + LLCACHE_LOG(("Filled buffer with %d spare", datasize)); + + *data_out = data; + *datasize_out = allocsize - datasize; + + return NSERROR_OK; + +overflow: + /* somehow we overflowed the buffer - hth? */ + LOG(("Overflowed metadata buffer")); + free(data); + return NSERROR_INVALID; +} static nserror llcache_process_metadata(llcache_object *object) @@ -1091,6 +1208,7 @@ llcache_process_metadata(llcache_object *object) int lnsize; size_t num_headers; size_t hloop; + struct tm ltm; /* attempt to retrieve object metadata from the cache */ res = guit->llcache->meta(object->url, &metadata, &metadatalen); @@ -1143,13 +1261,43 @@ llcache_process_metadata(llcache_object *object) goto format_error; object->source_alloc = object->source_len; - - /* metadata line 3 is the number of headers */ + /* metadata line 3 is the time of request */ line = 3; ln += lnsize + 1; lnsize = strlen(ln); if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.req_time = mktime(&ltm); + + /* metadata line 4 is the time of response */ + line = 4; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.res_time = mktime(&ltm); + + /* metadata line 5 is the time of request completion */ + line = 5; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.fin_time = mktime(&ltm); + + + /* metadata line 6 is the number of headers */ + line = 6; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || (sscanf(ln, "%zu", &num_headers) != 1)) goto format_error; @@ -1928,90 +2076,6 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) } /** - * Generate a serialised version of an objects metadata - * - * metadata includes object headers - */ -static nserror -llcache_serialise_metadata(llcache_object *object, - uint8_t **data_out, - size_t *datasize_out) -{ - size_t allocsize; - int datasize; - uint8_t *data; - char *op; - unsigned int hloop; - int use; - - allocsize = 10 + 1; /* object length */ - - allocsize += 10 + 1; /* space for number of header entries */ - - allocsize += nsurl_length(object->url) + 1; - - for (hloop = 0 ; hloop < object->num_headers ; hloop++) { - allocsize += strlen(object->headers[hloop].name) + 1; - allocsize += strlen(object->headers[hloop].value) + 1; - } - - data = malloc(allocsize); - if (data == NULL) { - return NSERROR_NOMEM; - } - - op = (char *)data; - datasize = allocsize; - - /* the url, used for checking for collisions */ - use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* object size */ - use = snprintf(op, datasize, "%zu%c", object->source_len, 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* number of headers */ - use = snprintf(op, datasize, "%zu%c", object->num_headers, 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* headers */ - for (hloop = 0 ; hloop < object->num_headers ; hloop++) { - use = snprintf(op, datasize, - "%s:%s%c", - object->headers[hloop].name, - object->headers[hloop].value, - 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - } - - LLCACHE_LOG(("Filled buffer with %d spare", datasize)); - - *data_out = data; - *datasize_out = allocsize - datasize; - - return NSERROR_OK; - -overflow: - /* somehow we overflowed the buffer - hth? */ - LOG(("Overflowed metadata buffer")); - free(data); - return NSERROR_INVALID; -} - -/** * construct a sorted list of objects available for writeout operation * * The list contains fresh cacheable objects held in RAM with no ----------------------------------------------------------------------- Summary of changes: content/llcache.c | 263 +++++++++++++++++++++++++++++---------------- content/llcache.h | 6 + content/llcache_private.h | 24 ++++- desktop/netsurf.c | 2 +- gtk/llcache.c | 143 +++++++++++++++++------- 5 files changed, 299 insertions(+), 139 deletions(-) diff --git a/content/llcache.c b/content/llcache.c index 944e0a5..e60c0c6 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -23,16 +23,29 @@ * stores source objects in memory and may use a persistant backing * store to extend their lifetime. * - * \todo fix writeout conditions and ordering - * \todo implement expiry from backing store of objects not currently in cache lists - * \todo make the reference backing store have an index to improve performance - * \todo make the reference backing store have a more efficient metadata store. - * \todo make the reference backing store cope with filesystems that have filename length limits and directory entry count limits - * \todo make backing store size bounded by configuration + * \todo fix writeout conditions and ordering. + * * \todo support mmaped retrieve - * \todo make metadata buffer owned by backing store (avoid allocation and freeing) + * + * \todo make metadata buffer owned by backing store (avoid allocation + * and freeing). + * * \todo instrument and (auto)tune + * * \todo turn llcache debugging off + * + * \todo implement expiry from backing store of objects not currently + * in cache lists. + * + * \todo make backing store size bounded by configuration + * + * \todo make backing store have an index to improve performance + * + * \todo make backing store have a more efficient metadata store. + * + * \todo make backing store cope with filesystems that have filename + * length limits and directory entry count limits. + * */ #include <stdlib.h> @@ -1077,6 +1090,123 @@ static nserror llcache_persist_retrieve(llcache_object *object) &object->source_len); } +/** + * Generate a serialised version of an objects metadata + * + * metadata includes object headers + */ +static nserror +llcache_serialise_metadata(llcache_object *object, + uint8_t **data_out, + size_t *datasize_out) +{ + size_t allocsize; + int datasize; + uint8_t *data; + char *op; + unsigned int hloop; + int use; + struct tm *ltm; + + allocsize = 10 + 1; /* object length */ + + allocsize += 10 + 1; /* request time */ + + allocsize += 10 + 1; /* response time */ + + allocsize += 10 + 1; /* completion time */ + + allocsize += 10 + 1; /* space for number of header entries */ + + allocsize += nsurl_length(object->url) + 1; + + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + allocsize += strlen(object->headers[hloop].name) + 1; + allocsize += strlen(object->headers[hloop].value) + 1; + } + + data = malloc(allocsize); + if (data == NULL) { + return NSERROR_NOMEM; + } + + op = (char *)data; + datasize = allocsize; + + /* the url, used for checking for collisions */ + use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* object size */ + use = snprintf(op, datasize, "%zu%c", object->source_len, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* Time of request */ + ltm = localtime(&object->cache.req_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* Time of response */ + ltm = localtime(&object->cache.res_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* Time of completion */ + ltm = localtime(&object->cache.fin_time); + use = strftime(op, datasize, "%s", ltm); + if (use == 0) + goto overflow; + use++; /* does not count the null */ + op += use; + datasize -= use; + + /* number of headers */ + use = snprintf(op, datasize, "%zu%c", object->num_headers, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* headers */ + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + use = snprintf(op, datasize, + "%s:%s%c", + object->headers[hloop].name, + object->headers[hloop].value, + 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + } + + LLCACHE_LOG(("Filled buffer with %d spare", datasize)); + + *data_out = data; + *datasize_out = allocsize - datasize; + + return NSERROR_OK; + +overflow: + /* somehow we overflowed the buffer - hth? */ + LOG(("Overflowed metadata buffer")); + free(data); + return NSERROR_INVALID; +} static nserror llcache_process_metadata(llcache_object *object) @@ -1091,6 +1221,7 @@ llcache_process_metadata(llcache_object *object) int lnsize; size_t num_headers; size_t hloop; + struct tm ltm; /* attempt to retrieve object metadata from the cache */ res = guit->llcache->meta(object->url, &metadata, &metadatalen); @@ -1143,13 +1274,43 @@ llcache_process_metadata(llcache_object *object) goto format_error; object->source_alloc = object->source_len; - - /* metadata line 3 is the number of headers */ + /* metadata line 3 is the time of request */ line = 3; ln += lnsize + 1; lnsize = strlen(ln); if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.req_time = mktime(&ltm); + + /* metadata line 4 is the time of response */ + line = 4; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.res_time = mktime(&ltm); + + /* metadata line 5 is the time of request completion */ + line = 5; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (strptime(ln, "%s", &ltm) == NULL)) + goto format_error; + object->cache.fin_time = mktime(&ltm); + + + /* metadata line 6 is the number of headers */ + line = 6; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || (sscanf(ln, "%zu", &num_headers) != 1)) goto format_error; @@ -1928,90 +2089,6 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) } /** - * Generate a serialised version of an objects metadata - * - * metadata includes object headers - */ -static nserror -llcache_serialise_metadata(llcache_object *object, - uint8_t **data_out, - size_t *datasize_out) -{ - size_t allocsize; - int datasize; - uint8_t *data; - char *op; - unsigned int hloop; - int use; - - allocsize = 10 + 1; /* object length */ - - allocsize += 10 + 1; /* space for number of header entries */ - - allocsize += nsurl_length(object->url) + 1; - - for (hloop = 0 ; hloop < object->num_headers ; hloop++) { - allocsize += strlen(object->headers[hloop].name) + 1; - allocsize += strlen(object->headers[hloop].value) + 1; - } - - data = malloc(allocsize); - if (data == NULL) { - return NSERROR_NOMEM; - } - - op = (char *)data; - datasize = allocsize; - - /* the url, used for checking for collisions */ - use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* object size */ - use = snprintf(op, datasize, "%zu%c", object->source_len, 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* number of headers */ - use = snprintf(op, datasize, "%zu%c", object->num_headers, 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - - /* headers */ - for (hloop = 0 ; hloop < object->num_headers ; hloop++) { - use = snprintf(op, datasize, - "%s:%s%c", - object->headers[hloop].name, - object->headers[hloop].value, - 0); - if (use > datasize) - goto overflow; - op += use; - datasize -= use; - } - - LLCACHE_LOG(("Filled buffer with %d spare", datasize)); - - *data_out = data; - *datasize_out = allocsize - datasize; - - return NSERROR_OK; - -overflow: - /* somehow we overflowed the buffer - hth? */ - LOG(("Overflowed metadata buffer")); - free(data); - return NSERROR_INVALID; -} - -/** * construct a sorted list of objects available for writeout operation * * The list contains fresh cacheable objects held in RAM with no diff --git a/content/llcache.h b/content/llcache.h index 6f12c48..854f54f 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -164,6 +164,12 @@ struct llcache_store_parameters { size_t limit; /**< The backing store upper bound target size */ size_t hysteresis; /**< The hysteresis around the target size */ + + /** length of hash to use for filename mapping. 0 indicates + * use no hash. 23 is a reasonable compromise for RISC OS. 160 + * uses sha1 in a manner similar to git. + */ + int hashsize; }; /** diff --git a/content/llcache_private.h b/content/llcache_private.h index dc6cb86..7791b98 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -32,13 +32,31 @@ * backing store using these operations. */ struct gui_llcache_table { - /** Initialise the backing store. */ + /** + * Initialise the backing store. + * + * @param parameters to configure backing store. + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*initialise)(const struct llcache_store_parameters *parameters); - /** Finalise the backing store. */ + /** + * Finalise the backing store. + * + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*finalise)(void); - /** Place a source object and its metadata in the backing store. */ + /** + * Place a source object and its metadata in the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param metadata The metadata associated with the object. + * @param metadatalen The length of the \a metadata. + * @param data The objects source data. + * @param datalen The length of the \a data. + * @return NSERROR_OK on success or error code on faliure. + */ nserror (*store)(struct nsurl *url, const uint8_t *metadata, const size_t metadatalen, const uint8_t *data, const size_t datalen); diff --git a/desktop/netsurf.c b/desktop/netsurf.c index 582c119..a4b4da2 100644 --- a/desktop/netsurf.c +++ b/desktop/netsurf.c @@ -191,7 +191,7 @@ nserror netsurf_init(const char *messages, struct gui_table *gt) /* set the path to the backing store */ /** \todo set the backing store path properly */ - hlcache_parameters.llcache.store.path = "/tmp/ns/"; + hlcache_parameters.llcache.store.path = "/tmp/ns"; /* image handler bitmap cache */ ret = image_cache_init(&image_cache_parameters); diff --git a/gtk/llcache.c b/gtk/llcache.c index 4fa4e1e..e4572cc 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -79,28 +79,110 @@ base64url_encode(const char *data, return encoded_data; } +struct llcache_store_state { + /** parameters controlling the backing store */ + struct llcache_store_parameters params; +}; + +struct llcache_store_state *storestate; + +/** + * Initialise the backing store. + * + * @param parameters to configure backing store. + * @return NSERROR_OK on success or error code on faliure. + */ +static nserror +initialise(const struct llcache_store_parameters *parameters) +{ + if (storestate != NULL) { + return NSERROR_INIT_FAILED; + } + storestate = calloc(1, sizeof(struct llcache_store_state)); + if (storestate == NULL) { + return NSERROR_NOMEM; + } + + memcpy(&storestate->params, parameters, sizeof(struct llcache_store_parameters)); + + return NSERROR_OK; +} + +/** + * Finalise the backing store. + * + * @return NSERROR_OK on success. + */ +static nserror +finalise(void) +{ + if (storestate != NULL) { + free(storestate); + storestate = NULL; + } + return NSERROR_OK; +} + +static char *store_name(nsurl *url, bool metadata) +{ + char *fname; + uint8_t *b64u; + size_t b64ulen; + char sep; + + if (metadata) { + sep ='i'; + } else { + sep ='d'; + } + + switch (storestate->params.hashsize) { + default: + /* no hashing, just base64 encode the url */ + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(strlen(storestate->params.path) + + SLEN("/x/") + b64ulen + 1); + + snprintf(fname, + (strlen(storestate->params.path) + + SLEN("/x/") + b64ulen + 1), + "%s/%c/%s", + storestate->params.path, + sep, + b64u); + free(b64u); + } + return fname; +} + +/** + * Place a source object and its metadata in the backing store. + * + * @param url The url is used as the unique primary key for the data. + * @param metadata The metadata associated with the object. + * @param metadatalen The length of the \a metadata. + * @param data The objects source data. + * @param datalen The length of the \a data. + * @return NSERROR_OK on success or error code on faliure. + */ static nserror store(nsurl *url, const uint8_t *metadata, const size_t metadatalen, const uint8_t *data, const size_t datalen) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; LOG(("Writing cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), strlen(nsurl_access(url)), &b64ulen); - - fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fname = store_name(url, false); LOG(("Opening data file %s", fname)); file = fopen(fname, "wb"); - + free(fname); if (file == NULL) { - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } @@ -108,21 +190,18 @@ store(nsurl *url, if (fwrite(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } fclose(file); - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + fname = store_name(url, true); LOG(("Opening info file %s", fname)); - file = fopen(fname, "wb"); + file = fopen(fname, "wb"); + free(fname); if (file == NULL) { - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } @@ -130,24 +209,17 @@ store(nsurl *url, if (fwrite(metadata, metadatalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(fname); - free(b64u); return NSERROR_SAVE_FAILED; } fclose(file); - free(fname); - free(b64u); - return NSERROR_OK; } static nserror fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; uint8_t *data; @@ -157,18 +229,14 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) datalen = *datalen_out; LOG(("retriving cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), - strlen(nsurl_access(url)), - &b64ulen); - fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fname = store_name(url, false); LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); free(fname); if (file == NULL) { - free(b64u); return NSERROR_NOT_FOUND; } @@ -184,7 +252,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) data = malloc(datalen); if (data == NULL) { fclose(file); - free(b64u); return NSERROR_NOMEM; } } @@ -193,7 +260,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) if (fread(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); - free(b64u); if ((*data_out) == NULL) { free(data); } @@ -201,7 +267,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) } fclose(file); - free(b64u); *data_out = data; *datalen_out = datalen; @@ -212,8 +277,6 @@ fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) static nserror fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) { - uint8_t *b64u; - size_t b64ulen; char *fname; FILE *file; uint8_t *data; @@ -223,18 +286,14 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) datalen = *metadatalen_out; LOG(("retriving cache file for url:%s", nsurl_access(url))); - b64u = base64url_encode(nsurl_access(url), - strlen(nsurl_access(url)), - &b64ulen); - fname = malloc(SLEN("/tmp/ns/i/") + b64ulen + 1); - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + fname = store_name(url, true); LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); free(fname); if (file == NULL) { - free(b64u); return NSERROR_NOT_FOUND; } @@ -250,7 +309,6 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) data = malloc(datalen); if (data == NULL) { fclose(file); - free(b64u); return NSERROR_NOMEM; } } @@ -258,8 +316,8 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) LOG(("Reading %d bytes into %p from file", datalen, data)); if (fread(data, datalen, 1, file) != 1) { LOG(("did not return 1")); + fclose(file); - free(b64u); if ((*metadata_out) == NULL) { free(data); } @@ -267,7 +325,6 @@ fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) } fclose(file); - free(b64u); *metadata_out = data; *metadatalen_out = datalen; @@ -283,6 +340,8 @@ invalidate(nsurl *url) static struct gui_llcache_table llcache_table = { + .initialise = initialise, + .finalise = finalise, .store = store, .fetch = fetch, .meta = fetchmeta, -- NetSurf Browser