Gitweb links: ...log http://git.netsurf-browser.org/netsurf.git/shortlog/5b625b4479a813c4284e4... ...commit http://git.netsurf-browser.org/netsurf.git/commit/5b625b4479a813c4284e40d... ...tree http://git.netsurf-browser.org/netsurf.git/tree/5b625b4479a813c4284e40d50... The branch, vince/disccache has been updated via 5b625b4479a813c4284e40d5011769204f66a19e (commit) via cc69c29fcd44a56d2af310cb3c98a7365c0ae1d0 (commit) from b16f63dc42635c5dfc10c67ae7f39f39663edb2b (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=5b625b4479a813c4284... commit 5b625b4479a813c4284e40d5011769204f66a19e Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; change metadata handling and cleanup header processing diff --git a/content/llcache.c b/content/llcache.c index 44224fe..6731a28 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -17,7 +17,11 @@ */ /** \file - * Low-level resource cache (implementation) + * Low-level resource cache implementation + * + * This is the implementation of the low level cache. This cache + * stores source objects in memory and may use a persistant backing + * store to extend their lifetime. */ #include <stdlib.h> @@ -47,6 +51,9 @@ #define LLCACHE_LOG(x) #endif +#define LLCACHE_MIN_DISC_LIFETIME 3600 +#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) + /** State of a low-level cache object fetch */ typedef enum { LLCACHE_FETCH_INIT, /**< Initial state, before fetch */ @@ -105,6 +112,30 @@ typedef struct { char *value; /**< Header value */ } llcache_header; +/** validation control */ +typedef enum { + LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ + LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ + LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ +} llcache_validate; + +/** cache control value for invalid age */ +#define INVALID_AGE -1 + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t fin_time; /**< Time of request completion */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + llcache_validate no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + /** Current status of objects data */ typedef enum { LLCACHE_STORE_RAM = 0, /**< sourec data is stored in RAM only */ @@ -399,7 +430,15 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, colon = (const uint8_t *) strchr((const char *) data, ':'); if (colon == NULL) { /* Failed, assume a key with no value */ - n = strdup((const char *) data); + colon = data + strlen((const char *)data); + + /* Strip trailing whitespace from name */ + while ((colon > data) && + (colon[-1] == ' ' || colon[-1] == '\t' || + colon[-1] == '\r' || colon[-1] == '\n')) { + colon--; + } + n = strndup((const char *) data, colon - data); if (n == NULL) return NSERROR_NOMEM; @@ -620,6 +659,13 @@ static nserror llcache_fetch_process_header(llcache_object *object, return error; } + /* deal with empty header */ + if (name[0] == 0) { + free(name); + free(value); + return NSERROR_OK; + } + /* Append header data to the object's headers array */ temp = realloc(object->headers, (object->num_headers + 1) * sizeof(llcache_header)); @@ -1017,14 +1063,19 @@ llcache_persist_retrieve(llcache_object *object) } /* Source data for the object may be in the persiatant store */ - return guit->llcache->retrieve(object->url, - NULL, - &object->source_data, - &object->source_len); + return guit->llcache->fetch(object->url, + &object->source_data, + &object->source_len); +} + +static nserror +llcache_process_metadata(llcache_object *object, uint8_t *data, size_t datasize) +{ + return NSERROR_NOT_FOUND; } /** - * attempt to retrieve an object from persistant storage + * attempt to retrieve an object from persistant storage. */ static nserror llcache_object_fetch_persistant(llcache_object *object, @@ -1036,12 +1087,11 @@ llcache_object_fetch_persistant(llcache_object *object, nserror error; nsurl *referer_clone = NULL; llcache_post_data *post_clone = NULL; + uint8_t *metadata = NULL; + size_t metadatalen = 0; - /* attempt to retrieve object from the cache */ - error = guit->llcache->retrieve(object->url, - &object->cache, - &object->source_data, - &object->source_len); + /* attempt to retrieve object metadata from the cache */ + error = guit->llcache->meta(object->url, &metadata, &metadatalen); if (error != NSERROR_OK) return error; @@ -1055,11 +1105,20 @@ llcache_object_fetch_persistant(llcache_object *object, if (referer != NULL) referer_clone = nsurl_ref(referer); + object->cache.req_time = time(NULL); + object->cache.fin_time = object->cache.req_time; + object->fetch.flags = flags; object->fetch.referer = referer_clone; object->fetch.post = post_clone; object->fetch.redirect_count = redirect_count; + /* process the metadata */ + error = llcache_process_metadata(object, metadata, metadatalen); + free(metadata); + if (error != NSERROR_OK) + return error; + /* fetch is "finished" */ object->fetch.state = LLCACHE_FETCH_COMPLETE; object->fetch.fetch = NULL; @@ -1772,9 +1831,89 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) return error; } -#define LLCACHE_MIN_DISC_LIFETIME 3600 -#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) +/** + * Generate a serialised version of an objects metadata + * + * metadata includes object headers + */ +static nserror +llcache_serialise_metadata(llcache_object *object, + uint8_t **data_out, + size_t *datasize_out) +{ + size_t allocsize; + int datasize; + uint8_t *data; + char *op; + unsigned int hloop; + int use; + + allocsize = 10 + 2; /* object length */ + + allocsize += 10 + 2; /* space for number of header entries */ + + allocsize += nsurl_length(object->url) + 2; + + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + allocsize += strlen(object->headers[hloop].name) + 1; + allocsize += strlen(object->headers[hloop].value) + 2; + } + + data = malloc(allocsize); + if (data == NULL) { + return NSERROR_NOMEM; + } + + op = (char *)data; + datasize = allocsize; + + /* the url, used for checking for collisions */ + use = snprintf(op, datasize, "%s%c%c", nsurl_access(object->url) ,13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* object size */ + use = snprintf(op, datasize, "%d%c%c", (unsigned int)object->source_len, 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* number of headers */ + use = snprintf(op, datasize, "%d%c%c", (unsigned int)object->num_headers, 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* headers */ + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + use = snprintf(op, datasize, + "%s:%s%c%c", + object->headers[hloop].name, + object->headers[hloop].value, + 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + } + + LOG(("Filled buffer with %d spare", datasize)); + + *data_out = data; + *datasize_out = allocsize - datasize; + return NSERROR_OK; + +overflow: + /* somehow we overflowed the buffer - hth? */ + LOG(("Overflowed metadata buffer")); + free(data); + return NSERROR_INVALID; +} /** * possibly push objects data to persiatant storage. @@ -1800,11 +1939,25 @@ static void llcache_persist(void *p) (object->fetch.outstanding_query == false) && (object->store_state == LLCACHE_STORE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + uint8_t *metadata; + size_t metadatasize; + + ret = llcache_serialise_metadata(object, &metadata, &metadatasize); + if (ret != NSERROR_OK) { + /* as there has been a serialisation + * error, give up on making any more + * objects persistant for now. + */ + return; + } + /* ok found an object to write */ - ret = guit->llcache->persist(object->url, - &object->cache, - object->source_data, - object->source_len); + ret = guit->llcache->store(object->url, + metadata, + metadatasize, + object->source_data, + object->source_len); + free(metadata); if (ret != NSERROR_OK) { /* as there has been a serialisation * error, give up on making any more @@ -1813,7 +1966,7 @@ static void llcache_persist(void *p) return; } object->store_state = LLCACHE_STORE_DISC; - size_written += object->source_len; + size_written += object->source_len + metadatasize; if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { break; diff --git a/content/llcache_private.h b/content/llcache_private.h index 926773b..f038c9d 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,34 +25,31 @@ #include "content/llcache.h" -/** validation control */ -typedef enum { - LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ - LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ - LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ -} llcache_validate; - -/** cache control value for invalid age */ -#define INVALID_AGE -1 - -/** Cache control data */ -typedef struct { - time_t req_time; /**< Time of request */ - time_t res_time; /**< Time of response */ - time_t fin_time; /**< Time of request completion */ - time_t date; /**< Date: response header */ - time_t expires; /**< Expires: response header */ - int age; /**< Age: response header */ - int max_age; /**< Max-Age Cache-control parameter */ - llcache_validate no_cache; /**< No-Cache Cache-control parameter */ - char *etag; /**< Etag: response header */ - time_t last_modified; /**< Last-Modified: response header */ -} llcache_cache_control; - -/** operation table */ +/** low level cache operation table + * + * The low level cache (source objects) has the capability to make + * objects and their metadata (headers etc) persistant by writing to a + * backing store using these operations. + */ struct gui_llcache_table { - nserror (*persist)(struct nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen); - nserror (*retrieve)(struct nsurl *url, llcache_cache_control *cache_control, uint8_t **data, size_t *datalen); + /** Place a source object and its metadata in the backing store. */ + nserror (*store)(struct nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen); + + /** retrive source object data from backing store. */ + nserror (*fetch)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** retrive source object metadata from backing store. */ + nserror (*meta)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** + * invalidate a source object from the backing store. + * + * The entry (if present in the backing store) must no longer + * be returned as a result to the fetch or meta operations. + */ + nserror (*invalidate)(struct nsurl *url); }; #endif diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index 37fee39..71fb226 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -345,19 +345,38 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } -static nserror gui_default_persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +static nserror +gui_default_store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { return NSERROR_SAVE_FAILED; } -static nserror gui_default_retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) +static nserror +gui_default_fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } +static nserror +gui_default_meta(nsurl *url, uint8_t **data_out, size_t *datalen_out) +{ + return NSERROR_NOT_FOUND; +} + +static nserror +gui_default_invalidate(nsurl *url) +{ + return NSERROR_NOT_FOUND; +} + + static struct gui_llcache_table default_llcache_table = { - .persist = gui_default_persist, - .retrieve = gui_default_retrieve, + .store = gui_default_store, + .fetch = gui_default_fetch, + .meta = gui_default_meta, + .invalidate = gui_default_invalidate, }; /** verify clipboard table is valid */ @@ -369,10 +388,16 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) } /* mandantory operations */ - if (glt->persist == NULL) { + if (glt->store == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->fetch == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->meta == NULL) { return NSERROR_BAD_PARAMETER; } - if (glt->retrieve == NULL) { + if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } return NSERROR_OK; diff --git a/gtk/llcache.c b/gtk/llcache.c index a2c27e5..4fa4e1e 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -41,9 +41,10 @@ static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', static unsigned int mod_table[] = {0, 2, 1}; -static uint8_t *base64url_encode(const char *data, - size_t input_length, - size_t *output_length) +static uint8_t * +base64url_encode(const char *data, + size_t input_length, + size_t *output_length) { uint8_t *encoded_data; size_t i; @@ -79,7 +80,9 @@ static uint8_t *base64url_encode(const char *data, } static nserror -persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { uint8_t *b64u; size_t b64ulen; @@ -101,7 +104,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - LOG(("Writing %d bytes at %p to file", datalen, data)); + LOG(("Writing %d bytes from %p", datalen, data)); if (fwrite(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); @@ -123,27 +126,14 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - fprintf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%s\n", - (long long)cache_control->req_time, - (long long)cache_control->res_time, - (long long)cache_control->fin_time, - (long long)cache_control->date, - (long long)cache_control->expires, - cache_control->age, - cache_control->max_age, - cache_control->no_cache, - (long long)cache_control->last_modified, - cache_control->etag == NULL?"":cache_control->etag); + LOG(("Writing %d bytes from %p", metadatalen, metadata)); + if (fwrite(metadata, metadatalen, 1, file) != 1) { + LOG(("did not return 1")); + fclose(file); + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } fclose(file); @@ -154,10 +144,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c } static nserror -retrieve(nsurl *url, - llcache_cache_control *cache_control, - uint8_t **data_out, - size_t *datalen_out) +fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { uint8_t *b64u; size_t b64ulen; @@ -166,7 +153,6 @@ retrieve(nsurl *url, uint8_t *data; size_t datalen; - data = *data_out; datalen = *datalen_out; @@ -176,51 +162,73 @@ retrieve(nsurl *url, &b64ulen); fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); - /* retrive cache control info if necessary */ - if (cache_control != NULL) { - int cnt; - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); + free(fname); + if (file == NULL) { + free(b64u); + return NSERROR_NOT_FOUND; + } - file = fopen(fname, "rb"); - if (file == NULL) { - LOG(("Unable to open %s", fname)); - free(fname); - free(b64u); - return NSERROR_NOT_FOUND; + /* need to deal with buffers */ + if (data == NULL) { + if (datalen == 0) { + /* caller did not know the files length */ + fseek(file, 0L, SEEK_END); + datalen = ftell(file); + fseek(file, 0L, SEEK_SET); } - LOG(("Opening info file %s", fname)); - - cnt = fscanf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%ms\n", - (long long*)&cache_control->req_time, - (long long*)&cache_control->res_time, - (long long*)&cache_control->fin_time, - (long long*)&cache_control->date, - (long long*)&cache_control->expires, - &cache_control->age, - &cache_control->max_age, - (int *)&cache_control->no_cache, - (long long*)&cache_control->last_modified, - &cache_control->etag); - - LOG(("Read %d items from cache constrol", cnt)); + data = malloc(datalen); + if (data == NULL) { + fclose(file); + free(b64u); + return NSERROR_NOMEM; + } + } + LOG(("Reading %d bytes into %p from file", datalen, data)); + if (fread(data, datalen, 1, file) != 1) { + LOG(("did not return 1")); fclose(file); - + free(b64u); + if ((*data_out) == NULL) { + free(data); + } + return NSERROR_NOT_FOUND; } - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fclose(file); + free(b64u); + + *data_out = data; + *datalen_out = datalen; + + return NSERROR_OK; +} + +static nserror +fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) +{ + uint8_t *b64u; + size_t b64ulen; + char *fname; + FILE *file; + uint8_t *data; + size_t datalen; + + data = *metadata_out; + datalen = *metadatalen_out; + + LOG(("retriving cache file for url:%s", nsurl_access(url))); + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(SLEN("/tmp/ns/i/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); LOG(("Opening file %s",fname)); file = fopen(fname, "rb"); @@ -230,7 +238,7 @@ retrieve(nsurl *url, return NSERROR_NOT_FOUND; } - /* need to deal with buffers */ + /* need to deal with allocating buffers */ if (data == NULL) { if (datalen == 0) { /* caller did not know the files length */ @@ -252,7 +260,7 @@ retrieve(nsurl *url, LOG(("did not return 1")); fclose(file); free(b64u); - if ((*data_out) == NULL) { + if ((*metadata_out) == NULL) { free(data); } return NSERROR_NOT_FOUND; @@ -261,15 +269,24 @@ retrieve(nsurl *url, fclose(file); free(b64u); - *data_out = data; - *datalen_out = datalen; + *metadata_out = data; + *metadatalen_out = datalen; return NSERROR_OK; } +static nserror +invalidate(nsurl *url) +{ + return NSERROR_OK; +} + + static struct gui_llcache_table llcache_table = { - .persist = persist, - .retrieve = retrieve, + .store = store, + .fetch = fetch, + .meta = fetchmeta, + .invalidate = invalidate, }; struct gui_llcache_table *nsgtk_llcache_table = &llcache_table; commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=cc69c29fcd44a56d2af... commit cc69c29fcd44a56d2af310cb3c98a7365c0ae1d0 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)kyllikki.org&gt; header names should have whitespace stripped regardless diff --git a/content/llcache.c b/content/llcache.c index a4bc9b4..44224fe 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -389,6 +389,12 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, char *n, *v; const uint8_t *colon; + /* Strip leading whitespace from name */ + while (data[0] == ' ' || data[0] == '\t' || + data[0] == '\r' || data[0] == '\n') { + data++; + } + /* Find colon */ colon = (const uint8_t *) strchr((const char *) data, ':'); if (colon == NULL) { @@ -405,12 +411,6 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, } else { /* Split header into name & value */ - /* Strip leading whitespace from name */ - while (data[0] == ' ' || data[0] == '\t' || - data[0] == '\r' || data[0] == '\n') { - data++; - } - /* Strip trailing whitespace from name */ while (colon > data && (colon[-1] == ' ' || colon[-1] == '\t' || colon[-1] == '\r' || @@ -1023,8 +1023,11 @@ llcache_persist_retrieve(llcache_object *object) &object->source_len); } +/** + * attempt to retrieve an object from persistant storage + */ static nserror -llcache_object_persist_fetch(llcache_object *object, +llcache_object_fetch_persistant(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, @@ -1114,7 +1117,7 @@ llcache_object_retrieve_from_cache(nsurl *url, return error; /* attempt to retrieve object from persistant store */ - error = llcache_object_persist_fetch(obj, flags, referer, post, redirect_count); + error = llcache_object_fetch_persistant(obj, flags, referer, post, redirect_count); if (error == NSERROR_OK) { /* set object from persistant store as newest */ newest = obj; ----------------------------------------------------------------------- Summary of changes: content/llcache.c | 208 +++++++++++++++++++++++++++++++++++++++------ content/llcache_private.h | 51 +++++------ desktop/gui_factory.c | 37 +++++++-- gtk/llcache.c | 165 ++++++++++++++++++++---------------- 4 files changed, 328 insertions(+), 133 deletions(-) diff --git a/content/llcache.c b/content/llcache.c index a4bc9b4..6731a28 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -17,7 +17,11 @@ */ /** \file - * Low-level resource cache (implementation) + * Low-level resource cache implementation + * + * This is the implementation of the low level cache. This cache + * stores source objects in memory and may use a persistant backing + * store to extend their lifetime. */ #include <stdlib.h> @@ -47,6 +51,9 @@ #define LLCACHE_LOG(x) #endif +#define LLCACHE_MIN_DISC_LIFETIME 3600 +#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) + /** State of a low-level cache object fetch */ typedef enum { LLCACHE_FETCH_INIT, /**< Initial state, before fetch */ @@ -105,6 +112,30 @@ typedef struct { char *value; /**< Header value */ } llcache_header; +/** validation control */ +typedef enum { + LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ + LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ + LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ +} llcache_validate; + +/** cache control value for invalid age */ +#define INVALID_AGE -1 + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t fin_time; /**< Time of request completion */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + llcache_validate no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + /** Current status of objects data */ typedef enum { LLCACHE_STORE_RAM = 0, /**< sourec data is stored in RAM only */ @@ -389,11 +420,25 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, char *n, *v; const uint8_t *colon; + /* Strip leading whitespace from name */ + while (data[0] == ' ' || data[0] == '\t' || + data[0] == '\r' || data[0] == '\n') { + data++; + } + /* Find colon */ colon = (const uint8_t *) strchr((const char *) data, ':'); if (colon == NULL) { /* Failed, assume a key with no value */ - n = strdup((const char *) data); + colon = data + strlen((const char *)data); + + /* Strip trailing whitespace from name */ + while ((colon > data) && + (colon[-1] == ' ' || colon[-1] == '\t' || + colon[-1] == '\r' || colon[-1] == '\n')) { + colon--; + } + n = strndup((const char *) data, colon - data); if (n == NULL) return NSERROR_NOMEM; @@ -405,12 +450,6 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, } else { /* Split header into name & value */ - /* Strip leading whitespace from name */ - while (data[0] == ' ' || data[0] == '\t' || - data[0] == '\r' || data[0] == '\n') { - data++; - } - /* Strip trailing whitespace from name */ while (colon > data && (colon[-1] == ' ' || colon[-1] == '\t' || colon[-1] == '\r' || @@ -620,6 +659,13 @@ static nserror llcache_fetch_process_header(llcache_object *object, return error; } + /* deal with empty header */ + if (name[0] == 0) { + free(name); + free(value); + return NSERROR_OK; + } + /* Append header data to the object's headers array */ temp = realloc(object->headers, (object->num_headers + 1) * sizeof(llcache_header)); @@ -1017,14 +1063,22 @@ llcache_persist_retrieve(llcache_object *object) } /* Source data for the object may be in the persiatant store */ - return guit->llcache->retrieve(object->url, - NULL, - &object->source_data, - &object->source_len); + return guit->llcache->fetch(object->url, + &object->source_data, + &object->source_len); +} + +static nserror +llcache_process_metadata(llcache_object *object, uint8_t *data, size_t datasize) +{ + return NSERROR_NOT_FOUND; } +/** + * attempt to retrieve an object from persistant storage. + */ static nserror -llcache_object_persist_fetch(llcache_object *object, +llcache_object_fetch_persistant(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, @@ -1033,12 +1087,11 @@ llcache_object_persist_fetch(llcache_object *object, nserror error; nsurl *referer_clone = NULL; llcache_post_data *post_clone = NULL; + uint8_t *metadata = NULL; + size_t metadatalen = 0; - /* attempt to retrieve object from the cache */ - error = guit->llcache->retrieve(object->url, - &object->cache, - &object->source_data, - &object->source_len); + /* attempt to retrieve object metadata from the cache */ + error = guit->llcache->meta(object->url, &metadata, &metadatalen); if (error != NSERROR_OK) return error; @@ -1052,11 +1105,20 @@ llcache_object_persist_fetch(llcache_object *object, if (referer != NULL) referer_clone = nsurl_ref(referer); + object->cache.req_time = time(NULL); + object->cache.fin_time = object->cache.req_time; + object->fetch.flags = flags; object->fetch.referer = referer_clone; object->fetch.post = post_clone; object->fetch.redirect_count = redirect_count; + /* process the metadata */ + error = llcache_process_metadata(object, metadata, metadatalen); + free(metadata); + if (error != NSERROR_OK) + return error; + /* fetch is "finished" */ object->fetch.state = LLCACHE_FETCH_COMPLETE; object->fetch.fetch = NULL; @@ -1114,7 +1176,7 @@ llcache_object_retrieve_from_cache(nsurl *url, return error; /* attempt to retrieve object from persistant store */ - error = llcache_object_persist_fetch(obj, flags, referer, post, redirect_count); + error = llcache_object_fetch_persistant(obj, flags, referer, post, redirect_count); if (error == NSERROR_OK) { /* set object from persistant store as newest */ newest = obj; @@ -1769,9 +1831,89 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) return error; } -#define LLCACHE_MIN_DISC_LIFETIME 3600 -#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) +/** + * Generate a serialised version of an objects metadata + * + * metadata includes object headers + */ +static nserror +llcache_serialise_metadata(llcache_object *object, + uint8_t **data_out, + size_t *datasize_out) +{ + size_t allocsize; + int datasize; + uint8_t *data; + char *op; + unsigned int hloop; + int use; + + allocsize = 10 + 2; /* object length */ + + allocsize += 10 + 2; /* space for number of header entries */ + allocsize += nsurl_length(object->url) + 2; + + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + allocsize += strlen(object->headers[hloop].name) + 1; + allocsize += strlen(object->headers[hloop].value) + 2; + } + + data = malloc(allocsize); + if (data == NULL) { + return NSERROR_NOMEM; + } + + op = (char *)data; + datasize = allocsize; + + /* the url, used for checking for collisions */ + use = snprintf(op, datasize, "%s%c%c", nsurl_access(object->url) ,13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* object size */ + use = snprintf(op, datasize, "%d%c%c", (unsigned int)object->source_len, 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* number of headers */ + use = snprintf(op, datasize, "%d%c%c", (unsigned int)object->num_headers, 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* headers */ + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + use = snprintf(op, datasize, + "%s:%s%c%c", + object->headers[hloop].name, + object->headers[hloop].value, + 13, 10); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + } + + LOG(("Filled buffer with %d spare", datasize)); + + *data_out = data; + *datasize_out = allocsize - datasize; + + return NSERROR_OK; + +overflow: + /* somehow we overflowed the buffer - hth? */ + LOG(("Overflowed metadata buffer")); + free(data); + return NSERROR_INVALID; +} /** * possibly push objects data to persiatant storage. @@ -1797,11 +1939,25 @@ static void llcache_persist(void *p) (object->fetch.outstanding_query == false) && (object->store_state == LLCACHE_STORE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + uint8_t *metadata; + size_t metadatasize; + + ret = llcache_serialise_metadata(object, &metadata, &metadatasize); + if (ret != NSERROR_OK) { + /* as there has been a serialisation + * error, give up on making any more + * objects persistant for now. + */ + return; + } + /* ok found an object to write */ - ret = guit->llcache->persist(object->url, - &object->cache, - object->source_data, - object->source_len); + ret = guit->llcache->store(object->url, + metadata, + metadatasize, + object->source_data, + object->source_len); + free(metadata); if (ret != NSERROR_OK) { /* as there has been a serialisation * error, give up on making any more @@ -1810,7 +1966,7 @@ static void llcache_persist(void *p) return; } object->store_state = LLCACHE_STORE_DISC; - size_written += object->source_len; + size_written += object->source_len + metadatasize; if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { break; diff --git a/content/llcache_private.h b/content/llcache_private.h index 926773b..f038c9d 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,34 +25,31 @@ #include "content/llcache.h" -/** validation control */ -typedef enum { - LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ - LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ - LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ -} llcache_validate; - -/** cache control value for invalid age */ -#define INVALID_AGE -1 - -/** Cache control data */ -typedef struct { - time_t req_time; /**< Time of request */ - time_t res_time; /**< Time of response */ - time_t fin_time; /**< Time of request completion */ - time_t date; /**< Date: response header */ - time_t expires; /**< Expires: response header */ - int age; /**< Age: response header */ - int max_age; /**< Max-Age Cache-control parameter */ - llcache_validate no_cache; /**< No-Cache Cache-control parameter */ - char *etag; /**< Etag: response header */ - time_t last_modified; /**< Last-Modified: response header */ -} llcache_cache_control; - -/** operation table */ +/** low level cache operation table + * + * The low level cache (source objects) has the capability to make + * objects and their metadata (headers etc) persistant by writing to a + * backing store using these operations. + */ struct gui_llcache_table { - nserror (*persist)(struct nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen); - nserror (*retrieve)(struct nsurl *url, llcache_cache_control *cache_control, uint8_t **data, size_t *datalen); + /** Place a source object and its metadata in the backing store. */ + nserror (*store)(struct nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen); + + /** retrive source object data from backing store. */ + nserror (*fetch)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** retrive source object metadata from backing store. */ + nserror (*meta)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** + * invalidate a source object from the backing store. + * + * The entry (if present in the backing store) must no longer + * be returned as a result to the fetch or meta operations. + */ + nserror (*invalidate)(struct nsurl *url); }; #endif diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index 37fee39..71fb226 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -345,19 +345,38 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } -static nserror gui_default_persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +static nserror +gui_default_store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { return NSERROR_SAVE_FAILED; } -static nserror gui_default_retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) +static nserror +gui_default_fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } +static nserror +gui_default_meta(nsurl *url, uint8_t **data_out, size_t *datalen_out) +{ + return NSERROR_NOT_FOUND; +} + +static nserror +gui_default_invalidate(nsurl *url) +{ + return NSERROR_NOT_FOUND; +} + + static struct gui_llcache_table default_llcache_table = { - .persist = gui_default_persist, - .retrieve = gui_default_retrieve, + .store = gui_default_store, + .fetch = gui_default_fetch, + .meta = gui_default_meta, + .invalidate = gui_default_invalidate, }; /** verify clipboard table is valid */ @@ -369,10 +388,16 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) } /* mandantory operations */ - if (glt->persist == NULL) { + if (glt->store == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->fetch == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->meta == NULL) { return NSERROR_BAD_PARAMETER; } - if (glt->retrieve == NULL) { + if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } return NSERROR_OK; diff --git a/gtk/llcache.c b/gtk/llcache.c index a2c27e5..4fa4e1e 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -41,9 +41,10 @@ static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', static unsigned int mod_table[] = {0, 2, 1}; -static uint8_t *base64url_encode(const char *data, - size_t input_length, - size_t *output_length) +static uint8_t * +base64url_encode(const char *data, + size_t input_length, + size_t *output_length) { uint8_t *encoded_data; size_t i; @@ -79,7 +80,9 @@ static uint8_t *base64url_encode(const char *data, } static nserror -persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { uint8_t *b64u; size_t b64ulen; @@ -101,7 +104,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - LOG(("Writing %d bytes at %p to file", datalen, data)); + LOG(("Writing %d bytes from %p", datalen, data)); if (fwrite(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); @@ -123,27 +126,14 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - fprintf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%s\n", - (long long)cache_control->req_time, - (long long)cache_control->res_time, - (long long)cache_control->fin_time, - (long long)cache_control->date, - (long long)cache_control->expires, - cache_control->age, - cache_control->max_age, - cache_control->no_cache, - (long long)cache_control->last_modified, - cache_control->etag == NULL?"":cache_control->etag); + LOG(("Writing %d bytes from %p", metadatalen, metadata)); + if (fwrite(metadata, metadatalen, 1, file) != 1) { + LOG(("did not return 1")); + fclose(file); + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } fclose(file); @@ -154,10 +144,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c } static nserror -retrieve(nsurl *url, - llcache_cache_control *cache_control, - uint8_t **data_out, - size_t *datalen_out) +fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { uint8_t *b64u; size_t b64ulen; @@ -166,7 +153,6 @@ retrieve(nsurl *url, uint8_t *data; size_t datalen; - data = *data_out; datalen = *datalen_out; @@ -176,51 +162,73 @@ retrieve(nsurl *url, &b64ulen); fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); - /* retrive cache control info if necessary */ - if (cache_control != NULL) { - int cnt; - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); + free(fname); + if (file == NULL) { + free(b64u); + return NSERROR_NOT_FOUND; + } - file = fopen(fname, "rb"); - if (file == NULL) { - LOG(("Unable to open %s", fname)); - free(fname); - free(b64u); - return NSERROR_NOT_FOUND; + /* need to deal with buffers */ + if (data == NULL) { + if (datalen == 0) { + /* caller did not know the files length */ + fseek(file, 0L, SEEK_END); + datalen = ftell(file); + fseek(file, 0L, SEEK_SET); } - LOG(("Opening info file %s", fname)); - - cnt = fscanf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%ms\n", - (long long*)&cache_control->req_time, - (long long*)&cache_control->res_time, - (long long*)&cache_control->fin_time, - (long long*)&cache_control->date, - (long long*)&cache_control->expires, - &cache_control->age, - &cache_control->max_age, - (int *)&cache_control->no_cache, - (long long*)&cache_control->last_modified, - &cache_control->etag); - - LOG(("Read %d items from cache constrol", cnt)); + data = malloc(datalen); + if (data == NULL) { + fclose(file); + free(b64u); + return NSERROR_NOMEM; + } + } + LOG(("Reading %d bytes into %p from file", datalen, data)); + if (fread(data, datalen, 1, file) != 1) { + LOG(("did not return 1")); fclose(file); - + free(b64u); + if ((*data_out) == NULL) { + free(data); + } + return NSERROR_NOT_FOUND; } - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fclose(file); + free(b64u); + + *data_out = data; + *datalen_out = datalen; + + return NSERROR_OK; +} + +static nserror +fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) +{ + uint8_t *b64u; + size_t b64ulen; + char *fname; + FILE *file; + uint8_t *data; + size_t datalen; + + data = *metadata_out; + datalen = *metadatalen_out; + + LOG(("retriving cache file for url:%s", nsurl_access(url))); + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(SLEN("/tmp/ns/i/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); LOG(("Opening file %s",fname)); file = fopen(fname, "rb"); @@ -230,7 +238,7 @@ retrieve(nsurl *url, return NSERROR_NOT_FOUND; } - /* need to deal with buffers */ + /* need to deal with allocating buffers */ if (data == NULL) { if (datalen == 0) { /* caller did not know the files length */ @@ -252,7 +260,7 @@ retrieve(nsurl *url, LOG(("did not return 1")); fclose(file); free(b64u); - if ((*data_out) == NULL) { + if ((*metadata_out) == NULL) { free(data); } return NSERROR_NOT_FOUND; @@ -261,15 +269,24 @@ retrieve(nsurl *url, fclose(file); free(b64u); - *data_out = data; - *datalen_out = datalen; + *metadata_out = data; + *metadatalen_out = datalen; return NSERROR_OK; } +static nserror +invalidate(nsurl *url) +{ + return NSERROR_OK; +} + + static struct gui_llcache_table llcache_table = { - .persist = persist, - .retrieve = retrieve, + .store = store, + .fetch = fetch, + .meta = fetchmeta, + .invalidate = invalidate, }; struct gui_llcache_table *nsgtk_llcache_table = &llcache_table; -- NetSurf Browser