Gitweb links: ...log http://git.netsurf-browser.org/netsurf.git/shortlog/5b29a5efcd996f2b218fa... ...commit http://git.netsurf-browser.org/netsurf.git/commit/5b29a5efcd996f2b218faf3... ...tree http://git.netsurf-browser.org/netsurf.git/tree/5b29a5efcd996f2b218faf304... The branch, vince/prstllcache has been created at 5b29a5efcd996f2b218faf30433da587041ed991 (commit) - Log ----------------------------------------------------------------- commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=5b29a5efcd996f2b218... commit 5b29a5efcd996f2b218faf30433da587041ed991 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; add sha1 implementation and tests diff --git a/test/Makefile b/test/Makefile index acf9d4e..825b582 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,6 +2,7 @@ CFLAGS := -std=c99 -g -O0 -D_BSD_SOURCE -D_POSIX_C_SOURCE -I.. \ $(shell pkg-config --cflags libcurl) LDFLAGS := $(shell pkg-config --libs libcurl) -lz + llcache_CFLAGS := $(shell pkg-config --cflags libparserutils libwapcaplet libdom) -O2 llcache_LDFLAGS := $(shell pkg-config --libs libparserutils libwapcaplet libdom) @@ -14,6 +15,7 @@ llcache_SRCS := content/fetch.c content/fetchers/curl.c \ utils/log.c utils/nsurl.c utils/messages.c utils/url.c \ utils/useragent.c utils/utils.c test/llcache.c + urldbtest_SRCS := content/urldb.c utils/url.c utils/utils.c utils/log.c \ desktop/options.c utils/messages.c utils/hashtable.c \ utils/filename.c utils/nsurl.c utils/corestrings.c \ @@ -22,16 +24,22 @@ urldbtest_SRCS := content/urldb.c utils/url.c utils/utils.c utils/log.c \ urldbtest_CFLAGS := $(shell pkg-config --cflags libwapcaplet libdom) -O2 urldbtest_LDFLAGS := $(shell pkg-config --libs libwapcaplet libdom) + nsurl_SRCS := utils/corestrings.c utils/log.c utils/nsurl.c test/nsurl.c nsurl_CFLAGS := $(shell pkg-config --cflags libwapcaplet libdom) nsurl_LDFLAGS := $(shell pkg-config --libs libwapcaplet libdom) + nsoption_SRCS := utils/log.c utils/nsoption.c test/nsoption.c nsoption_CFLAGS := -Dnsgtk +sha1_SRCS := utils/sha1.c utils/log.c test/sha1.c +sha1_CFLAGS := +sha1_LDFLAGS := + .PHONY: all -all: llcache urldbtest nsurl nsoption +all: llcache urldbtest nsurl nsoption sha1 llcache: $(addprefix ../,$(llcache_SRCS)) $(CC) $(CFLAGS) $(llcache_CFLAGS) $^ -o $@ $(LDFLAGS) $(llcache_LDFLAGS) @@ -45,6 +53,9 @@ nsurl: $(addprefix ../,$(nsurl_SRCS)) nsoption: $(addprefix ../,$(nsoption_SRCS)) $(CC) $(CFLAGS) $(nsoption_CFLAGS) $^ -o $@ $(LDFLAGS) $(nsoption_LDFLAGS) +sha1: $(addprefix ../,$(sha1_SRCS)) + $(CC) $(CFLAGS) $(sha1_CFLAGS) $^ -o $@ $(LDFLAGS) $(sha1_LDFLAGS) + .PHONY: clean clean: diff --git a/test/sha1.c b/test/sha1.c new file mode 100644 index 0000000..abfc25b --- /dev/null +++ b/test/sha1.c @@ -0,0 +1,50 @@ +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> + +#include "utils/log.h" +#include "utils/sha1.h" + + +struct { + const char *text; + const uint8_t hash[NSSHA1_DIGEST_SIZE]; +} checks[] = { + { + "The quick brown fox jumps over the lazy dog", + { 0x2f, 0xd4, 0xe1, 0xc6, 0x7a, 0x2d, 0x28, 0xfc, 0xed, 0x84, + 0x9e, 0xe1, 0xbb, 0x76, 0xe7, 0x39, 0x1b, 0x93, 0xeb, 0x12 } + }, + { + "The quick brown fox jumps over the lazy cog", + { 0xde, 0x9f, 0x2c, 0x7f, 0xd2, 0x5e, 0x1b, 0x3a, 0xfa, 0xd3, + 0xe8, 0x5a, 0x0b, 0xd1, 0x7d, 0x9b, 0x10, 0x0d, 0xb4, 0xb3 } + }, + { + "", + { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, + 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09 } + }, + { NULL, {0} }, +}; + +int main(int argc, char **argv) +{ + uint8_t res[NSSHA1_DIGEST_SIZE]; + int chk_loop = 0; + + verbose_log = true; + + while (checks[chk_loop].text != NULL) { + + NSSHA1(checks[chk_loop].text, strlen(checks[chk_loop].text), res); + if (memcmp(res, &checks[chk_loop].hash, NSSHA1_DIGEST_SIZE) == 0) { + LOG(("Test %d: PASS", chk_loop)); + } else { + LOG(("Test %d: FAIL", chk_loop)); + } + chk_loop++; + } +} diff --git a/utils/Makefile b/utils/Makefile index aef5799..48bba98 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -2,6 +2,6 @@ S_UTILS := base64.c corestrings.c filename.c filepath.c hashtable.c \ libdom.c locale.c log.c messages.c nsurl.c talloc.c url.c \ - utf8.c utils.c useragent.c bloom.c nsoption.c + utf8.c utils.c useragent.c bloom.c nsoption.c sha1.c S_UTILS := $(addprefix utils/,$(S_UTILS)) diff --git a/utils/sha1.c b/utils/sha1.c new file mode 100644 index 0000000..882f2b9 --- /dev/null +++ b/utils/sha1.c @@ -0,0 +1,264 @@ +/* + SHA-1 in C + By Steve Reid <sreid(a)sea-to-sky.net&gt; + 100% Public Domain + + ----------------- + Modified 7/98 + By James H. Brown <jbrown(a)burgoyne.com&gt; + Still 100% Public Domain + + Corrected a problem which generated improper hash values on 16 bit machines + Routine SHA1Update changed from + void SHA1Update(SHA1_CTX* context, unsigned char* data, unsigned int + len) + to + void SHA1Update(SHA1_CTX* context, unsigned char* data, unsigned + long len) + + The 'len' parameter was declared an int which works fine on 32 bit machines. + However, on 16 bit machines an int is too small for the shifts being done + against + it. This caused the hash function to generate incorrect values if len was + greater than 8191 (8K - 1) due to the 'len << 3' on line 3 of SHA1Update(). + + Since the file IO in main() reads 16K at a time, any file 8K or larger would + be guaranteed to generate the wrong hash (e.g. Test Vector #3, a million + "a"s). + + I also changed the declaration of variables i & j in SHA1Update to + unsigned long from unsigned int for the same reason. + + These changes should make no difference to any 32 bit implementations since + an + int and a long are the same size in those environments. + + -- + I also corrected a few compiler warnings generated by Borland C. + 1. Added #include <process.h> for exit() prototype + 2. Removed unused variable 'j' in SHA1Final + 3. Changed exit(0) to return(0) at end of main. + + ALL changes I made can be located by searching for comments containing 'JHB' + ----------------- + Modified 8/98 + By Steve Reid <sreid(a)sea-to-sky.net&gt; + Still 100% public domain + + 1- Removed #include <process.h> and used return() instead of exit() + 2- Fixed overwriting of finalcount in SHA1Final() (discovered by Chris Hall) + 3- Changed email address from steve(a)edmweb.com to sreid(a)sea-to-sky.net + + ----------------- + Modified 4/01 + By Saul Kravitz <Saul.Kravitz(a)celera.com&gt; + Still 100% PD + Modified to run on Compaq Alpha hardware. + + ----------------- + Modified 07/2002 + By Ralph Giles <giles(a)ghostscript.com&gt; + Still 100% public domain + modified for use with stdint types, autoconf + code cleanup, removed attribution comments + switched SHA1Final() argument order for consistency + use SHA1_ prefix for public api + move public api to sha1.h +*/ + +/** \file + * steve reid's public domain SHA-1 implementation. + * + * Modified by Vincent Sanders <vince(a)netsurf-browser.org&gt; + * + * This is a sligly modified version of this public domain + * implementation changes were made to: + * - introduce suitable prefixes to avoid namespace conflicts. + * - make the context structure self contained. + * - make the function signatures more suitable for this use case + */ + + +/* see https://en.wikipedia.org/wiki/SHA-1 */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <stdlib.h> + +#include "utils/errors.h" +#include "utils/sha1.h" + +struct nssha1_ctx { + uint32_t state[5]; + uint32_t count[2]; + uint8_t buffer[64]; +} ; + +#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) + +/* blk0() and blk() perform the initial expand. */ +/* I got the idea of expanding during the round function from SSLeay */ +/* FIXME: can we do this in an endian-proof way? */ +#ifdef WORDS_BIGENDIAN +#define blk0(i) block->l[i] +#else +#define blk0(i) (block->l[i] = (rol(block->l[i],24)&0xFF00FF00) \ + |(rol(block->l[i],8)&0x00FF00FF)) +#endif +#define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \ + ^block->l[(i+2)&15]^block->l[i&15],1)) + +/* (R0+R1), R2, R3, R4 are the different operations used in SHA1 */ +#define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30); +#define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30); +#define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30); + + + +/* Hash a single 512-bit block. This is the core of the algorithm. */ +static void SHA1_Transform(uint32_t state[5], const uint8_t buffer[64]) +{ + uint32_t a, b, c, d, e; + typedef union { + uint8_t c[64]; + uint32_t l[16]; + } CHAR64LONG16; + CHAR64LONG16* block; + + block = (CHAR64LONG16*)buffer; + + /* Copy context->state[] to working vars */ + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + + /* 4 rounds of 20 operations each. Loop unrolled. */ + R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); + R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); + R0(c,d,e,a,b, 8); R0(b,c,d,e,a, 9); R0(a,b,c,d,e,10); R0(e,a,b,c,d,11); + R0(d,e,a,b,c,12); R0(c,d,e,a,b,13); R0(b,c,d,e,a,14); R0(a,b,c,d,e,15); + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + + /* Wipe variables */ + a = b = c = d = e = 0; +} + + +/* SHA1Init - Initialize new context */ +nserror +NSSHA1_Init(struct nssha1_ctx **ret_ctx) +{ + struct nssha1_ctx *context; + + context = calloc(1, sizeof(struct nssha1_ctx)); + if (context == NULL) { + return NSERROR_NOMEM; + } + + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; + + *ret_ctx = context; + + return NSERROR_OK; +} + + +/* Run your data through this. */ +nserror +NSSHA1_Update(struct nssha1_ctx *context, const uint8_t *data, const size_t len) +{ + size_t i, j; + + j = (context->count[0] >> 3) & 63; + if ((context->count[0] += len << 3) < (len << 3)) { + context->count[1]++; + } + context->count[1] += (len >> 29); + if ((j + len) > 63) { + memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1_Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) { + SHA1_Transform(context->state, data + i); + } + j = 0; + } else { + i = 0; + } + memcpy(&context->buffer[j], &data[i], len - i); + + return NSERROR_OK; +} + + +/* Add padding and return the message digest. */ +nserror +NSSHA1_Final(struct nssha1_ctx *context, uint8_t digest[NSSHA1_DIGEST_SIZE]) +{ + uint32_t i; + uint8_t finalcount[8]; + + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + NSSHA1_Update(context, (uint8_t *)"\200", 1); + while ((context->count[0] & 504) != 448) { + NSSHA1_Update(context, (uint8_t *)"\0", 1); + } + NSSHA1_Update(context, finalcount, 8); /* Should cause a SHA1_Transform() */ + for (i = 0; i < NSSHA1_DIGEST_SIZE; i++) { + digest[i] = (uint8_t) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + + free(context); + + return NSERROR_OK; +} + +nserror +NSSHA1(const uint8_t *data, const size_t len, uint8_t digest[NSSHA1_DIGEST_SIZE]) +{ + struct nssha1_ctx *context; + nserror error; + + error = NSSHA1_Init(&context); + if (error != NSERROR_OK) + return error; + NSSHA1_Update(context, data, len); + NSSHA1_Final(context, digest); + + return NSERROR_OK; +} diff --git a/utils/sha1.h b/utils/sha1.h new file mode 100644 index 0000000..e9f930c --- /dev/null +++ b/utils/sha1.h @@ -0,0 +1,72 @@ +/* + * Copyright 2014 vincent Sanders <vince(a)netsurf-browser.org&gt; + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>;. + */ + +/** \file + * interface for steve reid's public domain SHA-1 implementation. + * + * The exposed API makes only minimal changes to the public domain + * implementation, mainly adding a prefix so this implementation is + * not confused with any other similarly named version that might be + * linked. + */ + +#ifndef _NETSURF_UTILS_SHA1_H_ +#define _NETSURF_UTILS_SHA1_H_ + +struct nssha1_ctx; + +#define NSSHA1_DIGEST_SIZE 20 + +/** + * Initialise a SHA1 context ready to take updates + * + * The context is allocated by this routine and destroyed by the finaliser. + * + * @param context receives an initialised SHA1 context. + * @return NSERROR_OK and \a context updated on sucess or + * NSERROR_NOMEM on memory exhaustion. + */ +nserror NSSHA1_Init(struct nssha1_ctx **context); + +/** + * Update an SHA1 context with additional data. + * + * @param context The context created by NSSHA1_Init + * @param data The data to hash. + * @param len The length of data to hash. + * @return NSERROR_OK + */ +nserror NSSHA1_Update(struct nssha1_ctx *context, const uint8_t *data, const size_t len); + +/** + * Finalise a SHA1 hash. + * + * Frees the context. + * + * @param context The context created by NSSHA1_Init + * @param digest a buffer to store the results in. + * @return NSERROR_OK + */ +nserror NSSHA1_Final(struct nssha1_ctx *context, uint8_t digest[NSSHA1_DIGEST_SIZE]); + +/** + * Generate SHA1 of some data + */ +nserror NSSHA1(const uint8_t *data, const size_t len, uint8_t digest[NSSHA1_DIGEST_SIZE]); + +#endif commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=bba5a32dadf3620df1c... commit bba5a32dadf3620df1c1d42018e1006614e7f396 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; allow initialisation, finalisation and configutarion of the backing store diff --git a/content/hlcache.c b/content/hlcache.c index 618f4fd..6595462 100644 --- a/content/hlcache.c +++ b/content/hlcache.c @@ -339,9 +339,10 @@ static nserror hlcache_migrate_ctx(hlcache_retrieval_ctx *ctx, ctx->migrate_target = true; - if (effective_type != NULL && - hlcache_type_is_acceptable(effective_type, - ctx->accepted_types, &type)) { + if ((effective_type != NULL) && + hlcache_type_is_acceptable(effective_type, + ctx->accepted_types, + &type)) { error = hlcache_find_content(ctx, effective_type); if (error != NSERROR_OK && error != NSERROR_NEED_DATA) { if (ctx->handle->cb != NULL) { @@ -524,9 +525,7 @@ hlcache_initialise(const struct hlcache_parameters *hlcache_parameters) return NSERROR_NOMEM; } - ret = llcache_initialise(hlcache_parameters->cb, - hlcache_parameters->cb_ctx, - hlcache_parameters->limit); + ret = llcache_initialise(&hlcache_parameters->llcache); if (ret != NSERROR_OK) { free(hlcache); hlcache = NULL; diff --git a/content/hlcache.h b/content/hlcache.h index 41f1ed6..746b3c8 100644 --- a/content/hlcache.h +++ b/content/hlcache.h @@ -23,11 +23,12 @@ #ifndef NETSURF_CONTENT_HLCACHE_H_ #define NETSURF_CONTENT_HLCACHE_H_ -#include "content/content.h" -#include "content/llcache.h" #include "utils/errors.h" #include "utils/nsurl.h" +#include "content/content.h" +#include "content/llcache.h" + /** High-level cache handle */ typedef struct hlcache_handle hlcache_handle; @@ -44,18 +45,10 @@ typedef struct { } hlcache_event; struct hlcache_parameters { - llcache_query_callback cb; /**< Query handler for llcache */ - void *cb_ctx; /**< Pointer to llcache query handler data */ - /** How frequently the background cache clean process is run (ms) */ unsigned int bg_clean_time; - /** The target upper bound for the cache size */ - size_t limit; - - /** The hysteresis allowed round the target size */ - size_t hysteresis; - + struct llcache_parameters llcache; }; /** @@ -67,13 +60,13 @@ struct hlcache_parameters { * \return NSERROR_OK on success, appropriate error otherwise. */ typedef nserror (*hlcache_handle_callback)(hlcache_handle *handle, - const hlcache_event *event, void *pw); + const hlcache_event *event, void *pw); /** Flags for high-level cache object retrieval */ enum hlcache_retrieve_flag { - /* Note: low-level cache retrieval flags occupy the bottom 16 bits of - * the flags word. High-level cache flags occupy the top 16 bits. - * To avoid confusion, high-level flags are allocated from bit 31 down. + /* Note: low-level cache retrieval flags occupy the bottom 16 bits of + * the flags word. High-level cache flags occupy the top 16 bits. + * To avoid confusion, high-level flags are allocated from bit 31 down. */ /** It's permitted to convert this request into a download */ HLCACHE_RETRIEVE_MAY_DOWNLOAD = (1 << 31), @@ -84,7 +77,7 @@ enum hlcache_retrieve_flag { /** * Initialise the high-level cache, preparing the llcache also. * - * \param hlcache_parameters Settings to initialise cache with + * \param hlcache_parameters Settings to initialise cache with * \return NSERROR_OK on success, appropriate error otherwise. */ nserror hlcache_initialise(const struct hlcache_parameters *hlcache_parameters); @@ -133,7 +126,7 @@ nserror hlcache_poll(void); nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags, nsurl *referer, llcache_post_data *post, hlcache_handle_callback cb, void *pw, - hlcache_child_context *child, + hlcache_child_context *child, content_type accepted_types, hlcache_handle **result); /** @@ -169,13 +162,13 @@ nserror hlcache_handle_replace_callback(hlcache_handle *handle, * \param handle Cache handle to dereference * \return Pointer to content object, or NULL if there is none * - * \todo This may not be correct. Ideally, the client should never need to - * directly access a content object. It may, therefore, be better to provide a - * bunch of veneers here that take a hlcache_handle and invoke the + * \todo This may not be correct. Ideally, the client should never need to + * directly access a content object. It may, therefore, be better to provide a + * bunch of veneers here that take a hlcache_handle and invoke the * corresponding content_ API. If there's no content object associated with the - * hlcache_handle (e.g. because the source data is still being fetched, so it - * doesn't exist yet), then these veneers would behave as a NOP. The important - * thing being that the client need not care about this possibility and can + * hlcache_handle (e.g. because the source data is still being fetched, so it + * doesn't exist yet), then these veneers would behave as a NOP. The important + * thing being that the client need not care about this possibility and can * just call the functions with impugnity. */ struct content *hlcache_handle_get_content(const hlcache_handle *handle); diff --git a/content/llcache.c b/content/llcache.c index 85b025f..944e0a5 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -24,8 +24,7 @@ * store to extend their lifetime. * * \todo fix writeout conditions and ordering - * \todo add backing store initialisation to set storage parameters like location and size limits. - * \todo implement expiry from backing store + * \todo implement expiry from backing store of objects not currently in cache lists * \todo make the reference backing store have an index to improve performance * \todo make the reference backing store have a more efficient metadata store. * \todo make the reference backing store cope with filesystems that have filename length limits and directory entry count limits @@ -2013,6 +2012,26 @@ overflow: } /** + * construct a sorted list of objects available for writeout operation + * + * The list contains fresh cacheable objects held in RAM with no + * pending fetches. Any objects with a remaining lifetime less than + * LLCACHE_MIN_DISC_LIFETIME time are simply not considered, they will + * become stale before pushing to backing store is worth the cost. + * + * The list is sorted by object size, largest first + * + * \todo calculate useful cost metrics to improve sorting + * + */ +#if 0 +static struct llcache_object** +build_candidate_list() +{ +} +#endif + +/** * possibly push objects data to persiatant storage. */ static void llcache_persist(void *p) @@ -2547,8 +2566,8 @@ static nserror llcache_object_notify_users(llcache_object *object) * \param snapshot Pointer to receive snapshot of \a object * \return NSERROR_OK on success, appropriate error otherwise */ -static nserror llcache_object_snapshot(llcache_object *object, - llcache_object **snapshot) +static nserror +llcache_object_snapshot(llcache_object *object, llcache_object **snapshot) { llcache_object *newobj; nserror error; @@ -2664,6 +2683,11 @@ void llcache_clean(void) llcache_object_remove_from_list(object, &llcache->cached_objects); + + if (object->store_state == LLCACHE_STORE_DISC) { + guit->llcache->invalidate(object->url); + } + llcache_object_destroy(object); } } else { @@ -2674,7 +2698,8 @@ void llcache_clean(void) /* if the cache limit is exceeded try to make some objects * persistant so their RAM can be reclaimed in the next - * step */ + * step + */ if (llcache->limit < llcache_size) { llcache_persist(NULL); } @@ -2733,20 +2758,21 @@ void llcache_clean(void) /* See llcache.h for documentation */ nserror -llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit) +llcache_initialise(const struct llcache_parameters *prm) { llcache = calloc(1, sizeof(struct llcache_s)); if (llcache == NULL) { return NSERROR_NOMEM; } - llcache->query_cb = cb; - llcache->query_cb_pw = pw; - llcache->limit = llcache_limit; + llcache->query_cb = prm->cb; + llcache->query_cb_pw = prm->cb_ctx; + llcache->limit = prm->limit; - LOG(("llcache initialised with a limit of %d bytes", llcache_limit)); + LOG(("llcache initialised with a limit of %d bytes", llcache->limit)); - return NSERROR_OK; + /* backing store initialisation */ + return guit->llcache->initialise(&prm->store); } /* See llcache.h for documentation */ @@ -2796,6 +2822,9 @@ void llcache_finalise(void) llcache_object_destroy(object); } + /* backing store finalisation */ + guit->llcache->finalise(); + free(llcache); llcache = NULL; } diff --git a/content/llcache.h b/content/llcache.h index 3d8232c..6f12c48 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -76,7 +76,7 @@ typedef struct { } data; /**< Event data */ } llcache_event; -/** +/** * Client callback for low-level cache events * * \param handle Handle for which event is issued @@ -84,18 +84,18 @@ typedef struct { * \param pw Pointer to client-specific data * \return NSERROR_OK on success, appropriate error otherwise. */ -typedef nserror (*llcache_handle_callback)(llcache_handle *handle, +typedef nserror (*llcache_handle_callback)(llcache_handle *handle, const llcache_event *event, void *pw); /** Flags for low-level cache object retrieval */ enum llcache_retrieve_flag { /* Note: We're permitted a maximum of 16 flags which must reside in the - * bottom 16 bits of the flags word. See hlcache.h for further details. + * bottom 16 bits of the flags word. See hlcache.h for further details. */ /** Force a new fetch */ - LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0), + LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0), /** Requested URL was verified */ - LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1), + LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1), /**< No error pages */ LLCACHE_RETRIEVE_NO_ERROR_PAGES = (1 << 2), /**< Stream data (implies that object is not cacheable) */ @@ -149,21 +149,44 @@ typedef nserror (*llcache_query_response)(bool proceed, void *cbpw); * \param cbpw Opaque value to pass into \a cb * \return NSERROR_OK on success, appropriate error otherwise * - * \note This callback should return immediately. Once a suitable answer to - * the query has been obtained, the provided response callback should be + * \note This callback should return immediately. Once a suitable answer to + * the query has been obtained, the provided response callback should be * called. This is intended to be an entirely asynchronous process. */ typedef nserror (*llcache_query_callback)(const llcache_query *query, void *pw, llcache_query_response cb, void *cbpw); /** + * Parameters to configure the low level cache backing store. + */ +struct llcache_store_parameters { + const char *path; /**< The path to the backing store */ + + size_t limit; /**< The backing store upper bound target size */ + size_t hysteresis; /**< The hysteresis around the target size */ +}; + +/** + * Parameters to configure the low level cache. + */ +struct llcache_parameters { + llcache_query_callback cb; /**< Query handler for llcache */ + void *cb_ctx; /**< Pointer to llcache query handler data */ + + size_t limit; /**< The target upper bound for the RAM cache size */ + size_t hysteresis; /**< The hysteresis around the target size */ + + struct llcache_store_parameters store; +}; + +/** * Initialise the low-level cache * * \param cb Query handler * \param pw Pointer to query handler data * \return NSERROR_OK on success, appropriate error otherwise. */ -nserror llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit); +nserror llcache_initialise(const struct llcache_parameters *parameters); /** * Finalise the low-level cache @@ -280,12 +303,12 @@ const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle, * \return Header value, or NULL if header does not exist * * \todo Make the key an enumeration, to avoid needless string comparisons - * \todo Forcing the client to parse the header value seems wrong. - * Better would be to return the actual value part and an array of + * \todo Forcing the client to parse the header value seems wrong. + * Better would be to return the actual value part and an array of * key-value pairs for any additional parameters. * \todo Deal with multiple headers of the same key (e.g. Set-Cookie) */ -const char *llcache_handle_get_header(const llcache_handle *handle, +const char *llcache_handle_get_header(const llcache_handle *handle, const char *key); /** @@ -295,7 +318,7 @@ const char *llcache_handle_get_header(const llcache_handle *handle, * \param b Second handle * \return True if handles reference the same object, false otherwise */ -bool llcache_handle_references_same_object(const llcache_handle *a, +bool llcache_handle_references_same_object(const llcache_handle *a, const llcache_handle *b); #endif diff --git a/content/llcache_private.h b/content/llcache_private.h index f038c9d..dc6cb86 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,13 +25,19 @@ #include "content/llcache.h" -/** low level cache operation table +/** low level cache backing store operation table * * The low level cache (source objects) has the capability to make * objects and their metadata (headers etc) persistant by writing to a * backing store using these operations. */ struct gui_llcache_table { + /** Initialise the backing store. */ + nserror (*initialise)(const struct llcache_store_parameters *parameters); + + /** Finalise the backing store. */ + nserror (*finalise)(void); + /** Place a source object and its metadata in the backing store. */ nserror (*store)(struct nsurl *url, const uint8_t *metadata, const size_t metadatalen, diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index 8a5bcc6..a3772f6 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -346,6 +346,18 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) } static nserror +gui_default_initialise(const struct llcache_store_parameters *parameters) +{ + return NSERROR_OK; +} + +static nserror +gui_default_finalise(void) +{ + return NSERROR_OK; +} + +static nserror gui_default_store(nsurl *url, const uint8_t *metadata, const size_t metadatalen, const uint8_t *data, const size_t datalen) @@ -373,6 +385,8 @@ gui_default_invalidate(nsurl *url) static struct gui_llcache_table default_llcache_table = { + .initialise = gui_default_initialise, + .finalise = gui_default_finalise, .store = gui_default_store, .fetch = gui_default_fetch, .meta = gui_default_meta, @@ -400,6 +414,15 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } + + /* fill in the optional entries with defaults */ + if (glt->initialise == NULL) { + glt->initialise = gui_default_initialise; + } + if (glt->finalise == NULL) { + glt->finalise = gui_default_finalise; + } + return NSERROR_OK; } diff --git a/desktop/netsurf.c b/desktop/netsurf.c index c3653b5..582c119 100644 --- a/desktop/netsurf.c +++ b/desktop/netsurf.c @@ -67,11 +67,17 @@ */ #define SPECULATE_SMALL 4096 -/* the time between cache clean runs in ms */ +/** the time between image cache clean runs in ms */ #define IMAGE_CACHE_CLEAN_TIME (10 * 1000) +/** default time between content cache cleans */ #define HL_CACHE_CLEAN_TIME (2 * IMAGE_CACHE_CLEAN_TIME) +/** ensure there is a minimal amount of memory for source objetcs and + * decoded bitmaps. + */ +#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024) + bool netsurf_quit = false; static void netsurf_lwc_iterator(lwc_string *str, void *pw) @@ -108,20 +114,19 @@ static nserror netsurf_llcache_query_handler(const llcache_query *query, return NSERROR_OK; } -#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024) - /** * Initialise components used by gui NetSurf. */ nserror netsurf_init(const char *messages, struct gui_table *gt) { - nserror error; + nserror ret; struct utsname utsname; - nserror ret = NSERROR_OK; struct hlcache_parameters hlcache_parameters = { .bg_clean_time = HL_CACHE_CLEAN_TIME, - .cb = netsurf_llcache_query_handler, + .llcache = { + .cb = netsurf_llcache_query_handler, + } }; struct image_cache_parameters image_cache_parameters = { .bg_clean_time = IMAGE_CACHE_CLEAN_TIME, @@ -149,82 +154,94 @@ nserror netsurf_init(const char *messages, struct gui_table *gt) utsname.version, utsname.machine)); /* register the gui handlers */ - error = gui_factory_register(gt); - if (error != NSERROR_OK) - return error; + ret = gui_factory_register(gt); + if (ret != NSERROR_OK) + return ret; messages_load(messages); /* corestrings init */ - error = corestrings_init(); - if (error != NSERROR_OK) - return error; + ret = corestrings_init(); + if (ret != NSERROR_OK) + return ret; /* set up cache limits based on the memory cache size option */ - hlcache_parameters.limit = nsoption_int(memory_cache_size); + hlcache_parameters.llcache.limit = nsoption_int(memory_cache_size); - if (hlcache_parameters.limit < MINIMUM_MEMORY_CACHE_SIZE) { - hlcache_parameters.limit = MINIMUM_MEMORY_CACHE_SIZE; - LOG(("Setting minimum memory cache size to %d", - hlcache_parameters.limit)); + if (hlcache_parameters.llcache.limit < MINIMUM_MEMORY_CACHE_SIZE) { + hlcache_parameters.llcache.limit = MINIMUM_MEMORY_CACHE_SIZE; + LOG(("Setting minimum memory cache size %d", + hlcache_parameters.llcache.limit)); } /* image cache is 25% of total memory cache size */ - image_cache_parameters.limit = (hlcache_parameters.limit * 25) / 100; + image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100; /* image cache hysteresis is 20% of the image cache size */ image_cache_parameters.hysteresis = (image_cache_parameters.limit * 20) / 100; /* account for image cache use from total */ - hlcache_parameters.limit -= image_cache_parameters.limit; + hlcache_parameters.llcache.limit -= image_cache_parameters.limit; + + /* set backing store target limit */ + hlcache_parameters.llcache.store.limit = nsoption_int(disc_cache_size); + + /* set backing store hysterissi to 20% */ + hlcache_parameters.llcache.store.hysteresis = (hlcache_parameters.llcache.store.limit * 20) / 100;; + + /* set the path to the backing store */ + /** \todo set the backing store path properly */ + hlcache_parameters.llcache.store.path = "/tmp/ns/"; /* image handler bitmap cache */ - error = image_cache_init(&image_cache_parameters); - if (error != NSERROR_OK) - return error; + ret = image_cache_init(&image_cache_parameters); + if (ret != NSERROR_OK) + return ret; /* content handler initialisation */ - error = nscss_init(); - if (error != NSERROR_OK) - return error; + ret = nscss_init(); + if (ret != NSERROR_OK) + return ret; - error = html_init(); - if (error != NSERROR_OK) - return error; + ret = html_init(); + if (ret != NSERROR_OK) + return ret; - error = image_init(); - if (error != NSERROR_OK) - return error; + ret = image_init(); + if (ret != NSERROR_OK) + return ret; - error = textplain_init(); - if (error != NSERROR_OK) - return error; + ret = textplain_init(); + if (ret != NSERROR_OK) + return ret; - error = mimesniff_init(); - if (error != NSERROR_OK) - return error; + ret = mimesniff_init(); + if (ret != NSERROR_OK) + return ret; url_init(); setlocale(LC_ALL, "C"); /* initialise the fetchers */ - error = fetch_init(); - if (error != NSERROR_OK) - return error; + ret = fetch_init(); + if (ret != NSERROR_OK) + return ret; /* Initialise the hlcache and allow it to init the llcache for us */ - hlcache_initialise(&hlcache_parameters); + ret = hlcache_initialise(&hlcache_parameters); + if (ret != NSERROR_OK) + return ret; /* Initialize system colours */ - error = ns_system_colour_init(); - if (error != NSERROR_OK) - return error; + ret = ns_system_colour_init(); + if (ret != NSERROR_OK) + return ret; js_initialise(); - return ret; + return NSERROR_OK; } commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=69f37191bd59d28183c... commit 69f37191bd59d28183c40003d83d2c196d7537e0 Author: Vincent Sanders <vince(a)kyllikki.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; change metadata handling and cleanup header processing diff --git a/content/llcache.c b/content/llcache.c index 6d279dd..85b025f 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -17,7 +17,23 @@ */ /** \file - * Low-level resource cache (implementation) + * Low-level resource cache implementation + * + * This is the implementation of the low level cache. This cache + * stores source objects in memory and may use a persistant backing + * store to extend their lifetime. + * + * \todo fix writeout conditions and ordering + * \todo add backing store initialisation to set storage parameters like location and size limits. + * \todo implement expiry from backing store + * \todo make the reference backing store have an index to improve performance + * \todo make the reference backing store have a more efficient metadata store. + * \todo make the reference backing store cope with filesystems that have filename length limits and directory entry count limits + * \todo make backing store size bounded by configuration + * \todo support mmaped retrieve + * \todo make metadata buffer owned by backing store (avoid allocation and freeing) + * \todo instrument and (auto)tune + * \todo turn llcache debugging off */ #include <stdlib.h> @@ -47,6 +63,9 @@ #define LLCACHE_LOG(x) #endif +#define LLCACHE_MIN_DISC_LIFETIME 3600 +#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) + /** State of a low-level cache object fetch */ typedef enum { LLCACHE_FETCH_INIT, /**< Initial state, before fetch */ @@ -99,6 +118,30 @@ typedef struct { bool outstanding_query; /**< Waiting for a query response */ } llcache_fetch_ctx; +/** validation control */ +typedef enum { + LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ + LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ + LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ +} llcache_validate; + +/** cache control value for invalid age */ +#define INVALID_AGE -1 + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t fin_time; /**< Time of request completion */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + llcache_validate no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + /** Representation of a fetch header */ typedef struct { char *name; /**< Header name */ @@ -169,7 +212,6 @@ struct llcache_s { /** low level cache state */ static struct llcache_s *llcache = NULL; - /* forward referenced callback function */ static void llcache_fetch_callback(const fetch_msg *msg, void *p); @@ -399,7 +441,15 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, colon = (const uint8_t *) strchr((const char *) data, ':'); if (colon == NULL) { /* Failed, assume a key with no value */ - n = strdup((const char *) data); + colon = data + strlen((const char *)data); + + /* Strip trailing whitespace from name */ + while ((colon > data) && + (colon[-1] == ' ' || colon[-1] == '\t' || + colon[-1] == '\r' || colon[-1] == '\n')) { + colon--; + } + n = strndup((const char *) data, colon - data); if (n == NULL) return NSERROR_NOMEM; @@ -620,6 +670,13 @@ static nserror llcache_fetch_process_header(llcache_object *object, return error; } + /* deal with empty header */ + if (name[0] == 0) { + free(name); + free(value); + return NSERROR_OK; + } + /* Append header data to the object's headers array */ temp = realloc(object->headers, (object->num_headers + 1) * sizeof(llcache_header)); @@ -980,8 +1037,8 @@ static nserror llcache_object_clone_cache_data(llcache_object *source, * \param list List to remove from * \return NSERROR_OK */ -static nserror llcache_object_remove_from_list(llcache_object *object, - llcache_object **list) +static nserror +llcache_object_remove_from_list(llcache_object *object, llcache_object **list) { if (object == *list) *list = object->next; @@ -1002,10 +1059,9 @@ static nserror llcache_object_remove_from_list(llcache_object *object, * source data. * * @param object the object to operate on. - * + * @return apropriate error code. */ -static nserror -llcache_persist_retrieve(llcache_object *object) +static nserror llcache_persist_retrieve(llcache_object *object) { /* ensure the source data is present if necessary */ if ((object->source_data != NULL) || @@ -1017,31 +1073,134 @@ llcache_persist_retrieve(llcache_object *object) } /* Source data for the object may be in the persiatant store */ - return guit->llcache->retrieve(object->url, - NULL, - &object->source_data, - &object->source_len); + return guit->llcache->fetch(object->url, + &object->source_data, + &object->source_len); +} + + +static nserror +llcache_process_metadata(llcache_object *object) +{ + nserror res; + uint8_t *metadata = NULL; + size_t metadatalen = 0; + nsurl *metadataurl; + unsigned int line; + uint8_t *end; + char *ln; + int lnsize; + size_t num_headers; + size_t hloop; + + /* attempt to retrieve object metadata from the cache */ + res = guit->llcache->meta(object->url, &metadata, &metadatalen); + if (res != NSERROR_OK) + return res; + + end = metadata + metadatalen; + + /* metadata line 1 is the url the metadata referrs to */ + line = 1; + ln = (char *)metadata; + lnsize = strlen(ln); + + if (lnsize < 7) + goto format_error; + + res = nsurl_create(ln, &metadataurl); + if (res != NSERROR_OK) { + free(metadata); + return res; + } + + if (nsurl_compare(object->url, metadataurl, NSURL_COMPLETE) != true) { + /* backing store returned the wrong object for the + * request. This may occour if the backing store had + * a collision in its stoage method. We cope with this + * by simply skipping caching of this object. + */ + + LOG(("Got metadata for %s instead of %s", + nsurl_access(metadataurl), + nsurl_access(object->url))); + + nsurl_unref(metadataurl); + + free(metadata); + + return NSERROR_BAD_URL; + } + nsurl_unref(metadataurl); + + + /* metadata line 2 is the objects length */ + line = 2; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (sscanf(ln, "%zu", &object->source_len) != 1)) + goto format_error; + object->source_alloc = object->source_len; + + + /* metadata line 3 is the number of headers */ + line = 3; + ln += lnsize + 1; + lnsize = strlen(ln); + + if ((lnsize < 1) || + (sscanf(ln, "%zu", &num_headers) != 1)) + goto format_error; + + + /* read headers */ + for (hloop = 0 ; hloop < num_headers; hloop++) { + line++; + ln += lnsize + 1; + lnsize = strlen(ln); + + res = llcache_fetch_process_header(object, (uint8_t *)ln, lnsize); + if (res != NSERROR_OK) { + free(metadata); + return res; + } + } + + free(metadata); + + /* object stored in backing store */ + object->store_state = LLCACHE_STORE_DISC; + + return NSERROR_OK; + +format_error: + LOG(("metadata error on line %d\n", line)); + free(metadata); + return NSERROR_INVALID; + } /** - * attempt to retrieve an object from persistant storage + * attempt to retrieve an object from persistant storage. */ static nserror llcache_object_fetch_persistant(llcache_object *object, - uint32_t flags, - nsurl *referer, - const llcache_post_data *post, - uint32_t redirect_count) + uint32_t flags, + nsurl *referer, + const llcache_post_data *post, + uint32_t redirect_count) { nserror error; nsurl *referer_clone = NULL; llcache_post_data *post_clone = NULL; - /* attempt to retrieve object from the cache */ - error = guit->llcache->retrieve(object->url, - &object->cache, - &object->source_data, - &object->source_len); + object->cache.req_time = time(NULL); + object->cache.fin_time = object->cache.req_time; + + /* retrieve and process metadata */ + error = llcache_process_metadata(object); if (error != NSERROR_OK) return error; @@ -1064,9 +1223,6 @@ llcache_object_fetch_persistant(llcache_object *object, object->fetch.state = LLCACHE_FETCH_COMPLETE; object->fetch.fetch = NULL; - object->store_state = LLCACHE_STORE_DISC; - object->source_alloc = object->source_len; - return NSERROR_OK; } @@ -1772,8 +1928,89 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) return error; } -#define LLCACHE_MIN_DISC_LIFETIME 3600 -#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) +/** + * Generate a serialised version of an objects metadata + * + * metadata includes object headers + */ +static nserror +llcache_serialise_metadata(llcache_object *object, + uint8_t **data_out, + size_t *datasize_out) +{ + size_t allocsize; + int datasize; + uint8_t *data; + char *op; + unsigned int hloop; + int use; + + allocsize = 10 + 1; /* object length */ + + allocsize += 10 + 1; /* space for number of header entries */ + + allocsize += nsurl_length(object->url) + 1; + + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + allocsize += strlen(object->headers[hloop].name) + 1; + allocsize += strlen(object->headers[hloop].value) + 1; + } + + data = malloc(allocsize); + if (data == NULL) { + return NSERROR_NOMEM; + } + + op = (char *)data; + datasize = allocsize; + + /* the url, used for checking for collisions */ + use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* object size */ + use = snprintf(op, datasize, "%zu%c", object->source_len, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* number of headers */ + use = snprintf(op, datasize, "%zu%c", object->num_headers, 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + + /* headers */ + for (hloop = 0 ; hloop < object->num_headers ; hloop++) { + use = snprintf(op, datasize, + "%s:%s%c", + object->headers[hloop].name, + object->headers[hloop].value, + 0); + if (use > datasize) + goto overflow; + op += use; + datasize -= use; + } + + LLCACHE_LOG(("Filled buffer with %d spare", datasize)); + + *data_out = data; + *datasize_out = allocsize - datasize; + + return NSERROR_OK; + +overflow: + /* somehow we overflowed the buffer - hth? */ + LOG(("Overflowed metadata buffer")); + free(data); + return NSERROR_INVALID; +} /** * possibly push objects data to persiatant storage. @@ -1799,11 +2036,25 @@ static void llcache_persist(void *p) (object->fetch.outstanding_query == false) && (object->store_state == LLCACHE_STORE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + uint8_t *metadata; + size_t metadatasize; + + ret = llcache_serialise_metadata(object, &metadata, &metadatasize); + if (ret != NSERROR_OK) { + /* as there has been a serialisation + * error, give up on making any more + * objects persistant for now. + */ + return; + } + /* ok found an object to write */ - ret = guit->llcache->persist(object->url, - &object->cache, - object->source_data, - object->source_len); + ret = guit->llcache->store(object->url, + metadata, + metadatasize, + object->source_data, + object->source_len); + free(metadata); if (ret != NSERROR_OK) { /* as there has been a serialisation * error, give up on making any more @@ -1812,7 +2063,7 @@ static void llcache_persist(void *p) return; } object->store_state = LLCACHE_STORE_DISC; - size_written += object->source_len; + size_written += object->source_len + metadatasize; if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { break; diff --git a/content/llcache_private.h b/content/llcache_private.h index 926773b..f038c9d 100644 --- a/content/llcache_private.h +++ b/content/llcache_private.h @@ -25,34 +25,31 @@ #include "content/llcache.h" -/** validation control */ -typedef enum { - LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ - LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ - LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ -} llcache_validate; - -/** cache control value for invalid age */ -#define INVALID_AGE -1 - -/** Cache control data */ -typedef struct { - time_t req_time; /**< Time of request */ - time_t res_time; /**< Time of response */ - time_t fin_time; /**< Time of request completion */ - time_t date; /**< Date: response header */ - time_t expires; /**< Expires: response header */ - int age; /**< Age: response header */ - int max_age; /**< Max-Age Cache-control parameter */ - llcache_validate no_cache; /**< No-Cache Cache-control parameter */ - char *etag; /**< Etag: response header */ - time_t last_modified; /**< Last-Modified: response header */ -} llcache_cache_control; - -/** operation table */ +/** low level cache operation table + * + * The low level cache (source objects) has the capability to make + * objects and their metadata (headers etc) persistant by writing to a + * backing store using these operations. + */ struct gui_llcache_table { - nserror (*persist)(struct nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen); - nserror (*retrieve)(struct nsurl *url, llcache_cache_control *cache_control, uint8_t **data, size_t *datalen); + /** Place a source object and its metadata in the backing store. */ + nserror (*store)(struct nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen); + + /** retrive source object data from backing store. */ + nserror (*fetch)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** retrive source object metadata from backing store. */ + nserror (*meta)(struct nsurl *url, uint8_t **data, size_t *datalen); + + /** + * invalidate a source object from the backing store. + * + * The entry (if present in the backing store) must no longer + * be returned as a result to the fetch or meta operations. + */ + nserror (*invalidate)(struct nsurl *url); }; #endif diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index 37fee39..8a5bcc6 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -345,19 +345,38 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } -static nserror gui_default_persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +static nserror +gui_default_store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { return NSERROR_SAVE_FAILED; } -static nserror gui_default_retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) +static nserror +gui_default_fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } +static nserror +gui_default_meta(nsurl *url, uint8_t **data_out, size_t *datalen_out) +{ + return NSERROR_NOT_FOUND; +} + +static nserror +gui_default_invalidate(nsurl *url) +{ + return NSERROR_NOT_FOUND; +} + + static struct gui_llcache_table default_llcache_table = { - .persist = gui_default_persist, - .retrieve = gui_default_retrieve, + .store = gui_default_store, + .fetch = gui_default_fetch, + .meta = gui_default_meta, + .invalidate = gui_default_invalidate, }; /** verify clipboard table is valid */ @@ -369,10 +388,16 @@ static nserror verify_llcache_register(struct gui_llcache_table *glt) } /* mandantory operations */ - if (glt->persist == NULL) { + if (glt->store == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->fetch == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->meta == NULL) { return NSERROR_BAD_PARAMETER; } - if (glt->retrieve == NULL) { + if (glt->invalidate == NULL) { return NSERROR_BAD_PARAMETER; } return NSERROR_OK; diff --git a/gtk/llcache.c b/gtk/llcache.c index a2c27e5..4fa4e1e 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -41,9 +41,10 @@ static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', static unsigned int mod_table[] = {0, 2, 1}; -static uint8_t *base64url_encode(const char *data, - size_t input_length, - size_t *output_length) +static uint8_t * +base64url_encode(const char *data, + size_t input_length, + size_t *output_length) { uint8_t *encoded_data; size_t i; @@ -79,7 +80,9 @@ static uint8_t *base64url_encode(const char *data, } static nserror -persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) +store(nsurl *url, + const uint8_t *metadata, const size_t metadatalen, + const uint8_t *data, const size_t datalen) { uint8_t *b64u; size_t b64ulen; @@ -101,7 +104,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - LOG(("Writing %d bytes at %p to file", datalen, data)); + LOG(("Writing %d bytes from %p", datalen, data)); if (fwrite(data, datalen, 1, file) != 1) { LOG(("did not return 1")); fclose(file); @@ -123,27 +126,14 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c return NSERROR_SAVE_FAILED; } - fprintf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%s\n", - (long long)cache_control->req_time, - (long long)cache_control->res_time, - (long long)cache_control->fin_time, - (long long)cache_control->date, - (long long)cache_control->expires, - cache_control->age, - cache_control->max_age, - cache_control->no_cache, - (long long)cache_control->last_modified, - cache_control->etag == NULL?"":cache_control->etag); + LOG(("Writing %d bytes from %p", metadatalen, metadata)); + if (fwrite(metadata, metadatalen, 1, file) != 1) { + LOG(("did not return 1")); + fclose(file); + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } fclose(file); @@ -154,10 +144,7 @@ persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, c } static nserror -retrieve(nsurl *url, - llcache_cache_control *cache_control, - uint8_t **data_out, - size_t *datalen_out) +fetch(nsurl *url, uint8_t **data_out, size_t *datalen_out) { uint8_t *b64u; size_t b64ulen; @@ -166,7 +153,6 @@ retrieve(nsurl *url, uint8_t *data; size_t datalen; - data = *data_out; datalen = *datalen_out; @@ -176,51 +162,73 @@ retrieve(nsurl *url, &b64ulen); fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); - /* retrive cache control info if necessary */ - if (cache_control != NULL) { - int cnt; - snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); + free(fname); + if (file == NULL) { + free(b64u); + return NSERROR_NOT_FOUND; + } - file = fopen(fname, "rb"); - if (file == NULL) { - LOG(("Unable to open %s", fname)); - free(fname); - free(b64u); - return NSERROR_NOT_FOUND; + /* need to deal with buffers */ + if (data == NULL) { + if (datalen == 0) { + /* caller did not know the files length */ + fseek(file, 0L, SEEK_END); + datalen = ftell(file); + fseek(file, 0L, SEEK_SET); } - LOG(("Opening info file %s", fname)); - - cnt = fscanf(file, - "req_time:%lld\n" - "res_time:%lld\n" - "fin_time:%lld\n" - "date:%lld\n" - "expires:%lld\n" - "age:%d\n" - "max_age:%d\n" - "no_cache:%d\n" - "last_modified:%lld\n" - "etag:%ms\n", - (long long*)&cache_control->req_time, - (long long*)&cache_control->res_time, - (long long*)&cache_control->fin_time, - (long long*)&cache_control->date, - (long long*)&cache_control->expires, - &cache_control->age, - &cache_control->max_age, - (int *)&cache_control->no_cache, - (long long*)&cache_control->last_modified, - &cache_control->etag); - - LOG(("Read %d items from cache constrol", cnt)); + data = malloc(datalen); + if (data == NULL) { + fclose(file); + free(b64u); + return NSERROR_NOMEM; + } + } + LOG(("Reading %d bytes into %p from file", datalen, data)); + if (fread(data, datalen, 1, file) != 1) { + LOG(("did not return 1")); fclose(file); - + free(b64u); + if ((*data_out) == NULL) { + free(data); + } + return NSERROR_NOT_FOUND; } - snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + fclose(file); + free(b64u); + + *data_out = data; + *datalen_out = datalen; + + return NSERROR_OK; +} + +static nserror +fetchmeta(nsurl *url, uint8_t **metadata_out, size_t *metadatalen_out) +{ + uint8_t *b64u; + size_t b64ulen; + char *fname; + FILE *file; + uint8_t *data; + size_t datalen; + + data = *metadata_out; + datalen = *metadatalen_out; + + LOG(("retriving cache file for url:%s", nsurl_access(url))); + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(SLEN("/tmp/ns/i/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); LOG(("Opening file %s",fname)); file = fopen(fname, "rb"); @@ -230,7 +238,7 @@ retrieve(nsurl *url, return NSERROR_NOT_FOUND; } - /* need to deal with buffers */ + /* need to deal with allocating buffers */ if (data == NULL) { if (datalen == 0) { /* caller did not know the files length */ @@ -252,7 +260,7 @@ retrieve(nsurl *url, LOG(("did not return 1")); fclose(file); free(b64u); - if ((*data_out) == NULL) { + if ((*metadata_out) == NULL) { free(data); } return NSERROR_NOT_FOUND; @@ -261,15 +269,24 @@ retrieve(nsurl *url, fclose(file); free(b64u); - *data_out = data; - *datalen_out = datalen; + *metadata_out = data; + *metadatalen_out = datalen; return NSERROR_OK; } +static nserror +invalidate(nsurl *url) +{ + return NSERROR_OK; +} + + static struct gui_llcache_table llcache_table = { - .persist = persist, - .retrieve = retrieve, + .store = store, + .fetch = fetch, + .meta = fetchmeta, + .invalidate = invalidate, }; struct gui_llcache_table *nsgtk_llcache_table = &llcache_table; commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=3ad83424d49fddc66c9... commit 3ad83424d49fddc66c9d30c5ce07d9d6582a8ac1 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; basic persistant sorage for gtk frontend diff --git a/gtk/llcache.c b/gtk/llcache.c index 3587c0d..a2c27e5 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -16,24 +16,255 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>;. */ +/** \file + * Low-level resource cache persistant storage implementation. + */ + +#include <string.h> + #include "utils/nsurl.h" #include "utils/log.h" +#include "utils/utils.h" #include "desktop/gui.h" #include "content/llcache_private.h" #include "gtk/llcache.h" +static uint8_t encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_'}; +static unsigned int mod_table[] = {0, 2, 1}; + + +static uint8_t *base64url_encode(const char *data, + size_t input_length, + size_t *output_length) +{ + uint8_t *encoded_data; + size_t i; + size_t j; + + *output_length = 4 * ((input_length + 2) / 3); + + encoded_data = malloc(*output_length + 1); + if (encoded_data == NULL) { + return NULL; + } + + for (i = 0, j = 0; i < input_length;) { + + uint32_t octet_a = i < input_length ? data[i++] : 0; + uint32_t octet_b = i < input_length ? data[i++] : 0; + uint32_t octet_c = i < input_length ? data[i++] : 0; + + uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + encoded_data[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 1 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 0 * 6) & 0x3F]; + } + + for (i = 0; i < mod_table[input_length % 3]; i++) { + encoded_data[*output_length - 1 - i] = 0; + (*output_length)--; + } + + return encoded_data; +} + static nserror persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) { + uint8_t *b64u; + size_t b64ulen; + char *fname; + FILE *file; + LOG(("Writing cache file for url:%s", nsurl_access(url))); + b64u = base64url_encode(nsurl_access(url), strlen(nsurl_access(url)), &b64ulen); + + fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); + snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + + LOG(("Opening data file %s", fname)); + file = fopen(fname, "wb"); + + if (file == NULL) { + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } + + LOG(("Writing %d bytes at %p to file", datalen, data)); + if (fwrite(data, datalen, 1, file) != 1) { + LOG(("did not return 1")); + fclose(file); + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } + + fclose(file); + + snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + + LOG(("Opening info file %s", fname)); + file = fopen(fname, "wb"); + + if (file == NULL) { + free(fname); + free(b64u); + return NSERROR_SAVE_FAILED; + } + + fprintf(file, + "req_time:%lld\n" + "res_time:%lld\n" + "fin_time:%lld\n" + "date:%lld\n" + "expires:%lld\n" + "age:%d\n" + "max_age:%d\n" + "no_cache:%d\n" + "last_modified:%lld\n" + "etag:%s\n", + (long long)cache_control->req_time, + (long long)cache_control->res_time, + (long long)cache_control->fin_time, + (long long)cache_control->date, + (long long)cache_control->expires, + cache_control->age, + cache_control->max_age, + cache_control->no_cache, + (long long)cache_control->last_modified, + cache_control->etag == NULL?"":cache_control->etag); + + fclose(file); + + free(fname); + free(b64u); + return NSERROR_OK; } static nserror -retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) +retrieve(nsurl *url, + llcache_cache_control *cache_control, + uint8_t **data_out, + size_t *datalen_out) { - return NSERROR_NOT_FOUND; + uint8_t *b64u; + size_t b64ulen; + char *fname; + FILE *file; + uint8_t *data; + size_t datalen; + + + data = *data_out; + datalen = *datalen_out; + + LOG(("retriving cache file for url:%s", nsurl_access(url))); + b64u = base64url_encode(nsurl_access(url), + strlen(nsurl_access(url)), + &b64ulen); + + fname = malloc(SLEN("/tmp/ns/d/") + b64ulen + 1); + + /* retrive cache control info if necessary */ + if (cache_control != NULL) { + int cnt; + snprintf(fname, SLEN("/tmp/ns/i/") + b64ulen + 1, "/tmp/ns/i/%s", b64u); + + file = fopen(fname, "rb"); + if (file == NULL) { + LOG(("Unable to open %s", fname)); + free(fname); + free(b64u); + return NSERROR_NOT_FOUND; + } + LOG(("Opening info file %s", fname)); + + cnt = fscanf(file, + "req_time:%lld\n" + "res_time:%lld\n" + "fin_time:%lld\n" + "date:%lld\n" + "expires:%lld\n" + "age:%d\n" + "max_age:%d\n" + "no_cache:%d\n" + "last_modified:%lld\n" + "etag:%ms\n", + (long long*)&cache_control->req_time, + (long long*)&cache_control->res_time, + (long long*)&cache_control->fin_time, + (long long*)&cache_control->date, + (long long*)&cache_control->expires, + &cache_control->age, + &cache_control->max_age, + (int *)&cache_control->no_cache, + (long long*)&cache_control->last_modified, + &cache_control->etag); + + LOG(("Read %d items from cache constrol", cnt)); + + + fclose(file); + + } + + snprintf(fname, SLEN("/tmp/ns/d/") + b64ulen + 1, "/tmp/ns/d/%s", b64u); + + LOG(("Opening file %s",fname)); + file = fopen(fname, "rb"); + free(fname); + if (file == NULL) { + free(b64u); + return NSERROR_NOT_FOUND; + } + + /* need to deal with buffers */ + if (data == NULL) { + if (datalen == 0) { + /* caller did not know the files length */ + fseek(file, 0L, SEEK_END); + datalen = ftell(file); + fseek(file, 0L, SEEK_SET); + } + + data = malloc(datalen); + if (data == NULL) { + fclose(file); + free(b64u); + return NSERROR_NOMEM; + } + } + + LOG(("Reading %d bytes into %p from file", datalen, data)); + if (fread(data, datalen, 1, file) != 1) { + LOG(("did not return 1")); + fclose(file); + free(b64u); + if ((*data_out) == NULL) { + free(data); + } + return NSERROR_NOT_FOUND; + } + + fclose(file); + free(b64u); + + *data_out = data; + *datalen_out = datalen; + + return NSERROR_OK; } static struct gui_llcache_table llcache_table = { commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=9666c6dbc7a3dbb2b21... commit 9666c6dbc7a3dbb2b216e83b908a568f7837207a Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; fetch from persistant storage for new objects before fetching from network diff --git a/content/llcache.c b/content/llcache.c index 46dc6af..6d279dd 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -389,6 +389,12 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, char *n, *v; const uint8_t *colon; + /* Strip leading whitespace from name */ + while (data[0] == ' ' || data[0] == '\t' || + data[0] == '\r' || data[0] == '\n') { + data++; + } + /* Find colon */ colon = (const uint8_t *) strchr((const char *) data, ':'); if (colon == NULL) { @@ -405,12 +411,6 @@ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, } else { /* Split header into name & value */ - /* Strip leading whitespace from name */ - while (data[0] == ' ' || data[0] == '\t' || - data[0] == '\r' || data[0] == '\n') { - data++; - } - /* Strip trailing whitespace from name */ while (colon > data && (colon[-1] == ' ' || colon[-1] == '\t' || colon[-1] == '\r' || @@ -1018,9 +1018,56 @@ llcache_persist_retrieve(llcache_object *object) /* Source data for the object may be in the persiatant store */ return guit->llcache->retrieve(object->url, + NULL, + &object->source_data, + &object->source_len); +} + +/** + * attempt to retrieve an object from persistant storage + */ +static nserror +llcache_object_fetch_persistant(llcache_object *object, + uint32_t flags, + nsurl *referer, + const llcache_post_data *post, + uint32_t redirect_count) +{ + nserror error; + nsurl *referer_clone = NULL; + llcache_post_data *post_clone = NULL; + + /* attempt to retrieve object from the cache */ + error = guit->llcache->retrieve(object->url, &object->cache, &object->source_data, &object->source_len); + if (error != NSERROR_OK) + return error; + + /* entry came out of cache - need to setup object state */ + if (post != NULL) { + error = llcache_post_data_clone(post, &post_clone); + if (error != NSERROR_OK) + return error; + } + + if (referer != NULL) + referer_clone = nsurl_ref(referer); + + object->fetch.flags = flags; + object->fetch.referer = referer_clone; + object->fetch.post = post_clone; + object->fetch.redirect_count = redirect_count; + + /* fetch is "finished" */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + object->fetch.fetch = NULL; + + object->store_state = LLCACHE_STORE_DISC; + object->source_alloc = object->source_len; + + return NSERROR_OK; } /** @@ -1045,8 +1092,8 @@ llcache_object_retrieve_from_cache(nsurl *url, nserror error; llcache_object *obj, *newest = NULL; - LLCACHE_LOG(("Searching cache for %s (%x %p %p)", - nsurl_access(url), flags, referer, post)); + LLCACHE_LOG(("Searching cache for %s flags:%x referer:%s post:%p", + nsurl_access(url), flags, referer==NULL?"":nsurl_access(referer), post)); /* Search for the most recently fetched matching object */ for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) { @@ -1059,9 +1106,33 @@ llcache_object_retrieve_from_cache(nsurl *url, } } + /* No viable object found in cache create one and attempt to + * pull from persistant store. + */ + if (newest == NULL) { + LLCACHE_LOG(("No viable object found in cache")); + + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* attempt to retrieve object from persistant store */ + error = llcache_object_fetch_persistant(obj, flags, referer, post, redirect_count); + if (error == NSERROR_OK) { + /* set object from persistant store as newest */ + newest = obj; + + /* Add new object to cached object list */ + llcache_object_add_to_list(obj, &llcache->cached_objects); + + } + /* else no object found and unretrivable from cache, + * fall through to start fetch + */ + } + if ((newest != NULL) && (llcache_object_is_fresh(newest))) { /* Found a suitable object, and it's still fresh */ - LLCACHE_LOG(("Found fresh %p", newest)); /* The client needs to catch up with the object's state. @@ -1088,6 +1159,10 @@ llcache_object_retrieve_from_cache(nsurl *url, llcache_object_remove_from_list(newest, &llcache->cached_objects); llcache_object_destroy(newest); + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) { + return error; + } } else if (newest != NULL) { /* Found a candidate object but it needs freshness validation */ @@ -1132,22 +1207,22 @@ llcache_object_retrieve_from_cache(nsurl *url, LLCACHE_LOG(("Persistant retrival failed for %p", newest)); + /* retrival of source data from persistant store + * failed, destroy cache object and fall though to + * cache miss to re-retch + */ llcache_object_remove_from_list(newest, &llcache->cached_objects); llcache_object_destroy(newest); - } - - /* No viable object found in cache; create a new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; - - LLCACHE_LOG(("Not found %p", obj)); + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) { + return error; + } + } /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); + error = llcache_object_fetch(obj, flags, referer, post, redirect_count); if (error != NSERROR_OK) { llcache_object_destroy(obj); return error; @@ -1181,8 +1256,8 @@ static nserror llcache_object_retrieve(nsurl *url, uint32_t flags, bool has_query; nsurl *defragmented_url; - LLCACHE_LOG(("Retrieve %s (%x, %p, %p)", - nsurl_access(url), flags, referer, post)); + LLCACHE_LOG(("Retrieve %s (%x, %s, %p)", nsurl_access(url), flags, + referer==NULL?"":nsurl_access(referer), post)); /** * Caching Rules: @@ -1708,7 +1783,7 @@ static void llcache_persist(void *p) llcache_object *object, *next; int remaining_lifetime; nserror ret; - size_t size_written; + size_t size_written = 0; for (object = llcache->cached_objects; object != NULL; object = next) { next = object->next; @@ -2372,8 +2447,8 @@ void llcache_clean(void) llcache_size -= object->source_len; LLCACHE_LOG(("Freeing source data for %p len:%d", - object, - object->source_len)); + object, + object->source_len)); } } @@ -2390,11 +2465,10 @@ void llcache_clean(void) (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && (object->store_state == LLCACHE_STORE_RAM)) { - LLCACHE_LOG(("destroying cacahable object:%p len:%d age:%d", - object, - object->source_len, - time(NULL) - object->last_used)); - + LLCACHE_LOG(("discarding object:%p len:%d age:%d", + object, + object->source_len, + time(NULL) - object->last_used)); llcache_size -= object->source_len + sizeof(*object); llcache_object_remove_from_list(object, commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=bbe3ef1627c6150a1ff... commit bbe3ef1627c6150a1ff111488b202cca1bf07e74 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; improve frontend operation interface diff --git a/content/llcache.c b/content/llcache.c index 1345845..46dc6af 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -34,7 +34,7 @@ #include "desktop/gui_factory.h" #include "content/fetch.h" -#include "content/llcache.h" +#include "content/llcache_private.h" #include "content/urldb.h" /** Define to enable tracing of llcache operations. */ @@ -99,27 +99,6 @@ typedef struct { bool outstanding_query; /**< Waiting for a query response */ } llcache_fetch_ctx; -typedef enum { - LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ - LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ - LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ -} llcache_validate; - -/** Cache control data */ -typedef struct { - time_t req_time; /**< Time of request */ - time_t res_time; /**< Time of response */ - time_t fin_time; /**< Time of request completion */ - time_t date; /**< Date: response header */ - time_t expires; /**< Expires: response header */ -#define INVALID_AGE -1 - int age; /**< Age: response header */ - int max_age; /**< Max-Age Cache-control parameter */ - llcache_validate no_cache; /**< No-Cache Cache-control parameter */ - char *etag; /**< Etag: response header */ - time_t last_modified; /**< Last-Modified: response header */ -} llcache_cache_control; - /** Representation of a fetch header */ typedef struct { char *name; /**< Header name */ @@ -1039,7 +1018,7 @@ llcache_persist_retrieve(llcache_object *object) /* Source data for the object may be in the persiatant store */ return guit->llcache->retrieve(object->url, - NULL, + &object->cache, &object->source_data, &object->source_len); } @@ -1747,7 +1726,7 @@ static void llcache_persist(void *p) (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { /* ok found an object to write */ ret = guit->llcache->persist(object->url, - remaining_lifetime, + &object->cache, object->source_data, object->source_len); if (ret != NSERROR_OK) { diff --git a/content/llcache.h b/content/llcache.h index 1faba81..3d8232c 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -131,12 +131,6 @@ typedef struct { } data; } llcache_query; -/** operation table */ -struct gui_llcache_table { - nserror (*persist)(struct nsurl *url, int remaining, const uint8_t *data, const size_t datalen); - nserror (*retrieve)(struct nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out); -}; - /** * Response handler for fetch-related queries * diff --git a/content/llcache_private.h b/content/llcache_private.h new file mode 100644 index 0000000..926773b --- /dev/null +++ b/content/llcache_private.h @@ -0,0 +1,58 @@ +/* + * Copyright 2014 Vincent Sanders <vince(a)netsurf-browser.org&gt; + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>;. + */ + +/** \file + * Low-level resource cache private interface + */ + +#ifndef NETSURF_CONTENT_LLCACHE_PRIVATE_H_ +#define NETSURF_CONTENT_LLCACHE_PRIVATE_H_ + +#include "content/llcache.h" + +/** validation control */ +typedef enum { + LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */ + LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */ + LLCACHE_VALIDATE_ONCE /**< Revalidate once only */ +} llcache_validate; + +/** cache control value for invalid age */ +#define INVALID_AGE -1 + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t fin_time; /**< Time of request completion */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + llcache_validate no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + +/** operation table */ +struct gui_llcache_table { + nserror (*persist)(struct nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen); + nserror (*retrieve)(struct nsurl *url, llcache_cache_control *cache_control, uint8_t **data, size_t *datalen); +}; + +#endif diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index e7d2994..37fee39 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -17,6 +17,8 @@ */ #include "content/hlcache.h" +#include "content/llcache_private.h" + #include "desktop/download.h" #include "desktop/gui_factory.h" @@ -343,12 +345,12 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } -static nserror gui_default_persist(nsurl *url, int remaining, const uint8_t *data, const size_t datalen) +static nserror gui_default_persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) { return NSERROR_SAVE_FAILED; } -static nserror gui_default_retrieve(nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out) +static nserror gui_default_retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } diff --git a/gtk/llcache.c b/gtk/llcache.c index ef00641..3587c0d 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -19,19 +19,19 @@ #include "utils/nsurl.h" #include "utils/log.h" #include "desktop/gui.h" -#include "content/llcache.h" +#include "content/llcache_private.h" #include "gtk/llcache.h" static nserror -persist(nsurl *url, int remaining, const uint8_t *data, const size_t datalen) +persist(nsurl *url, llcache_cache_control *cache_control, const uint8_t *data, const size_t datalen) { LOG(("Writing cache file for url:%s", nsurl_access(url))); return NSERROR_OK; } static nserror -retrieve(nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out) +retrieve(nsurl *url, llcache_cache_control *cache_control, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=e5550e3c2ab0b297681... commit e5550e3c2ab0b2976812803e1f56ba79a5cdf690 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; make the object retrival from persistant storage much more obvious diff --git a/content/llcache.c b/content/llcache.c index 279b544..1345845 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -1016,6 +1016,35 @@ static nserror llcache_object_remove_from_list(llcache_object *object, } /** + * Retrieve source data for an object from persistant store if necessary. + * + * If an objects source data has been placed in the persistant store + * and the in memory copy freed this will attempt to retrive the + * source data. + * + * @param object the object to operate on. + * + */ +static nserror +llcache_persist_retrieve(llcache_object *object) +{ + /* ensure the source data is present if necessary */ + if ((object->source_data != NULL) || + (object->store_state != LLCACHE_STORE_DISC)) { + /* source data does not require retriving from + * persistant store. + */ + return NSERROR_OK; + } + + /* Source data for the object may be in the persiatant store */ + return guit->llcache->retrieve(object->url, + NULL, + &object->source_data, + &object->source_len); +} + +/** * Retrieve a potentially cached object * * \param url URL of object to retrieve @@ -1044,114 +1073,110 @@ llcache_object_retrieve_from_cache(nsurl *url, for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) { if ((newest == NULL || - obj->cache.req_time > newest->cache.req_time) && - nsurl_compare(obj->url, url, - NSURL_COMPLETE) == true) { + obj->cache.req_time > newest->cache.req_time) && + nsurl_compare(obj->url, url, + NSURL_COMPLETE) == true) { newest = obj; } } - if (newest != NULL && llcache_object_is_fresh(newest)) { - /* Found a suitable object, and it's still fresh, so use it */ - obj = newest; + if ((newest != NULL) && (llcache_object_is_fresh(newest))) { + /* Found a suitable object, and it's still fresh */ - LLCACHE_LOG(("Found fresh %p", obj)); + LLCACHE_LOG(("Found fresh %p", newest)); /* The client needs to catch up with the object's state. * This will occur the next time that llcache_poll is called. */ - /* the objects source data may be in the persiatant store */ - if ((obj->store_state == LLCACHE_STORE_DISC) && - (obj->source_data == NULL)) { - int remaining; - error = guit->llcache->retrieve(obj->url, - &remaining, - &obj->source_data, - &obj->source_len); - if (error != NSERROR_OK) { - /* persistant store failed, destroy - * cache object and re-retch - */ -#ifdef LLCACHE_TRACE - LOG(("Persistant retrival failed for %p", obj)); -#endif + /* ensure the source data is present */ + error = llcache_persist_retrieve(newest); + if (error == NSERROR_OK) { + /* source data was sucessfully retrived from + * persistant store + */ + *result = newest; - llcache_object_remove_from_list(obj, - &llcache->cached_objects); - llcache_object_destroy(obj); + return NSERROR_OK; + } - /* Create new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; + /* retrival of source data from persistant store + * failed, destroy cache object and fall though to + * cache miss to re-retch + */ + LLCACHE_LOG(("Persistant retrival failed for %p", newest)); - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, - referer, post, - redirect_count); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - return error; - } + llcache_object_remove_from_list(newest, &llcache->cached_objects); + llcache_object_destroy(newest); - /* Add new object to cache */ - llcache_object_add_to_list(obj, &llcache->cached_objects); - } - } } else if (newest != NULL) { /* Found a candidate object but it needs freshness validation */ - /* Create a new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; + /* ensure the source data is present */ + error = llcache_persist_retrieve(newest); + if (error == NSERROR_OK) { - LLCACHE_LOG(("Found candidate %p (%p)", obj, newest)); + /* Create a new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; - /* Clone candidate's cache data */ - error = llcache_object_clone_cache_data(newest, obj, true); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - return error; - } + LLCACHE_LOG(("Found candidate %p (%p)", obj, newest)); + + /* Clone candidate's cache data */ + error = llcache_object_clone_cache_data(newest, obj, true); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } - /* Record candidate, so we can fall back if it is still fresh */ - newest->candidate_count++; - obj->candidate = newest; + /* Record candidate, so we can fall back if it is still fresh */ + newest->candidate_count++; + obj->candidate = newest; - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); - if (error != NSERROR_OK) { - newest->candidate_count--; - llcache_object_destroy(obj); - return error; - } + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post, + redirect_count); + if (error != NSERROR_OK) { + newest->candidate_count--; + llcache_object_destroy(obj); + return error; + } - /* Add new object to cache */ - llcache_object_add_to_list(obj, &llcache->cached_objects); - } else { - /* No object found; create a new one */ - /* Create new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache->cached_objects); - LLCACHE_LOG(("Not found %p", obj)); + *result = obj; - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - return error; + return NSERROR_OK; } - /* Add new object to cache */ - llcache_object_add_to_list(obj, &llcache->cached_objects); + LLCACHE_LOG(("Persistant retrival failed for %p", newest)); + + llcache_object_remove_from_list(newest, + &llcache->cached_objects); + llcache_object_destroy(newest); + } + + /* No viable object found in cache; create a new object */ + + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + LLCACHE_LOG(("Not found %p", obj)); + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post, + redirect_count); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; } + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache->cached_objects); + *result = obj; return NSERROR_OK; @@ -2386,12 +2411,11 @@ void llcache_clean(void) (object->fetch.fetch == NULL) && (object->fetch.outstanding_query == false) && (object->store_state == LLCACHE_STORE_RAM)) { -#ifdef LLCACHE_TRACE - LOG(("destroying cacahable object:%p len:%d age:%d", + LLCACHE_LOG(("destroying cacahable object:%p len:%d age:%d", object, object->source_len, time(NULL) - object->last_used)); -#endif + llcache_size -= object->source_len + sizeof(*object); llcache_object_remove_from_list(object, commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=ebc6d557af378269541... commit ebc6d557af3782695417e1d2bf10b0c9faf5fb55 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; persistant store now used and deals with faliure diff --git a/content/llcache.c b/content/llcache.c index 1b0cb13..279b544 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -38,7 +38,8 @@ #include "content/urldb.h" /** Define to enable tracing of llcache operations. */ -#undef LLCACHE_TRACE +//#undef LLCACHE_TRACE +#define LLCACHE_TRACE 1 #ifdef LLCACHE_TRACE #define LLCACHE_LOG(x) LOG(x) @@ -108,6 +109,7 @@ typedef enum { typedef struct { time_t req_time; /**< Time of request */ time_t res_time; /**< Time of response */ + time_t fin_time; /**< Time of request completion */ time_t date; /**< Date: response header */ time_t expires; /**< Expires: response header */ #define INVALID_AGE -1 @@ -145,7 +147,7 @@ struct llcache_object { size_t source_len; /**< Byte length of source data */ size_t source_alloc; /**< Allocated size of source buffer */ - llcache_store_state store_state; + llcache_store_state store_state; /**< where the data for the object is stored */ llcache_object_user *users; /**< List of users */ @@ -161,6 +163,13 @@ struct llcache_object { llcache_header *headers; /**< Fetch headers */ size_t num_headers; /**< Number of fetch headers */ + + /* Instrumentation. These elemnts are strictly for information + * to improve the cache performance and to provide performace + * metrics. The values are non-authorative and must not be used to + * determine object lifetime etc. + */ + time_t last_used; /**< time the last user was removed from the object */ }; struct llcache_s { @@ -275,6 +284,11 @@ static nserror llcache_object_remove_user(llcache_object *object, user->next = user->prev = NULL; + /* record the time the last user was removed from the object */ + if (object->users == NULL) { + object->last_used = time(NULL); + } + LLCACHE_LOG(("Removing user %p from %p", user, object)); return NSERROR_OK; @@ -710,6 +724,7 @@ static nserror llcache_object_refetch(llcache_object *object) /* Reset cache control data */ llcache_invalidate_cache_control_data(object); object->cache.req_time = time(NULL); + object->cache.fin_time = object->cache.req_time; /* Reset fetch state */ object->fetch.state = LLCACHE_FETCH_INIT; @@ -956,6 +971,7 @@ static nserror llcache_object_clone_cache_data(llcache_object *source, destination->cache.req_time = source->cache.req_time; destination->cache.res_time = source->cache.res_time; + destination->cache.fin_time = source->cache.fin_time; if (source->cache.date != 0) destination->cache.date = source->cache.date; @@ -979,6 +995,27 @@ static nserror llcache_object_clone_cache_data(llcache_object *source, } /** + * Remove a low-level cache object from a cache list + * + * \param object Object to remove + * \param list List to remove from + * \return NSERROR_OK + */ +static nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list) +{ + if (object == *list) + *list = object->next; + else + object->prev->next = object->next; + + if (object->next != NULL) + object->next->prev = object->prev; + + return NSERROR_OK; +} + +/** * Retrieve a potentially cached object * * \param url URL of object to retrieve @@ -989,9 +1026,13 @@ static nserror llcache_object_clone_cache_data(llcache_object *source, * \param result Pointer to location to recieve retrieved object * \return NSERROR_OK on success, appropriate error otherwise */ -static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count, llcache_object **result) +static nserror +llcache_object_retrieve_from_cache(nsurl *url, + uint32_t flags, + nsurl *referer, + const llcache_post_data *post, + uint32_t redirect_count, + llcache_object **result) { nserror error; llcache_object *obj, *newest = NULL; @@ -1019,6 +1060,45 @@ static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, /* The client needs to catch up with the object's state. * This will occur the next time that llcache_poll is called. */ + + /* the objects source data may be in the persiatant store */ + if ((obj->store_state == LLCACHE_STORE_DISC) && + (obj->source_data == NULL)) { + int remaining; + error = guit->llcache->retrieve(obj->url, + &remaining, + &obj->source_data, + &obj->source_len); + if (error != NSERROR_OK) { + /* persistant store failed, destroy + * cache object and re-retch + */ +#ifdef LLCACHE_TRACE + LOG(("Persistant retrival failed for %p", obj)); +#endif + + llcache_object_remove_from_list(obj, + &llcache->cached_objects); + llcache_object_destroy(obj); + + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, + referer, post, + redirect_count); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache->cached_objects); + } + } } else if (newest != NULL) { /* Found a candidate object but it needs freshness validation */ @@ -1617,9 +1697,9 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) #define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) /** - * possibly push objects to disc cache. + * possibly push objects data to persiatant storage. */ -static void llcache_discwrite(void *p) +static void llcache_persist(void *p) { llcache_object *object, *next; int remaining_lifetime; @@ -1631,7 +1711,7 @@ static void llcache_discwrite(void *p) remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache); - /* cachable objects with no pending fetches, not + /* cacehable objects with no pending fetches, not * already on disc and with sufficient lifetime to * make disc cache worthwile */ @@ -1663,7 +1743,7 @@ static void llcache_discwrite(void *p) if (object != NULL) { /* schedule completion of cache write */ - schedule(100, llcache_discwrite, NULL); + schedule(100, llcache_persist, NULL); } } @@ -1767,7 +1847,10 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p) llcache_object_cache_update(object); - schedule(500, llcache_discwrite, NULL); + /* record when the fetch finished */ + object->cache.fin_time = time(NULL); + + schedule(500, llcache_persist, NULL); } break; @@ -1877,26 +1960,6 @@ static llcache_object_user *llcache_object_find_user(const llcache_handle *handl return user; } -/** - * Remove a low-level cache object from a cache list - * - * \param object Object to remove - * \param list List to remove from - * \return NSERROR_OK - */ -static nserror llcache_object_remove_from_list(llcache_object *object, - llcache_object **list) -{ - if (object == *list) - *list = object->next; - else - object->prev->next = object->next; - - if (object->next != NULL) - object->next->prev = object->prev; - - return NSERROR_OK; -} /** * Determine if a low-level cache object resides in a given list @@ -2224,15 +2287,10 @@ void llcache_clean(void) LLCACHE_LOG(("Attempting cache clean")); - /* Candidates for cleaning are (in order of priority): - * - * 1) Uncacheable objects with no users - * 2) Stale cacheable objects with no users or pending fetches - * 3) Fresh cacheable objects with no users or pending fetches - */ - - /* 1) Uncacheable objects with no users or fetches */ - for (object = llcache->uncached_objects; object != NULL; object = next) { + /* Uncacheable objects with no users or fetches */ + for (object = llcache->uncached_objects; + object != NULL; + object = next) { next = object->next; /* The candidate count of uncacheable objects is always 0 */ @@ -2250,8 +2308,11 @@ void llcache_clean(void) } } - /* 2) Stale cacheable objects with no users or pending fetches */ - for (object = llcache->cached_objects; object != NULL; object = next) { + + /* Stale cacheable objects with no users or pending fetches */ + for (object = llcache->cached_objects; + object != NULL; + object = next) { next = object->next; remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache); @@ -2263,7 +2324,10 @@ void llcache_clean(void) if (remaining_lifetime > 0) { /* object is fresh */ - llcache_size += object->source_len + sizeof(*object); + llcache_size += sizeof(*object); + if (object->source_data != NULL) { + llcache_size += object->source_len; + } } else { /* object is not fresh */ LLCACHE_LOG(("Found stale cacheable object (%p) with no users or pending fetches", object)); @@ -2273,31 +2337,66 @@ void llcache_clean(void) llcache_object_destroy(object); } } else { + /* object has users */ llcache_size += object->source_len + sizeof(*object); } } - /* 3) Fresh cacheable objects with no users or pending - * fetches, only if the cache exceeds the configured size. + /* if the cache limit is exceeded try to make some objects + * persistant so their RAM can be reclaimed in the next + * step */ + if (llcache->limit < llcache_size) { + llcache_persist(NULL); + } + + /* Fresh cacheable objects with no users, no pending fetches + * and pushed to persistant store while the cache exceeds + * the configured size. */ - if (llcache->limit < llcache_size) { - for (object = llcache->cached_objects; object != NULL; - object = next) { - next = object->next; + for (object = llcache->cached_objects; + ((llcache->limit < llcache_size) && (object != NULL)); + object = next) { + next = object->next; + if ((object->users == NULL) && + (object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false) && + (object->store_state == LLCACHE_STORE_DISC)) { + free(object->source_data); + object->source_data = NULL; - if ((object->users == NULL) && - (object->candidate_count == 0) && - (object->fetch.fetch == NULL) && - (object->fetch.outstanding_query == false)) { - LLCACHE_LOG(("Found victim %p", object)); + llcache_size -= object->source_len; - llcache_size -= - object->source_len + sizeof(*object); + LLCACHE_LOG(("Freeing source data for %p len:%d", + object, + object->source_len)); + } + } - llcache_object_remove_from_list(object, + /* Fresh cacheable objects with no users or pending + * fetches while the cache exceeds the configured size. + */ + for (object = llcache->cached_objects; + ((llcache->limit < llcache_size) && (object != NULL)); + object = next) { + next = object->next; + + if ((object->users == NULL) && + (object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false) && + (object->store_state == LLCACHE_STORE_RAM)) { +#ifdef LLCACHE_TRACE + LOG(("destroying cacahable object:%p len:%d age:%d", + object, + object->source_len, + time(NULL) - object->last_used)); +#endif + llcache_size -= object->source_len + sizeof(*object); + + llcache_object_remove_from_list(object, &llcache->cached_objects); - llcache_object_destroy(object); - } + llcache_object_destroy(object); } } diff --git a/content/llcache.h b/content/llcache.h index 3d8232c..1faba81 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -131,6 +131,12 @@ typedef struct { } data; } llcache_query; +/** operation table */ +struct gui_llcache_table { + nserror (*persist)(struct nsurl *url, int remaining, const uint8_t *data, const size_t datalen); + nserror (*retrieve)(struct nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out); +}; + /** * Response handler for fetch-related queries * diff --git a/desktop/gui.h b/desktop/gui.h index bcf4963..1a84bd4 100644 --- a/desktop/gui.h +++ b/desktop/gui.h @@ -467,10 +467,8 @@ struct gui_browser_table { }; -struct gui_llcache_table { - nserror (*persist)(struct nsurl *url, int remaining, const uint8_t *data, size_t datalen); - nserror (*retrieve)(struct nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out); -}; + +struct gui_llcache_table; /** Graphical user interface function table * diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index f8be89a..e7d2994 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -343,12 +343,12 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } -static nserror gui_default_persist(nsurl *url, int remaining, const uint8_t *data, size_t datalen) +static nserror gui_default_persist(nsurl *url, int remaining, const uint8_t *data, const size_t datalen) { return NSERROR_SAVE_FAILED; } -static nserror gui_default_retrieve(nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out) +static nserror gui_default_retrieve(nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } diff --git a/gtk/llcache.c b/gtk/llcache.c index 56d6d12..ef00641 100644 --- a/gtk/llcache.c +++ b/gtk/llcache.c @@ -19,18 +19,19 @@ #include "utils/nsurl.h" #include "utils/log.h" #include "desktop/gui.h" +#include "content/llcache.h" #include "gtk/llcache.h" static nserror -persist(nsurl *url, int remaining, const uint8_t *data, size_t datalen) +persist(nsurl *url, int remaining, const uint8_t *data, const size_t datalen) { LOG(("Writing cache file for url:%s", nsurl_access(url))); return NSERROR_OK; } static nserror -retrieve(nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out) +retrieve(nsurl *url, int *remaining_out, uint8_t **data_out, size_t *datalen_out) { return NSERROR_NOT_FOUND; } commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=008b92a797ccf91314d... commit 008b92a797ccf91314d1eb1ba4a1fd0f71898d53 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; use llcache operation table and move stub to gtk diff --git a/content/llcache.c b/content/llcache.c index 9d88e1e..1b0cb13 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -23,17 +23,19 @@ #include <stdlib.h> #include <string.h> #include <time.h> - #include <curl/curl.h> -#include "content/fetch.h" -#include "content/llcache.h" -#include "content/urldb.h" #include "utils/corestrings.h" #include "utils/log.h" #include "utils/messages.h" #include "utils/nsurl.h" #include "utils/utils.h" +#include "utils/schedule.h" +#include "desktop/gui_factory.h" + +#include "content/fetch.h" +#include "content/llcache.h" +#include "content/urldb.h" /** Define to enable tracing of llcache operations. */ #undef LLCACHE_TRACE @@ -1614,15 +1616,6 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) #define LLCACHE_MIN_DISC_LIFETIME 3600 #define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) -static nserror write_llcache_file(nsurl *url, - int remaining, - const uint8_t *data, - size_t datalen) -{ - LLCACHE_LOG(("Writing cache file for url:%s", nsurl_access(url))); - return NSERROR_OK; -} - /** * possibly push objects to disc cache. */ @@ -1648,7 +1641,7 @@ static void llcache_discwrite(void *p) (object->store_state == LLCACHE_STORE_RAM) && (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { /* ok found an object to write */ - ret = write_llcache_file(object->url, + ret = guit->llcache->persist(object->url, remaining_lifetime, object->source_data, object->source_len); diff --git a/gtk/Makefile.target b/gtk/Makefile.target index ec19d1b..df9ac73 100644 --- a/gtk/Makefile.target +++ b/gtk/Makefile.target @@ -110,7 +110,7 @@ S_GTK := font_pango.c bitmap.c gui.c schedule.c thumbnail.c plotters.c \ treeview.c scaffolding.c gdk.c completion.c login.c throbber.c \ selection.c history.c window.c fetch.c download.c menu.c \ print.c search.c tabs.c theme.c toolbar.c gettext.c \ - compat.c cookies.c hotlist.c \ + compat.c cookies.c hotlist.c llcache.c \ $(addprefix dialogs/,preferences.c about.c source.c) S_GTK := $(addprefix gtk/,$(S_GTK)) $(addprefix utils/,container.c) diff --git a/gtk/gui.c b/gtk/gui.c index a3bf57c..1399fb9 100644 --- a/gtk/gui.c +++ b/gtk/gui.c @@ -69,6 +69,7 @@ #include "gtk/window.h" #include "gtk/schedule.h" #include "gtk/selection.h" +#include "gtk/llcache.h" #include "render/form.h" #include "utils/filepath.h" @@ -1014,6 +1015,7 @@ int main(int argc, char** argv) .clipboard = nsgtk_clipboard_table, .download = nsgtk_download_table, .fetch = nsgtk_fetch_table, + .llcache = nsgtk_llcache_table, }; /* check home directory is available */ diff --git a/gtk/llcache.c b/gtk/llcache.c new file mode 100644 index 0000000..56d6d12 --- /dev/null +++ b/gtk/llcache.c @@ -0,0 +1,43 @@ +/* + * Copyright 2014 Vincent Sanders <vince(a)netsurf-browser.org&gt; + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>;. + */ + +#include "utils/nsurl.h" +#include "utils/log.h" +#include "desktop/gui.h" + +#include "gtk/llcache.h" + +static nserror +persist(nsurl *url, int remaining, const uint8_t *data, size_t datalen) +{ + LOG(("Writing cache file for url:%s", nsurl_access(url))); + return NSERROR_OK; +} + +static nserror +retrieve(nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out) +{ + return NSERROR_NOT_FOUND; +} + +static struct gui_llcache_table llcache_table = { + .persist = persist, + .retrieve = retrieve, +}; + +struct gui_llcache_table *nsgtk_llcache_table = &llcache_table; diff --git a/gtk/llcache.h b/gtk/llcache.h new file mode 100644 index 0000000..6afdfe9 --- /dev/null +++ b/gtk/llcache.h @@ -0,0 +1,24 @@ +/* + * Copyright 2014 Vincent Sanders <vince(a)netsurf-browser.org&gt; + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>;. + */ + +#ifndef GTK_LLCACHE_H +#define GTK_LLCACHE_H + +extern struct gui_llcache_table* nsgtk_llcache_table; + +#endif commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=a1e5b4af318828bb644... commit a1e5b4af318828bb644df33b4af8360953bac648 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; add llcache operation table diff --git a/content/llcache.c b/content/llcache.c index 7728bbb..9d88e1e 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -1614,7 +1614,10 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) #define LLCACHE_MIN_DISC_LIFETIME 3600 #define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) -nserror write_llcache_file(nsurl *url, const uint8_t *data, size_t datalen) +static nserror write_llcache_file(nsurl *url, + int remaining, + const uint8_t *data, + size_t datalen) { LLCACHE_LOG(("Writing cache file for url:%s", nsurl_access(url))); return NSERROR_OK; @@ -1646,12 +1649,19 @@ static void llcache_discwrite(void *p) (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { /* ok found an object to write */ ret = write_llcache_file(object->url, + remaining_lifetime, object->source_data, object->source_len); - if (ret == NSERROR_OK) { - object->store_state = LLCACHE_STORE_DISC; - size_written += object->source_len; + if (ret != NSERROR_OK) { + /* as there has been a serialisation + * error, give up on making any more + * objects persistant for now. + */ + return; } + object->store_state = LLCACHE_STORE_DISC; + size_written += object->source_len; + if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { break; } diff --git a/desktop/gui.h b/desktop/gui.h index 85f1265..bcf4963 100644 --- a/desktop/gui.h +++ b/desktop/gui.h @@ -467,6 +467,10 @@ struct gui_browser_table { }; +struct gui_llcache_table { + nserror (*persist)(struct nsurl *url, int remaining, const uint8_t *data, size_t datalen); + nserror (*retrieve)(struct nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out); +}; /** Graphical user interface function table * @@ -500,6 +504,14 @@ struct gui_table { * implies the local encoding is utf8. */ struct gui_utf8_table *utf8; + + /** Low level cache table + * + * Used by the low level cache to push objects to persistant + * storage. The table is optional and may be NULL which + * disables persistant caching of objects + */ + struct gui_llcache_table *llcache; }; diff --git a/desktop/gui_factory.c b/desktop/gui_factory.c index 8996ab5..f8be89a 100644 --- a/desktop/gui_factory.c +++ b/desktop/gui_factory.c @@ -343,6 +343,40 @@ static nserror verify_utf8_register(struct gui_utf8_table *gut) return NSERROR_OK; } +static nserror gui_default_persist(nsurl *url, int remaining, const uint8_t *data, size_t datalen) +{ + return NSERROR_SAVE_FAILED; +} + +static nserror gui_default_retrieve(nsurl *url, int *remaining_out, const uint8_t **data_out, size_t *datalen_out) +{ + return NSERROR_NOT_FOUND; +} + +static struct gui_llcache_table default_llcache_table = { + .persist = gui_default_persist, + .retrieve = gui_default_retrieve, +}; + +/** verify clipboard table is valid */ +static nserror verify_llcache_register(struct gui_llcache_table *glt) +{ + /* check table is present */ + if (glt == NULL) { + return NSERROR_BAD_PARAMETER; + } + + /* mandantory operations */ + if (glt->persist == NULL) { + return NSERROR_BAD_PARAMETER; + } + if (glt->retrieve == NULL) { + return NSERROR_BAD_PARAMETER; + } + return NSERROR_OK; +} + + static nsurl *gui_default_get_resource_url(const char *path) { return NULL; @@ -529,6 +563,16 @@ nserror gui_factory_register(struct gui_table *gt) return err; } + /* llcache table */ + if (gt->llcache == NULL) { + /* set default clipboard table */ + gt->llcache = &default_llcache_table; + } + err = verify_llcache_register(gt->llcache); + if (err != NSERROR_OK) { + return err; + } + guit = gt; return NSERROR_OK; commitdiff http://git.netsurf-browser.org/netsurf.git/commit/?id=d9b171a919b41041ebb... commit d9b171a919b41041ebb4d792156091530d40a266 Author: Vincent Sanders <vince(a)netsurf-browser.org&gt; Commit: Vincent Sanders <vince(a)netsurf-browser.org&gt; initial low level cache persistant writeout diff --git a/content/llcache.c b/content/llcache.c index bd7ae93..7728bbb 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -122,32 +122,43 @@ typedef struct { char *value; /**< Header value */ } llcache_header; +/** Current status of objects data */ +typedef enum { + LLCACHE_STORE_RAM = 0, /**< sourec data is stored in RAM only */ + LLCACHE_STORE_MMAP, /**< source data is mmaped (implies on disc too) */ + LLCACHE_STORE_DISC, /**< source data is stored on disc */ +} llcache_store_state; + /** Low-level cache object */ /** \todo Consider whether a list is a sane container */ struct llcache_object { - llcache_object *prev; /**< Previous in list */ - llcache_object *next; /**< Next in list */ + llcache_object *prev; /**< Previous in list */ + llcache_object *next; /**< Next in list */ - nsurl *url; /**< Post-redirect URL for object */ - bool has_query; /**< URL has a query segment */ + nsurl *url; /**< Post-redirect URL for object */ + bool has_query; /**< URL has a query segment */ /** \todo We need a generic dynamic buffer object */ - uint8_t *source_data; /**< Source data for object */ - size_t source_len; /**< Byte length of source data */ - size_t source_alloc; /**< Allocated size of source buffer */ + uint8_t *source_data; /**< Source data for object */ + size_t source_len; /**< Byte length of source data */ + size_t source_alloc; /**< Allocated size of source buffer */ - llcache_object_user *users; /**< List of users */ + llcache_store_state store_state; - llcache_fetch_ctx fetch; /**< Fetch context for object */ + llcache_object_user *users; /**< List of users */ - llcache_cache_control cache; /**< Cache control data for object */ - llcache_object *candidate; /**< Object to use, if fetch determines - * that it is still fresh */ - uint32_t candidate_count; /**< Count of objects this is a - * candidate for */ + llcache_fetch_ctx fetch; /**< Fetch context for object */ - llcache_header *headers; /**< Fetch headers */ - size_t num_headers; /**< Number of fetch headers */ + llcache_cache_control cache; /**< Cache control data for object */ + llcache_object *candidate; /**< Object to use, if fetch determines + * that it is still fresh + */ + uint32_t candidate_count; /**< Count of objects this is a + * candidate for + */ + + llcache_header *headers; /**< Fetch headers */ + size_t num_headers; /**< Number of fetch headers */ }; struct llcache_s { @@ -1600,6 +1611,59 @@ static nserror llcache_fetch_ssl_error(llcache_object *object) return error; } +#define LLCACHE_MIN_DISC_LIFETIME 3600 +#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024) + +nserror write_llcache_file(nsurl *url, const uint8_t *data, size_t datalen) +{ + LLCACHE_LOG(("Writing cache file for url:%s", nsurl_access(url))); + return NSERROR_OK; +} + +/** + * possibly push objects to disc cache. + */ +static void llcache_discwrite(void *p) +{ + llcache_object *object, *next; + int remaining_lifetime; + nserror ret; + size_t size_written; + + for (object = llcache->cached_objects; object != NULL; object = next) { + next = object->next; + + remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache); + + /* cachable objects with no pending fetches, not + * already on disc and with sufficient lifetime to + * make disc cache worthwile + */ + if ((object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false) && + (object->store_state == LLCACHE_STORE_RAM) && + (remaining_lifetime > LLCACHE_MIN_DISC_LIFETIME)) { + /* ok found an object to write */ + ret = write_llcache_file(object->url, + object->source_data, + object->source_len); + if (ret == NSERROR_OK) { + object->store_state = LLCACHE_STORE_DISC; + size_written += object->source_len; + } + if (size_written > LLCACHE_MAX_DISC_BANDWIDTH) { + break; + } + } + } + + if (object != NULL) { + /* schedule completion of cache write */ + schedule(100, llcache_discwrite, NULL); + } +} + /** * Handler for fetch events * @@ -1699,6 +1763,8 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p) } llcache_object_cache_update(object); + + schedule(500, llcache_discwrite, NULL); } break; @@ -2139,7 +2205,6 @@ static nserror llcache_object_snapshot(llcache_object *object, return NSERROR_OK; } - /****************************************************************************** * Public API * ******************************************************************************/ diff --git a/utils/schedule.h b/utils/schedule.h index b5fe386..a75db85 100644 --- a/utils/schedule.h +++ b/utils/schedule.h @@ -26,6 +26,16 @@ /* In platform specific schedule.c. */ typedef void (*schedule_callback_fn)(void *p); +/** + * Schedule a callback. + * + * \param t interval before the callback should be made / cs + * \param callback callback function + * \param p user parameter, passed to callback function + * + * The callback function will be called as soon as possible after t cs have + * passed. + */ void schedule(int t, schedule_callback_fn callback, void *p); void schedule_remove(schedule_callback_fn callback, void *p); ----------------------------------------------------------------------- -- NetSurf Browser