Author: jmb Date: Sat Oct 10 07:03:47 2009 New Revision: 9619 URL: http://source.netsurf-browser.org?rev=9619&view=rev Log: Most of a low-level cache implementation. It compiles. Don't expect it to work yet -- there's a bunch of the fetch state machine it doesn't implement right now. Added: branches/jmb/new-cache/content/llcache.c Modified: branches/jmb/new-cache/Makefile.sources Modified: branches/jmb/new-cache/Makefile.sources URL: http://source.netsurf-browser.org/branches/jmb/new-cache/Makefile.sources... ============================================================================== --- branches/jmb/new-cache/Makefile.sources (original) +++ branches/jmb/new-cache/Makefile.sources Sat Oct 10 07:03:47 2009 @@ -5,7 +5,7 @@ # for each build. # -S_CONTENT := content.c fetch.c fetchcache.c urldb.c \ +S_CONTENT := content.c fetch.c fetchcache.c llcache.c urldb.c \ fetchers/fetch_curl.c fetchers/fetch_data.c S_CSS := css.c dump.c internal.c select.c utils.c S_RENDER := box.c box_construct.c box_normalise.c directory.c \ Added: branches/jmb/new-cache/content/llcache.c URL: http://source.netsurf-browser.org/branches/jmb/new-cache/content/llcache.... ============================================================================== --- branches/jmb/new-cache/content/llcache.c (added) +++ branches/jmb/new-cache/content/llcache.c Sat Oct 10 07:03:47 2009 @@ -1,0 +1,1119 @@ +/* + * Copyright 2009 John-Mark Bell <jmb(a)netsurf-browser.org&gt; + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>;. + */ + +/** \file + * Low-level resource cache (implementation) + */ + +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "content/fetch.h" +#include "content/llcache.h" +#include "utils/url.h" +#include "utils/utils.h" + +/** State of a low-level cache object fetch */ +typedef enum { + LLCACHE_FETCH_INIT, /**< Initial state, before fetch */ + LLCACHE_FETCH_HEADERS, /**< Fetching headers */ + LLCACHE_FETCH_DATA, /**< Fetching object data */ + LLCACHE_FETCH_COMPLETE /**< Fetch completed */ +} llcache_fetch_state; + +/** Handle to low-level cache object */ +struct llcache_handle { + llcache_object *object; /**< Pointer to associated object */ + + llcache_handle_callback cb; /**< Client callback */ + void *pw; /**< Client data */ + + llcache_fetch_state state; /**< Last known state of object fetch */ + size_t bytes; /**< Last reported byte count */ +}; + +/** Low-level cache object user record */ +typedef struct llcache_object_user { + /* Must be first in struct */ + llcache_handle handle; /**< Handle data for client */ + + struct llcache_object_user *prev; /**< Previous in list */ + struct llcache_object_user *next; /**< Next in list */ +} llcache_object_user; + +/** Low-level cache object fetch context */ +typedef struct { + uint32_t flags; /**< Fetch flags */ + char *referer; /**< Referring URL, or NULL if none */ + llcache_post_data *post; /**< POST data, or NULL for GET */ + + struct fetch *fetch; /**< Fetch handle for this object */ + + llcache_fetch_state state; /**< Current state of object fetch */ +} llcache_fetch_ctx; + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ +#define INVALID_AGE -1 + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + bool no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + +/** Low-level cache object */ +/** \todo Consider whether a list is a sane container */ +struct llcache_object { + llcache_object *prev; /**< Previous in list */ + llcache_object *next; /**< Next in list */ + + char *url; /**< Post-redirect URL for object */ + + /** \todo We need a generic dynamic buffer object */ + uint8_t *source_data; /**< Source data for object */ + size_t source_len; /**< Byte length of source data */ + size_t source_alloc; /**< Allocated size of source buffer */ + + llcache_object_user *users; /**< List of users */ + + llcache_fetch_ctx fetch; /**< Fetch context for object */ + + llcache_cache_control cache; /**< Cache control data for object */ + llcache_object *candidate; /**< Object to use, if fetch determines + * that it is still fresh */ + uint32_t candidate_count; /**< Count of objects this is a + * candidate for */ + + /** \todo Need fetch headers */ +}; + +/** Handler for fetch-related queries */ +static llcache_query_callback query_cb; +/** Data for fetch-related query handler */ +static void *query_cb_pw; + +/** Head of the low-level cached object list */ +static llcache_object *llcache_cached_objects; +/** Head of the low-level uncached object list */ +static llcache_object *llcache_uncached_objects; + +static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, + llcache_object_user **user); +static nserror llcache_object_user_destroy(llcache_object_user *user); + +static nserror llcache_object_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result); +static nserror llcache_object_retrieve_from_cache(const char *url, + uint32_t flags, const char *referer, + const llcache_post_data *post, llcache_object **result); +static bool llcache_object_is_fresh(const llcache_object *object); +static nserror llcache_object_cache_update(llcache_object *object); +static nserror llcache_object_clone_cache_data(const llcache_object *source, + llcache_object *destination, bool deep); +static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, + const char *referer, const llcache_post_data *post); + +static nserror llcache_object_new(const char *url, llcache_object **result); +static nserror llcache_object_destroy(llcache_object *object); +static nserror llcache_object_add_user(llcache_object *object, + llcache_object_user *user); +static nserror llcache_object_remove_user(llcache_object *object, + llcache_object_user *user); + +static nserror llcache_object_add_to_list(llcache_object *object, + llcache_object **list); +static nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list); + +static nserror llcache_object_notify_users(llcache_object *object); + +static nserror llcache_clean(void); + +static void llcache_fetch_callback(fetch_msg msg, void *p, const void *data, + unsigned long size); +static nserror llcache_fetch_redirect(llcache_object *object, + const char *target, llcache_object **replacement); +static nserror llcache_fetch_notmodified(llcache_object *object, + llcache_object **replacement); + +/****************************************************************************** + * Public API * + ******************************************************************************/ + +/** + * Initialise the low-level cache + * + * \param cb Query handler + * \param pw Pointer to query handler data + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_initialise(llcache_query_callback cb, void *pw) +{ + query_cb = cb; + query_cb_pw = pw; + + return NSERROR_OK; +} + +/** + * Poll the low-level cache + * + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_poll(void) +{ + llcache_object *object; + + /* Catch new users up with state of objects */ + for (object = llcache_cached_objects; object != NULL; + object = object->next) { + llcache_object_notify_users(object); + } + + for (object = llcache_uncached_objects; object != NULL; + object = object->next) { + llcache_object_notify_users(object); + } + + /* Attempt to clean the cache */ + llcache_clean(); + + return NSERROR_OK; +} + +/** + * Retrieve a handle for a low-level cache object + * + * \param url URL of the object to fetch + * \param flags Object retrieval flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param cb Client callback for events + * \param pw Pointer to client-specific data + * \param result Pointer to location to recieve cache handle + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_handle_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_handle_callback cb, void *pw, + llcache_handle **result) +{ + nserror error; + llcache_object_user *user; + llcache_object *object; + + /* Create a new object user */ + error = llcache_object_user_new(cb, pw, &user); + if (error != NSERROR_OK) + return error; + + /* Retrieve a suitable object from the cache, + * creating a new one if needed. */ + error = llcache_object_retrieve(url, flags, referer, post, &object); + if (error != NSERROR_OK) { + llcache_object_user_destroy(user); + return error; + } + + /* Add user to object */ + llcache_object_add_user(object, user); + + *result = &user->handle; + + return NSERROR_OK; +} + +/** + * Release a low-level cache handle + * + * \param handle Handle to release + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_handle_release(llcache_handle *handle) +{ + nserror error = NSERROR_OK; + llcache_object *object = handle->object; + llcache_object_user *user = (llcache_object_user *) handle; + + /* Remove the user from the object and destroy it */ + error = llcache_object_remove_user(object, user); + if (error == NSERROR_OK) + error = llcache_object_user_destroy(user); + + return error; +} + +/** + * Retrieve the low-level cache object associated with a handle + * + * \param handle Handle to dereference + * \return Pointer to low-level cache object + */ +const llcache_object *llcache_object_from_handle(const llcache_handle *handle) +{ + return handle->object; +} + +/** + * Retrieve the post-redirect URL of a low-level cache object + * + * \param object Object to retrieve URL from + * \return Post-redirect URL of cache object + */ +const char *llcache_object_get_url(const llcache_object *object) +{ + return object->url; +} + +/** + * Retrieve a header value associated with a low-level cache object + * + * \param object Object to retrieve header from + * \param key Header name + * \return Header value, or NULL if header does not exist + * + * \todo Make the key an enumeration, to avoid needless string comparisons + * \todo Forcing the client to parse the header value seems wrong. + * Better would be to return the actual value part and an array of + * key-value pairs for any additional parameters. + */ +const char *llcache_object_get_header(const llcache_object *object, + const char *key) +{ + /** \todo implement */ + return NULL; +} + +/****************************************************************************** + * Low-level cache internals * + ******************************************************************************/ + +/** + * Create a new object user + * + * \param cb Callback routine + * \param pw Private data for callback + * \param user Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, + llcache_object_user **user) +{ + llcache_object_user *u = calloc(1, sizeof(llcache_object_user)); + if (u == NULL) + return NSERROR_NOMEM; + + u->handle.cb = cb; + u->handle.pw = pw; + + *user = u; + + return NSERROR_OK; +} + +/** + * Destroy an object user + * + * \param user User to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre User is not attached to an object + */ +nserror llcache_object_user_destroy(llcache_object_user *user) +{ + free(user); + + return NSERROR_OK; +} + +/** + * Retrieve an object from the cache, fetching it if necessary. + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result) +{ + nserror error; + llcache_object *obj; + + /** + * Caching Rules: + * + * 1) Forced fetches are never cached + * 2) GET requests with query segments are never cached + * 3) POST requests are never cached + * + * \todo Find out if restriction (2) can be removed + * \todo Actually check for GET with a query part + */ + if (flags & LLCACHE_RETRIEVE_FORCE_FETCH || post != NULL) { + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to uncached list */ + llcache_object_add_to_list(obj, &llcache_uncached_objects); + } else { + error = llcache_object_retrieve_from_cache(url, flags, referer, + post, &obj); + if (error != NSERROR_OK) + return error; + + /* Returned object is already in the cached list */ + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Retrieve a potentially cached object + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_retrieve_from_cache(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result) +{ + nserror error; + llcache_object *obj, *newest = NULL; + + /* Search for the most recently fetched matching object */ + for (obj = llcache_cached_objects; obj != NULL; obj = obj->next) { + if (strcasecmp(obj->url, url) == 0 && (newest == NULL || + obj->cache.req_time > newest->cache.req_time)) + newest = obj; + } + + if (newest != NULL && llcache_object_is_fresh(newest)) { + /* Found a suitable object, and it's still fresh, so use it */ + obj = newest; + + /* The client needs to catch up with the object's state. + * This will occur the next time that llcache_poll is called. + */ + } else if (newest != NULL) { + /* Found a candidate object but it needs freshness validation */ + /* Create a new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Clone candidate's cache data */ + error = llcache_object_clone_cache_data(newest, obj, true); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Record candidate, so we can fall back if it is still fresh */ + newest->candidate_count++; + obj->candidate = newest; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + newest->candidate_count--; + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache_cached_objects); + } else { + /* No object found; create a new one */ + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache_cached_objects); + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Determine if an object is still fresh + * + * \param object Object to consider + * \return True if object is still fresh, false otherwise + */ +bool llcache_object_is_fresh(const llcache_object *object) +{ + const llcache_cache_control *cd = &object->cache; + int current_age, freshness_lifetime; + time_t now = time(NULL); + + /* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */ + current_age = max(0, (cd->res_time - cd->date)); + current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age); + current_age += cd->res_time - cd->req_time + now - cd->res_time; + + /* Determine freshness lifetime of this object */ + if (cd->max_age != INVALID_AGE) + freshness_lifetime = cd->max_age; + else if (cd->expires != 0) + freshness_lifetime = cd->expires - cd->date; + else if (cd->last_modified != 0) + freshness_lifetime = (now - cd->last_modified) / 10; + else + freshness_lifetime = 0; + + /* The object is fresh if its current age is within the freshness + * lifetime or if we're still fetching the object */ + return (freshness_lifetime > current_age || + object->fetch.state != LLCACHE_FETCH_COMPLETE); +} + +/** + * Update an object's cache state + * + * \param object Object to update cache for + * \return NSERROR_OK. + */ +nserror llcache_object_cache_update(llcache_object *object) +{ + if (object->cache.date == 0) + object->cache.date = time(NULL); + + /** \todo Any magic we need to do for no_cache? */ + + return NSERROR_OK; +} + +/** + * Clone an object's cache data + * + * \param source Source object containing cache data to clone + * \param destination Destination object to clone cache data into + * \param deep Whether to deep-copy the data or not + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_clone_cache_data(const llcache_object *source, + llcache_object *destination, bool deep) +{ + /* ETag must be first, as it can fail when deep cloning */ + if (source->cache.etag != NULL) { + char *etag = source->cache.etag; + + if (deep) { + /* Copy the etag */ + etag = strdup(source->cache.etag); + if (etag == NULL) + return NSERROR_NOMEM; + } + + if (destination->cache.etag != NULL) + free(destination->cache.etag); + + destination->cache.etag = etag; + } + + destination->cache.req_time = source->cache.req_time; + destination->cache.res_time = source->cache.res_time; + + if (source->cache.date != 0) + destination->cache.date = source->cache.date; + + if (source->cache.expires != 0) + destination->cache.expires = source->cache.expires; + + if (source->cache.age != INVALID_AGE) + destination->cache.age = source->cache.age; + + if (source->cache.max_age != INVALID_AGE) + destination->cache.max_age = source->cache.max_age; + + if (source->cache.no_cache) + destination->cache.no_cache = source->cache.no_cache; + + if (source->cache.last_modified != 0) + destination->cache.last_modified = source->cache.last_modified; + + return NSERROR_OK; +} + +/** + * Kick-off a fetch for an object + * + * \param object Object to fetch + * \param flags Fetch flags + * \param referer Referring URL, or NULL for none + * \param post POST data, or NULL for GET + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre object::url must contain the URL to fetch + * \pre If there is a freshness validation candidate, + * object::candidate and object::cache must be filled in + * \pre There must not be a fetch in progress for \a object + */ +nserror llcache_object_fetch(llcache_object *object, uint32_t flags, + const char *referer, const llcache_post_data *post) +{ + char *referer_clone; + llcache_post_data *post_clone; + const char *urlenc = NULL; + struct form_successful_control *multipart = NULL; + + referer_clone = strdup(referer); + if (referer_clone == NULL) + return NSERROR_NOMEM; + + /** \todo clone post */ + post_clone = (llcache_post_data *) post; + + object->fetch.flags = flags; + object->fetch.referer = referer_clone; + object->fetch.post = post_clone; + + if (object->fetch.post != NULL) { + if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) + urlenc = object->fetch.post->data.urlenc; + else + multipart = object->fetch.post->data.multipart; + } + + object->fetch.fetch = fetch_start(object->url, object->fetch.referer, + llcache_fetch_callback, object, + flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES, + urlenc, multipart, + flags & LLCACHE_RETRIEVE_VERIFIABLE, + NULL, /** \todo Remove parent from this API */ + NULL /** \todo Generate cache-control headers */); + if (object->fetch.fetch == NULL) + return NSERROR_NOMEM; + + return NSERROR_OK; +} + +/** + * Create a new low-level cache object + * + * \param url URL of object to create + * \param result Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_new(const char *url, llcache_object **result) +{ + llcache_object *obj = calloc(1, sizeof(llcache_object)); + if (obj == NULL) + return NSERROR_NOMEM; + + obj->url = strdup(url); + if (obj->url == NULL) { + free(obj); + return NSERROR_NOMEM; + } + + *result = obj; + + return NSERROR_NOMEM; +} + +/** + * Destroy a low-level cache object + * + * \param object Object to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre Object is detached from cache list + * \pre Object has no users + * \pre Object is not a candidate (i.e. object::candidate_count == 0) + */ +nserror llcache_object_destroy(llcache_object *object) +{ + free(object->url); + free(object->source_data); + + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + } + + free(object->fetch.referer); + /** \todo Destroy POST data */ + + free(object->cache.etag); + + /** \todo Headers */ + + free(object); + + return NSERROR_OK; +} + +/** + * Add a user to a low-level cache object + * + * \param object Object to add user to + * \param user User to add + * \return NSERROR_OK. + */ +nserror llcache_object_add_user(llcache_object *object, + llcache_object_user *user) +{ + user->handle.object = object; + + user->prev = NULL; + user->next = object->users; + + if (object->users != NULL) + object->users->prev = user; + object->users = user; + + return NSERROR_OK; +} + +/** + * Remove a user from a low-level cache object + * + * \param object Object to remove user from + * \param user User to remove + * \return NSERROR_OK. + */ +nserror llcache_object_remove_user(llcache_object *object, + llcache_object_user *user) +{ + if (user == object->users) + object->users = user->next; + else + user->prev->next = user->next; + + if (user->next != NULL) + user->next->prev = user->prev; + + return NSERROR_OK; +} + +/** + * Add a low-level cache object to a cache list + * + * \param object Object to add + * \param list List to add to + * \return NSERROR_OK + */ +nserror llcache_object_add_to_list(llcache_object *object, + llcache_object **list) +{ + object->prev = NULL; + object->next = *list; + + if (*list != NULL) + (*list)->prev = object; + *list = object; + + return NSERROR_OK; +} + +/** + * Remove a low-level cache object from a cache list + * + * \param object Object to remove + * \param list List to remove from + * \return NSERROR_OK + */ +nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list) +{ + if (object == *list) + *list = object->next; + else + object->prev->next = object->next; + + if (object->next != NULL) + object->next->prev = object->next; + + return NSERROR_OK; +} + +/** + * Notify users of an object's current state + * + * \param object Object to notify users about + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_notify_users(llcache_object *object) +{ + nserror error; + llcache_object_user *user; + llcache_event event; + + /** + * State transitions and event emission for users. + * Rows: user state. Cols: object state. + * + * User\Obj INIT HEADERS DATA COMPLETE + * INIT - T T* T* + * HEADERS - - T T* + * DATA - - M T + * COMPLETE - - - - + * + * T => transition user to object state + * M => no transition required, but may need to emit event + * + * The transitions marked with an asterisk can be removed by moving + * the user context into the subsequent state and then reevaluating. + * + * Events are issued as follows: + * + * HAD_HEADERS: on transition from HEADERS -> DATA state + * HAD_DATA : in DATA state, whenever there's new source data + * DONE : on transition from DATA -> COMPLETE state + */ + + /** \todo How are we going to handle errors here? */ + + for (user = object->users; user != NULL; user = user->next) { + /* Emit necessary events to bring the user up-to-date */ + llcache_handle *handle = &user->handle; + llcache_fetch_state hstate = handle->state; + llcache_fetch_state objstate = object->fetch.state; + + /* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */ + if (hstate == LLCACHE_FETCH_INIT && + objstate > LLCACHE_FETCH_INIT) { + hstate = LLCACHE_FETCH_HEADERS; + } + + /* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */ + if (hstate == LLCACHE_FETCH_HEADERS && + objstate > LLCACHE_FETCH_HEADERS) { + /* Emit HAD_HEADERS event */ + event.type = LLCACHE_EVENT_HAD_HEADERS; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) + return error; + + hstate = LLCACHE_FETCH_DATA; + } + + /* User: DATA, Obj: DATA, COMPLETE, more source available */ + if (hstate == LLCACHE_FETCH_DATA && + objstate >= LLCACHE_FETCH_DATA && + object->source_len > handle->bytes) { + /* Emit HAD_DATA event */ + event.type = LLCACHE_EVENT_HAD_DATA; + event.data.buf = object->source_data + handle->bytes; + event.data.len = object->source_len - handle->bytes; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) + return error; + + /* Update record of last byte emitted */ + handle->bytes = object->source_len; + } + + /* User: DATA, Obj: COMPLETE => User->COMPLETE */ + if (hstate == LLCACHE_FETCH_DATA && + objstate > LLCACHE_FETCH_DATA) { + /* Emit DONE event */ + event.type = LLCACHE_EVENT_DONE; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) + return error; + + hstate = LLCACHE_FETCH_COMPLETE; + } + + /* Sync handle's state with reality */ + handle->state = hstate; + } + + return NSERROR_OK; +} + +/** + * Attempt to clean the cache + * + * \return NSERROR_OK. + */ +nserror llcache_clean(void) +{ + llcache_object *object, *next; + + /* Candidates for cleaning are (in order of priority): + * + * 1) Uncacheable objects with no users + * 2) Stale cacheable objects with no users or pending fetches + * 3) Fresh cacheable objects with no users or pending fetches + */ + + /* 1) Uncacheable objects with no users */ + for (object = llcache_uncached_objects; object != NULL; object = next) { + next = object->next; + + /* The candidate count of uncacheable objects is always 0 */ + if (object->users == NULL && object->candidate_count == 0) { + llcache_object_remove_from_list(object, + &llcache_uncached_objects); + llcache_object_destroy(object); + } + } + + /* 2) Stale cacheable objects with no users or pending fetches */ + for (object = llcache_cached_objects; object != NULL; object = next) { + next = object->next; + + if (object->users == NULL && object->candidate_count == 0 && + llcache_object_is_fresh(object) == false) { + llcache_object_remove_from_list(object, + &llcache_cached_objects); + llcache_object_destroy(object); + } + } + + /* 3) Fresh cacheable objects with no users or pending fetches */ + /** \todo This one only happens if the cache is too large */ + + return NSERROR_OK; +} + +/** + * Handler for fetch events + * + * \param msg Type of fetch event + * \param p Our private data + * \param data Event data + * \param size Length of data in bytes + */ +void llcache_fetch_callback(fetch_msg msg, void *p, const void *data, + unsigned long size) +{ + nserror error; + llcache_object *object = p; + + switch (msg) { + /* 3xx responses */ + case FETCH_REDIRECT: + /* Request resulted in a redirect */ + llcache_fetch_redirect(object, data, &object); + break; + case FETCH_NOTMODIFIED: + /* Conditional request determined that cached object is fresh */ + llcache_fetch_notmodified(object, &object); + break; + + /** \todo Handle the rest of the FETCH_ events */ + + /* Normal 2xx state machine */ + /** \todo Merge FETCH_TYPE and FETCH_HEADER */ + case FETCH_TYPE: + /* Received MIME type for object */ + break; + case FETCH_HEADER: + /* Received a fetch header */ + object->fetch.state = LLCACHE_FETCH_HEADERS; + break; + case FETCH_DATA: + /* Received some data */ + object->fetch.state = LLCACHE_FETCH_DATA; + break; + case FETCH_FINISHED: + /* Finished fetching */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + break; + + /* Out-of-band progress information */ + case FETCH_PROGRESS: + /* Progress update */ + break; + + /* Events requiring action */ + case FETCH_ERROR: + /* An error occurred while fetching */ + break; + case FETCH_AUTH: + /* Need Authentication */ + break; + case FETCH_CERT_ERR: + /* Something went wrong when validating TLS certificates */ + break; + } + + /* Keep users in sync with reality */ + error = llcache_object_notify_users(object); + if (error != NSERROR_OK) { + /** \todo Error handling */ + } +} + +/** + * Handle FETCH_REDIRECT event + * + * \param object Object being redirected + * \param target Target of redirect (may be relative) + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_redirect(llcache_object *object, const char *target, + llcache_object **replacement) +{ + nserror error; + llcache_object *dest; + llcache_object_user *user, *next; + const llcache_post_data *post = object->fetch.post; + char *url, *absurl; + url_func_result result; + /* Extract HTTP response code from the fetch object */ + long http_code = fetch_http_code(object->fetch.fetch); + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /** \todo Limit redirect depth, or detect cycles */ + + /* Make target absolute */ + result = url_join(target, object->url, &absurl); + if (result != URL_FUNC_OK) { + /** \todo handle error */ + } + + /* Ensure target is normalised */ + result = url_normalize(absurl, &url); + + /* No longer require absolute url */ + free(absurl); + + if (result != URL_FUNC_OK) { + /** \todo handle error */ + } + + /** \todo Ensure that redirects to file:/// don't happen? */ + + /** \todo What happens if we've no way of handling this URL? */ + + /** \todo All the magical processing for the various redirect types */ + if (http_code == 301 || http_code == 302 || http_code == 303) { + /* 301, 302, 303 redirects are all unconditional GET requests */ + post = NULL; + } else { + /** \todo 300, 305, 307 */ + } + + /* Attempt to fetch target URL */ + error = llcache_object_retrieve(url, object->fetch.flags, + object->fetch.referer, object->fetch.post, + &dest); + + /* No longer require url */ + free(url); + + if (error != NSERROR_OK) { + /** \todo handle error */ + } + + /* Move user(s) to replacement object */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(dest, user); + } + + /* Dest is now our object */ + *replacement = dest; + + return NSERROR_OK; +} + +/** + * Handle FETCH_NOTMODIFIED event + * + * \param object Object to process + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK. + */ +nserror llcache_fetch_notmodified(llcache_object *object, + llcache_object **replacement) +{ + llcache_object_user *user, *next; + + /* Move user(s) to candidate content */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(object->candidate, user); + } + + /* Candidate is no longer a candidate for us */ + object->candidate->candidate_count--; + + /* Clone our cache control data into the candidate */ + llcache_object_clone_cache_data(object, object->candidate, false); + /* Bring candidate's cache data up to date */ + llcache_object_cache_update(object->candidate); + + /* Invalidate our cache-control data */ + memset(&object->cache, 0, sizeof(llcache_cache_control)); + + /* Ensure fetch has stopped */ + /** \todo Are there any other fields that need invalidating? */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Candidate is now our object */ + *replacement = object->candidate; + + /** \todo Ensure that old object gets flushed from the cache */ + + return NSERROR_OK; +}