WebSVN – Kolibri OS – Blame – /contrib/network/netsurf/netsurf/content/llcache.c

Rev	Author	Line No.	Line
3584	sourcerer	1	/*
		2	* Copyright 2009 John-Mark Bell
		3	*
		4	* This file is part of NetSurf, http://www.netsurf-browser.org/
		5	*
		6	* NetSurf is free software; you can redistribute it and/or modify
		7	* it under the terms of the GNU General Public License as published by
		8	* the Free Software Foundation; version 2 of the License.
		9	*
		10	* NetSurf is distributed in the hope that it will be useful,
		11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
		12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	* GNU General Public License for more details.
		14	*
		15	* You should have received a copy of the GNU General Public License
		16	* along with this program. If not, see .
		17	*/
		18
		19	/** \file
		20	* Low-level resource cache (implementation)
		21	*/
		22
		23	#include
		24	#include
		25	#include
		26
		27	#include
		28
		29	#include "content/fetch.h"
		30	#include "content/llcache.h"
		31	#include "content/urldb.h"
		32	#include "utils/log.h"
		33	#include "utils/messages.h"
		34	#include "utils/nsurl.h"
		35	#include "utils/utils.h"
		36
		37	/** Define to enable tracing of llcache operations. */
		38	#undef LLCACHE_TRACE
		39
		40	/** State of a low-level cache object fetch */
		41	typedef enum {
		42	LLCACHE_FETCH_INIT, /*< Initial state, before fetch /
		43	LLCACHE_FETCH_HEADERS, /*< Fetching headers /
		44	LLCACHE_FETCH_DATA, /*< Fetching object data /
		45	LLCACHE_FETCH_COMPLETE /*< Fetch completed /
		46	} llcache_fetch_state;
		47
		48	/** Type of low-level cache object */
		49	typedef struct llcache_object llcache_object;
		50
		51	/** Handle to low-level cache object */
		52	struct llcache_handle {
		53	llcache_object object; /< Pointer to associated object /
		54
		55	llcache_handle_callback cb; /*< Client callback /
		56	void pw; /< Client data /
		57
		58	llcache_fetch_state state; /*< Last known state of object fetch /
		59	size_t bytes; /*< Last reported byte count /
		60	};
		61
		62	/** Low-level cache object user record */
		63	typedef struct llcache_object_user {
		64	llcache_handle handle; /< Handle data for client /
		65
		66	bool iterator_target; /*< This is the an iterator target /
		67	bool queued_for_delete; /*< This user is queued for deletion /
		68
		69	struct llcache_object_user prev; /< Previous in list /
		70	struct llcache_object_user next; /< Next in list /
		71	} llcache_object_user;
		72
		73	/** Low-level cache object fetch context */
		74	typedef struct {
		75	uint32_t flags; /*< Fetch flags /
		76	nsurl referer; /< Referring URL, or NULL if none /
		77	llcache_post_data post; /< POST data, or NULL for GET /
		78
		79	struct fetch fetch; /< Fetch handle for this object /
		80
		81	llcache_fetch_state state; /*< Current state of object fetch /
		82
		83	uint32_t redirect_count; /*< Count of redirects followed /
		84
		85	bool tried_with_auth; /*< Whether we've tried with auth /
		86
		87	bool tried_with_tls_downgrade; /*< Whether we've tried TLS <= 1.0 /
		88
		89	bool outstanding_query; /*< Waiting for a query response /
		90	} llcache_fetch_ctx;
		91
		92	typedef enum {
		93	LLCACHE_VALIDATE_FRESH, /*< Only revalidate if not fresh /
		94	LLCACHE_VALIDATE_ALWAYS, /*< Always revalidate /
		95	LLCACHE_VALIDATE_ONCE /*< Revalidate once only /
		96	} llcache_validate;
		97
		98	/** Cache control data */
		99	typedef struct {
		100	time_t req_time; /*< Time of request /
		101	time_t res_time; /*< Time of response /
		102	time_t date; /*< Date: response header /
		103	time_t expires; /*< Expires: response header /
		104	#define INVALID_AGE -1
		105	int age; /*< Age: response header /
		106	int max_age; /*< Max-Age Cache-control parameter /
		107	llcache_validate no_cache; /*< No-Cache Cache-control parameter /
		108	char etag; /< Etag: response header /
		109	time_t last_modified; /*< Last-Modified: response header /
		110	} llcache_cache_control;
		111
		112	/** Representation of a fetch header */
		113	typedef struct {
		114	char name; /< Header name /
		115	char value; /< Header value /
		116	} llcache_header;
		117
		118	/** Low-level cache object */
		119	/** \todo Consider whether a list is a sane container */
		120	struct llcache_object {
		121	llcache_object prev; /< Previous in list /
		122	llcache_object next; /< Next in list /
		123
		124	nsurl url; /< Post-redirect URL for object /
		125	bool has_query; /*< URL has a query segment /
		126
		127	/** \todo We need a generic dynamic buffer object */
		128	uint8_t source_data; /< Source data for object /
		129	size_t source_len; /*< Byte length of source data /
		130	size_t source_alloc; /*< Allocated size of source buffer /
		131
		132	llcache_object_user users; /< List of users /
		133
		134	llcache_fetch_ctx fetch; /*< Fetch context for object /
		135
		136	llcache_cache_control cache; /*< Cache control data for object /
		137	llcache_object candidate; /*< Object to use, if fetch determines
		138	* that it is still fresh */
		139	uint32_t candidate_count; /**< Count of objects this is a
		140	* candidate for */
		141
		142	llcache_header headers; /< Fetch headers /
		143	size_t num_headers; /*< Number of fetch headers /
		144	};
		145
		146	struct llcache_s {
		147	/** Handler for fetch-related queries */
		148	llcache_query_callback query_cb;
		149
		150	/** Data for fetch-related query handler */
		151	void *query_cb_pw;
		152
		153	/** Head of the low-level cached object list */
		154	llcache_object *cached_objects;
		155
		156	/** Head of the low-level uncached object list */
		157	llcache_object *uncached_objects;
		158
		159	uint32_t limit;
		160	};
		161
		162	/** low level cache state */
		163	static struct llcache_s *llcache = NULL;
		164
		165	/* Static lwc_strings */
		166	static lwc_string *llcache_file_lwc;
		167	static lwc_string *llcache_about_lwc;
		168	static lwc_string *llcache_resource_lwc;
		169
		170	/* forward referenced callback function */
		171	static void llcache_fetch_callback(const fetch_msg msg, void p);
		172
		173
		174	/******************************************************************************
		175	* Low-level cache internals *
		176	******************************************************************************/
		177
		178	/**
		179	* Create a new object user
		180	*
		181	* \param cb Callback routine
		182	* \param pw Private data for callback
		183	* \param user Pointer to location to receive result
		184	* \return NSERROR_OK on success, appropriate error otherwise
		185	*/
		186	static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw,
		187	llcache_object_user **user)
		188	{
		189	llcache_handle *h;
		190	llcache_object_user *u;
		191
		192	h = calloc(1, sizeof(llcache_handle));
		193	if (h == NULL)
		194	return NSERROR_NOMEM;
		195
		196	u = calloc(1, sizeof(llcache_object_user));
		197	if (u == NULL) {
		198	free(h);
		199	return NSERROR_NOMEM;
		200	}
		201
		202	h->cb = cb;
		203	h->pw = pw;
		204
		205	u->handle = h;
		206
		207	#ifdef LLCACHE_TRACE
		208	LOG(("Created user %p (%p, %p, %p)", u, h, (void *) cb, pw));
		209	#endif
		210
		211	*user = u;
		212
		213	return NSERROR_OK;
		214	}
		215
		216	/**
		217	* Destroy an object user
		218	*
		219	* \param user User to destroy
		220	* \return NSERROR_OK on success, appropriate error otherwise
		221	*
		222	* \pre User is not attached to an object
		223	*/
		224	static nserror llcache_object_user_destroy(llcache_object_user *user)
		225	{
		226	#ifdef LLCACHE_TRACE
		227	LOG(("Destroyed user %p", user));
		228	#endif
		229
		230	assert(user->next == NULL);
		231	assert(user->prev == NULL);
		232
		233	if (user->handle != NULL)
		234	free(user->handle);
		235
		236	free(user);
		237
		238	return NSERROR_OK;
		239	}
		240
		241	/**
		242	* Remove a user from a low-level cache object
		243	*
		244	* \param object Object to remove user from
		245	* \param user User to remove
		246	* \return NSERROR_OK.
		247	*/
		248	static nserror llcache_object_remove_user(llcache_object *object,
		249	llcache_object_user *user)
		250	{
		251	assert(user != NULL);
		252	assert(object != NULL);
		253	assert(object->users != NULL);
		254	assert(user->handle == NULL \|\| user->handle->object == object);
		255	assert((user->prev != NULL) \|\| (object->users == user));
		256
		257	if (user == object->users)
		258	object->users = user->next;
		259	else
		260	user->prev->next = user->next;
		261
		262	if (user->next != NULL)
		263	user->next->prev = user->prev;
		264
		265	user->next = user->prev = NULL;
		266
		267	#ifdef LLCACHE_TRACE
		268	LOG(("Removing user %p from %p", user, object));
		269	#endif
		270
		271	return NSERROR_OK;
		272	}
		273
		274	/**
		275	* Iterate the users of an object, calling their callbacks.
		276	*
		277	* \param object The object to iterate
		278	* \param event The event to pass to the callback.
		279	* \return NSERROR_OK on success, appropriate error otherwise.
		280	*/
		281	static nserror llcache_send_event_to_users(llcache_object *object,
		282	llcache_event *event)
		283	{
		284	nserror error = NSERROR_OK;
		285	llcache_object_user user, next_user;
		286
		287	user = object->users;
		288	while (user != NULL) {
		289	user->iterator_target = true;
		290
		291	error = user->handle->cb(user->handle, event,
		292	user->handle->pw);
		293
		294	next_user = user->next;
		295
		296	user->iterator_target = false;
		297
		298	if (user->queued_for_delete) {
		299	llcache_object_remove_user(object, user);
		300	llcache_object_user_destroy(user);
		301	}
		302
		303	if (error != NSERROR_OK)
		304	break;
		305
		306	user = next_user;
		307	}
		308
		309	return error;
		310	}
		311
		312	/**
		313	* Create a new low-level cache object
		314	*
		315	* \param url URL of object to create
		316	* \param result Pointer to location to receive result
		317	* \return NSERROR_OK on success, appropriate error otherwise
		318	*/
		319	static nserror llcache_object_new(nsurl url, llcache_object *result)
		320	{
		321	llcache_object *obj = calloc(1, sizeof(llcache_object));
		322	if (obj == NULL)
		323	return NSERROR_NOMEM;
		324
		325	#ifdef LLCACHE_TRACE
		326	LOG(("Created object %p (%s)", obj, nsurl_access(url)));
		327	#endif
		328
		329	obj->url = nsurl_ref(url);
		330
		331	*result = obj;
		332
		333	return NSERROR_OK;
		334	}
		335
		336	/**
		337	* Clone a POST data object
		338	*
		339	* \param orig Object to clone
		340	* \param clone Pointer to location to receive clone
		341	* \return NSERROR_OK on success, appropriate error otherwise
		342	*/
		343	static nserror llcache_post_data_clone(const llcache_post_data *orig,
		344	llcache_post_data **clone)
		345	{
		346	llcache_post_data *post_clone;
		347
		348	post_clone = calloc(1, sizeof(llcache_post_data));
		349	if (post_clone == NULL)
		350	return NSERROR_NOMEM;
		351
		352	post_clone->type = orig->type;
		353
		354	/* Deep-copy the type-specific data */
		355	if (orig->type == LLCACHE_POST_URL_ENCODED) {
		356	post_clone->data.urlenc = strdup(orig->data.urlenc);
		357	if (post_clone->data.urlenc == NULL) {
		358	free(post_clone);
		359
		360	return NSERROR_NOMEM;
		361	}
		362	} else {
		363	post_clone->data.multipart = fetch_multipart_data_clone(
		364	orig->data.multipart);
		365	if (post_clone->data.multipart == NULL) {
		366	free(post_clone);
		367
		368	return NSERROR_NOMEM;
		369	}
		370	}
		371
		372	*clone = post_clone;
		373
		374	return NSERROR_OK;
		375	}
		376
		377	/**
		378	* Split a fetch header into name and value
		379	*
		380	* \param data Header string
		381	* \param len Byte length of header
		382	* \param name Pointer to location to receive header name
		383	* \param value Pointer to location to receive header value
		384	* \return NSERROR_OK on success, appropriate error otherwise
		385	*/
		386	static nserror llcache_fetch_split_header(const uint8_t *data, size_t len,
		387	char name, char value)
		388	{
		389	char n, v;
		390	const uint8_t *colon;
		391
		392	/* Find colon */
		393	colon = (const uint8_t ) strchr((const char ) data, ':');
		394	if (colon == NULL) {
		395	/* Failed, assume a key with no value */
		396	n = strdup((const char *) data);
		397	if (n == NULL)
		398	return NSERROR_NOMEM;
		399
		400	v = strdup("");
		401	if (v == NULL) {
		402	free(n);
		403	return NSERROR_NOMEM;
		404	}
		405	} else {
		406	/* Split header into name & value */
		407
		408	/* Strip leading whitespace from name */
		409	while (data[0] == ' ' \|\| data[0] == '\t' \|\|
		410	data[0] == '\r' \|\| data[0] == '\n') {
		411	data++;
		412	}
		413
		414	/* Strip trailing whitespace from name */
		415	while (colon > data && (colon[-1] == ' ' \|\|
		416	colon[-1] == '\t' \|\| colon[-1] == '\r' \|\|
		417	colon[-1] == '\n'))
		418	colon--;
		419
		420	n = strndup((const char *) data, colon - data);
		421	if (n == NULL)
		422	return NSERROR_NOMEM;
		423
		424	/* Find colon again */
		425	while (*colon != ':') {
		426	colon++;
		427	}
		428
		429	/* Skip over colon and any subsequent whitespace */
		430	do {
		431	colon++;
		432	} while (colon == ' ' \|\| colon == '\t' \|\|
		433	colon == '\r' \|\| colon == '\n');
		434
		435	/* Strip trailing whitespace from value */
		436	while (len > 0 && (data[len - 1] == ' ' \|\|
		437	data[len - 1] == '\t' \|\|
		438	data[len - 1] == '\r' \|\|
		439	data[len - 1] == '\n')) {
		440	len--;
		441	}
		442
		443	v = strndup((const char *) colon, len - (colon - data));
		444	if (v == NULL) {
		445	free(n);
		446	return NSERROR_NOMEM;
		447	}
		448	}
		449
		450	*name = n;
		451	*value = v;
		452
		453	return NSERROR_OK;
		454	}
		455
		456	/**
		457	* Parse a fetch header
		458	*
		459	* \param object Object to parse header for
		460	* \param data Header string
		461	* \param len Byte length of header
		462	* \param name Pointer to location to receive header name
		463	* \param value Pointer to location to receive header value
		464	* \return NSERROR_OK on success, appropriate error otherwise
		465	*
		466	* \note This function also has the side-effect of updating
		467	* the cache control data for the object if an interesting
		468	* header is encountered
		469	*/
		470	static nserror llcache_fetch_parse_header(llcache_object *object,
		471	const uint8_t data, size_t len, char name, char *value)
		472	{
		473	nserror error;
		474
		475	/* Set fetch response time if not already set */
		476	if (object->cache.res_time == 0)
		477	object->cache.res_time = time(NULL);
		478
		479	/* Decompose header into name-value pair */
		480	error = llcache_fetch_split_header(data, len, name, value);
		481	if (error != NSERROR_OK)
		482	return error;
		483
		484	/* Parse cache headers to populate cache control data */
		485	#define SKIP_ST(p) while (p != '\0' && (p == ' ' \|\| *p == '\t')) p++
		486
		487	if (5 < len && strcasecmp(*name, "Date") == 0) {
		488	/* extract Date header */
		489	object->cache.date = curl_getdate(*value, NULL);
		490	} else if (4 < len && strcasecmp(*name, "Age") == 0) {
		491	/* extract Age header */
		492	if ('0' <= value && value <= '9')
		493	object->cache.age = atoi(*value);
		494	} else if (8 < len && strcasecmp(*name, "Expires") == 0) {
		495	/* extract Expires header */
		496	object->cache.expires = curl_getdate(*value, NULL);
		497	} else if (14 < len && strcasecmp(*name, "Cache-Control") == 0) {
		498	/* extract and parse Cache-Control header */
		499	const char start = value;
		500	const char comma = value;
		501
		502	while (*comma != '\0') {
		503	while (comma != '\0' && comma != ',')
		504	comma++;
		505
		506	if (8 < comma - start && (strncasecmp(start,
		507	"no-cache", 8) == 0 \|\|
		508	strncasecmp(start, "no-store", 8) == 0))
		509	/* When we get a disk cache we should
		510	* distinguish between these two */
		511	object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS;
		512	else if (7 < comma - start &&
		513	strncasecmp(start, "max-age", 7) == 0) {
		514	/* Find '=' */
		515	while (start < comma && *start != '=')
		516	start++;
		517
		518	/* Skip over it */
		519	start++;
		520
		521	/* Skip whitespace */
		522	SKIP_ST(start);
		523
		524	if (start < comma)
		525	object->cache.max_age = atoi(start);
		526	}
		527
		528	if (*comma != '\0') {
		529	/* Skip past comma */
		530	comma++;
		531	/* Skip whitespace */
		532	SKIP_ST(comma);
		533	}
		534
		535	/* Set start for next token */
		536	start = comma;
		537	}
		538	} else if (5 < len && strcasecmp(*name, "ETag") == 0) {
		539	/* extract ETag header */
		540	free(object->cache.etag);
		541	object->cache.etag = strdup(*value);
		542	if (object->cache.etag == NULL)
		543	return NSERROR_NOMEM;
		544	} else if (14 < len && strcasecmp(*name, "Last-Modified") == 0) {
		545	/* extract Last-Modified header */
		546	object->cache.last_modified = curl_getdate(*value, NULL);
		547	}
		548
		549	#undef SKIP_ST
		550
		551	return NSERROR_OK;
		552	}
		553
		554	/* Destroy headers */
		555	static inline void llcache_destroy_headers(llcache_object *object)
		556	{
		557	while (object->num_headers > 0) {
		558	object->num_headers--;
		559
		560	free(object->headers[object->num_headers].name);
		561	free(object->headers[object->num_headers].value);
		562	}
		563	free(object->headers);
		564	object->headers = NULL;
		565	}
		566
		567	/* Invalidate cache control data */
		568	static inline void llcache_invalidate_cache_control_data(llcache_object *object)
		569	{
		570	free(object->cache.etag);
		571	memset(&(object->cache), 0, sizeof(llcache_cache_control));
		572
		573	object->cache.age = INVALID_AGE;
		574	object->cache.max_age = INVALID_AGE;
		575	}
		576
		577	/**
		578	* Process a fetch header
		579	*
		580	* \param object Object being fetched
		581	* \param data Header string
		582	* \param len Byte length of header
		583	* \return NSERROR_OK on success, appropriate error otherwise
		584	*/
		585	static nserror llcache_fetch_process_header(llcache_object *object,
		586	const uint8_t *data, size_t len)
		587	{
		588	nserror error;
		589	char name, value;
		590	llcache_header *temp;
		591
		592	/* The headers for multiple HTTP responses may be delivered to us if
		593	* the fetch layer receives a 401 response for which it has
		594	* authentication credentials. This will result in a silent re-request
		595	* after which we'll receive the actual response headers for the
		596	* object we want to fetch (assuming that the credentials were correct
		597	* of course)
		598	*
		599	* Therefore, if the header is an HTTP response start marker, then we
		600	* must discard any headers we've read so far, reset the cache data
		601	* that we might have computed, and start again.
		602	*/
		603	/** \todo Properly parse the response line */
		604	if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) {
		605	time_t req_time = object->cache.req_time;
		606
		607	llcache_invalidate_cache_control_data(object);
		608
		609	/* Restore request time, so we compute object's age correctly */
		610	object->cache.req_time = req_time;
		611
		612	llcache_destroy_headers(object);
		613	}
		614
		615	error = llcache_fetch_parse_header(object, data, len, &name, &value);
		616	if (error != NSERROR_OK)
		617	return error;
		618
		619	/* Append header data to the object's headers array */
		620	temp = realloc(object->headers, (object->num_headers + 1) *
		621	sizeof(llcache_header));
		622	if (temp == NULL) {
		623	free(name);
		624	free(value);
		625	return NSERROR_NOMEM;
		626	}
		627
		628	object->headers = temp;
		629
		630	object->headers[object->num_headers].name = name;
		631	object->headers[object->num_headers].value = value;
		632
		633	object->num_headers++;
		634
		635	return NSERROR_OK;
		636	}
		637
		638	/**
		639	* (Re)fetch an object
		640	*
		641	* \param object Object to refetch
		642	* \return NSERROR_OK on success, appropriate error otherwise
		643	*
		644	* \pre The fetch parameters in object->fetch must be populated
		645	*/
		646	static nserror llcache_object_refetch(llcache_object *object)
		647	{
		648	const char *urlenc = NULL;
		649	struct fetch_multipart_data *multipart = NULL;
		650	char **headers = NULL;
		651	int header_idx = 0;
		652
		653	if (object->fetch.post != NULL) {
		654	if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED)
		655	urlenc = object->fetch.post->data.urlenc;
		656	else
		657	multipart = object->fetch.post->data.multipart;
		658	}
		659
		660	/* Generate cache-control headers */
		661	headers = malloc(3 * sizeof(char *));
		662	if (headers == NULL)
		663	return NSERROR_NOMEM;
		664
		665	if (object->cache.etag != NULL) {
		666	const size_t len = SLEN("If-None-Match: ") +
		667	strlen(object->cache.etag) + 1;
		668
		669	headers[header_idx] = malloc(len);
		670	if (headers[header_idx] == NULL) {
		671	free(headers);
		672	return NSERROR_NOMEM;
		673	}
		674
		675	snprintf(headers[header_idx], len, "If-None-Match: %s",
		676	object->cache.etag);
		677
		678	header_idx++;
		679	}
		680	if (object->cache.date != 0) {
		681	/* Maximum length of an RFC 1123 date is 29 bytes */
		682	const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
		683
		684	headers[header_idx] = malloc(len);
		685	if (headers[header_idx] == NULL) {
		686	while (--header_idx >= 0)
		687	free(headers[header_idx]);
		688	free(headers);
		689	return NSERROR_NOMEM;
		690	}
		691
		692	snprintf(headers[header_idx], len, "If-Modified-Since: %s",
		693	rfc1123_date(object->cache.date));
		694
		695	header_idx++;
		696	}
		697	headers[header_idx] = NULL;
		698
		699	/* Reset cache control data */
		700	llcache_invalidate_cache_control_data(object);
		701	object->cache.req_time = time(NULL);
		702
		703	/* Reset fetch state */
		704	object->fetch.state = LLCACHE_FETCH_INIT;
		705
		706	#ifdef LLCACHE_TRACE
		707	LOG(("Refetching %p", object));
		708	#endif
		709
		710	/* Kick off fetch */
		711	object->fetch.fetch = fetch_start(object->url, object->fetch.referer,
		712	llcache_fetch_callback, object,
		713	object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES,
		714	urlenc, multipart,
		715	object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE,
		716	object->fetch.tried_with_tls_downgrade,
		717	(const char **) headers);
		718
		719	/* Clean up cache-control headers */
		720	while (--header_idx >= 0)
		721	free(headers[header_idx]);
		722	free(headers);
		723
		724	/* Did we succeed in creating a fetch? */
		725	if (object->fetch.fetch == NULL)
		726	return NSERROR_NOMEM;
		727
		728	return NSERROR_OK;
		729	}
		730
		731	/**
		732	* Kick-off a fetch for an object
		733	*
		734	* \param object Object to fetch
		735	* \param flags Fetch flags
		736	* \param referer Referring URL, or NULL for none
		737	* \param post POST data, or NULL for GET
		738	* \param redirect_count Number of redirects followed so far
		739	* \return NSERROR_OK on success, appropriate error otherwise
		740	*
		741	* \pre object::url must contain the URL to fetch
		742	* \pre If there is a freshness validation candidate,
		743	* object::candidate and object::cache must be filled in
		744	* \pre There must not be a fetch in progress for \a object
		745	*/
		746	static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
		747	nsurl referer, const llcache_post_data post,
		748	uint32_t redirect_count)
		749	{
		750	nserror error;
		751	nsurl *referer_clone = NULL;
		752	llcache_post_data *post_clone = NULL;
		753
		754	#ifdef LLCACHE_TRACE
		755	LOG(("Starting fetch for %p", object));
		756	#endif
		757
		758	if (post != NULL) {
		759	error = llcache_post_data_clone(post, &post_clone);
		760	if (error != NSERROR_OK)
		761	return error;
		762	}
		763
		764	if (referer != NULL)
		765	referer_clone = nsurl_ref(referer);
		766
		767	object->fetch.flags = flags;
		768	object->fetch.referer = referer_clone;
		769	object->fetch.post = post_clone;
		770	object->fetch.redirect_count = redirect_count;
		771
		772	return llcache_object_refetch(object);
		773	}
		774
		775	/**
		776	* Destroy a low-level cache object
		777	*
		778	* \param object Object to destroy
		779	* \return NSERROR_OK on success, appropriate error otherwise
		780	*
		781	* \pre Object is detached from cache list
		782	* \pre Object has no users
		783	* \pre Object is not a candidate (i.e. object::candidate_count == 0)
		784	*/
		785	static nserror llcache_object_destroy(llcache_object *object)
		786	{
		787	size_t i;
		788
		789	#ifdef LLCACHE_TRACE
		790	LOG(("Destroying object %p", object));
		791	#endif
		792
		793	nsurl_unref(object->url);
		794	free(object->source_data);
		795
		796	if (object->fetch.fetch != NULL) {
		797	fetch_abort(object->fetch.fetch);
		798	object->fetch.fetch = NULL;
		799	}
		800
		801	if (object->fetch.referer != NULL)
		802	nsurl_unref(object->fetch.referer);
		803
		804	if (object->fetch.post != NULL) {
		805	if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
		806	free(object->fetch.post->data.urlenc);
		807	} else {
		808	fetch_multipart_data_destroy(
		809	object->fetch.post->data.multipart);
		810	}
		811
		812	free(object->fetch.post);
		813	}
		814
		815	free(object->cache.etag);
		816
		817	for (i = 0; i < object->num_headers; i++) {
		818	free(object->headers[i].name);
		819	free(object->headers[i].value);
		820	}
		821	free(object->headers);
		822
		823	free(object);
		824
		825	return NSERROR_OK;
		826	}
		827
		828	/**
		829	* Add a low-level cache object to a cache list
		830	*
		831	* \param object Object to add
		832	* \param list List to add to
		833	* \return NSERROR_OK
		834	*/
		835	static nserror llcache_object_add_to_list(llcache_object *object,
		836	llcache_object **list)
		837	{
		838	object->prev = NULL;
		839	object->next = *list;
		840
		841	if (*list != NULL)
		842	(*list)->prev = object;
		843	*list = object;
		844
		845	return NSERROR_OK;
		846	}
		847
		848	/**
		849	* Determine the remaining lifetime of a cache object using the
		850	*
		851	* \param object Object to consider
		852	* \return True if object is still fresh, false otherwise
		853	*/
		854	static int
		855	llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
		856	{
		857	int current_age, freshness_lifetime;
		858	time_t now = time(NULL);
		859
		860	/* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */
		861	current_age = max(0, (cd->res_time - cd->date));
		862	current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age);
		863	current_age += cd->res_time - cd->req_time + now - cd->res_time;
		864
		865	/* Determine freshness lifetime of this object */
		866	if (cd->max_age != INVALID_AGE)
		867	freshness_lifetime = cd->max_age;
		868	else if (cd->expires != 0)
		869	freshness_lifetime = cd->expires - cd->date;
		870	else if (cd->last_modified != 0)
		871	freshness_lifetime = (now - cd->last_modified) / 10;
		872	else
		873	freshness_lifetime = 0;
		874
		875	#ifdef LLCACHE_TRACE
		876	LOG(("%d:%d", freshness_lifetime, current_age));
		877	#endif
		878
		879	if ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
		880	(freshness_lifetime > current_age)) {
		881	/* object was not forbidden from being returned from
		882	* the cache unvalidated (i.e. the response contained
		883	* a no-cache directive)
		884	*
		885	* The object current age is within the freshness lifetime.
		886	*/
		887	return freshness_lifetime - current_age;
		888	}
		889
		890	return 0; /* object has no remaining lifetime */
		891	}
		892
		893	/**
		894	* Determine if an object is still fresh
		895	*
		896	* \param object Object to consider
		897	* \return True if object is still fresh, false otherwise
		898	*/
		899	static bool llcache_object_is_fresh(const llcache_object *object)
		900	{
		901	int remaining_lifetime;
		902	const llcache_cache_control *cd = &object->cache;
		903
		904	remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(cd);
		905
		906	#ifdef LLCACHE_TRACE
		907	LOG(("%p: (%d > 0 \|\| %d != %d)", object,
		908	remaining_lifetime,
		909	object->fetch.state, LLCACHE_FETCH_COMPLETE));
		910	#endif
		911
		912	/* The object is fresh if:
		913	* - it was not forbidden from being returned from the cache
		914	* unvalidated.
		915	*
		916	* - it has remaining lifetime or still being fetched.
		917	*/
		918	return ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
		919	((remaining_lifetime > 0) \|\|
		920	(object->fetch.state != LLCACHE_FETCH_COMPLETE)));
		921	}
		922
		923	/**
		924	* Clone an object's cache data
		925	*
		926	* \param source Source object containing cache data to clone
		927	* \param destination Destination object to clone cache data into
		928	* \param deep Whether to deep-copy the data or not
		929	* \return NSERROR_OK on success, appropriate error otherwise
		930	*
		931	* \post If \a deep is false, then any pointers in \a source will be set to NULL
		932	*/
		933	static nserror llcache_object_clone_cache_data(llcache_object *source,
		934	llcache_object *destination, bool deep)
		935	{
		936	/* ETag must be first, as it can fail when deep cloning */
		937	if (source->cache.etag != NULL) {
		938	char *etag = source->cache.etag;
		939
		940	if (deep) {
		941	/* Copy the etag */
		942	etag = strdup(source->cache.etag);
		943	if (etag == NULL)
		944	return NSERROR_NOMEM;
		945	} else {
		946	/* Destination takes ownership */
		947	source->cache.etag = NULL;
		948	}
		949
		950	if (destination->cache.etag != NULL)
		951	free(destination->cache.etag);
		952
		953	destination->cache.etag = etag;
		954	}
		955
		956	destination->cache.req_time = source->cache.req_time;
		957	destination->cache.res_time = source->cache.res_time;
		958
		959	if (source->cache.date != 0)
		960	destination->cache.date = source->cache.date;
		961
		962	if (source->cache.expires != 0)
		963	destination->cache.expires = source->cache.expires;
		964
		965	if (source->cache.age != INVALID_AGE)
		966	destination->cache.age = source->cache.age;
		967
		968	if (source->cache.max_age != INVALID_AGE)
		969	destination->cache.max_age = source->cache.max_age;
		970
		971	if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH)
		972	destination->cache.no_cache = source->cache.no_cache;
		973
		974	if (source->cache.last_modified != 0)
		975	destination->cache.last_modified = source->cache.last_modified;
		976
		977	return NSERROR_OK;
		978	}
		979
		980	/**
		981	* Retrieve a potentially cached object
		982	*
		983	* \param url URL of object to retrieve
		984	* \param flags Fetch flags
		985	* \param referer Referring URL, or NULL if none
		986	* \param post POST data, or NULL for a GET request
		987	* \param redirect_count Number of redirects followed so far
		988	* \param result Pointer to location to recieve retrieved object
		989	* \return NSERROR_OK on success, appropriate error otherwise
		990	*/
		991	static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags,
		992	nsurl referer, const llcache_post_data post,
		993	uint32_t redirect_count, llcache_object **result)
		994	{
		995	nserror error;
		996	llcache_object obj, newest = NULL;
		997
		998	#ifdef LLCACHE_TRACE
		999	LOG(("Searching cache for %s (%x %s %p)", url, flags, referer, post));
		1000	#endif
		1001
		1002	/* Search for the most recently fetched matching object */
		1003	for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) {
		1004
		1005	if ((newest == NULL \|\|
		1006	obj->cache.req_time > newest->cache.req_time) &&
		1007	nsurl_compare(obj->url, url,
		1008	NSURL_COMPLETE) == true) {
		1009	newest = obj;
		1010	}
		1011	}
		1012
		1013	if (newest != NULL && llcache_object_is_fresh(newest)) {
		1014	/* Found a suitable object, and it's still fresh, so use it */
		1015	obj = newest;
		1016
		1017	#ifdef LLCACHE_TRACE
		1018	LOG(("Found fresh %p", obj));
		1019	#endif
		1020
		1021	/* The client needs to catch up with the object's state.
		1022	* This will occur the next time that llcache_poll is called.
		1023	*/
		1024	} else if (newest != NULL) {
		1025	/* Found a candidate object but it needs freshness validation */
		1026
		1027	/* Create a new object */
		1028	error = llcache_object_new(url, &obj);
		1029	if (error != NSERROR_OK)
		1030	return error;
		1031
		1032	#ifdef LLCACHE_TRACE
		1033	LOG(("Found candidate %p (%p)", obj, newest));
		1034	#endif
		1035
		1036	/* Clone candidate's cache data */
		1037	error = llcache_object_clone_cache_data(newest, obj, true);
		1038	if (error != NSERROR_OK) {
		1039	llcache_object_destroy(obj);
		1040	return error;
		1041	}
		1042
		1043	/* Record candidate, so we can fall back if it is still fresh */
		1044	newest->candidate_count++;
		1045	obj->candidate = newest;
		1046
		1047	/* Attempt to kick-off fetch */
		1048	error = llcache_object_fetch(obj, flags, referer, post,
		1049	redirect_count);
		1050	if (error != NSERROR_OK) {
		1051	newest->candidate_count--;
		1052	llcache_object_destroy(obj);
		1053	return error;
		1054	}
		1055
		1056	/* Add new object to cache */
		1057	llcache_object_add_to_list(obj, &llcache->cached_objects);
		1058	} else {
		1059	/* No object found; create a new one */
		1060	/* Create new object */
		1061	error = llcache_object_new(url, &obj);
		1062	if (error != NSERROR_OK)
		1063	return error;
		1064
		1065	#ifdef LLCACHE_TRACE
		1066	LOG(("Not found %p", obj));
		1067	#endif
		1068
		1069	/* Attempt to kick-off fetch */
		1070	error = llcache_object_fetch(obj, flags, referer, post,
		1071	redirect_count);
		1072	if (error != NSERROR_OK) {
		1073	llcache_object_destroy(obj);
		1074	return error;
		1075	}
		1076
		1077	/* Add new object to cache */
		1078	llcache_object_add_to_list(obj, &llcache->cached_objects);
		1079	}
		1080
		1081	*result = obj;
		1082
		1083	return NSERROR_OK;
		1084	}
		1085
		1086	/**
		1087	* Retrieve an object from the cache, fetching it if necessary.
		1088	*
		1089	* \param url URL of object to retrieve
		1090	* \param flags Fetch flags
		1091	* \param referer Referring URL, or NULL if none
		1092	* \param post POST data, or NULL for a GET request
		1093	* \param redirect_count Number of redirects followed so far
		1094	* \param result Pointer to location to recieve retrieved object
		1095	* \return NSERROR_OK on success, appropriate error otherwise
		1096	*/
		1097	static nserror llcache_object_retrieve(nsurl *url, uint32_t flags,
		1098	nsurl referer, const llcache_post_data post,
		1099	uint32_t redirect_count, llcache_object **result)
		1100	{
		1101	nserror error;
		1102	llcache_object *obj;
		1103	bool has_query;
		1104	nsurl *defragmented_url;
		1105
		1106	#ifdef LLCACHE_TRACE
		1107	LOG(("Retrieve %s (%x, %s, %p)", url, flags, referer, post));
		1108	#endif
		1109
		1110	/**
		1111	* Caching Rules:
		1112	*
		1113	* 1) Forced fetches are never cached
		1114	* 2) POST requests are never cached
		1115	*/
		1116
		1117	/* Look for a query segment */
		1118	has_query = nsurl_has_component(url, NSURL_QUERY);
		1119
		1120	/* Get rid of any url fragment */
		1121	if (nsurl_has_component(url, NSURL_FRAGMENT)) {
		1122	error = nsurl_defragment(url, &defragmented_url);
		1123	if (error != NSERROR_OK)
		1124	return error;
		1125	} else {
		1126	defragmented_url = nsurl_ref(url);
		1127	}
		1128
		1129	if (flags & LLCACHE_RETRIEVE_FORCE_FETCH \|\| post != NULL) {
		1130	/* Create new object */
		1131	error = llcache_object_new(defragmented_url, &obj);
		1132	if (error != NSERROR_OK) {
		1133	nsurl_unref(defragmented_url);
		1134	return error;
		1135	}
		1136
		1137	/* Attempt to kick-off fetch */
		1138	error = llcache_object_fetch(obj, flags, referer, post,
		1139	redirect_count);
		1140	if (error != NSERROR_OK) {
		1141	llcache_object_destroy(obj);
		1142	nsurl_unref(defragmented_url);
		1143	return error;
		1144	}
		1145
		1146	/* Add new object to uncached list */
		1147	llcache_object_add_to_list(obj, &llcache->uncached_objects);
		1148	} else {
		1149	error = llcache_object_retrieve_from_cache(defragmented_url,
		1150	flags, referer, post, redirect_count, &obj);
		1151	if (error != NSERROR_OK) {
		1152	nsurl_unref(defragmented_url);
		1153	return error;
		1154	}
		1155
		1156	/* Returned object is already in the cached list */
		1157	}
		1158
		1159	obj->has_query = has_query;
		1160
		1161	#ifdef LLCACHE_TRACE
		1162	LOG(("Retrieved %p", obj));
		1163	#endif
		1164
		1165	*result = obj;
		1166
		1167	nsurl_unref(defragmented_url);
		1168
		1169	return NSERROR_OK;
		1170	}
		1171
		1172	/**
		1173	* Add a user to a low-level cache object
		1174	*
		1175	* \param object Object to add user to
		1176	* \param user User to add
		1177	* \return NSERROR_OK.
		1178	*/
		1179	static nserror llcache_object_add_user(llcache_object *object,
		1180	llcache_object_user *user)
		1181	{
		1182	assert(user->next == NULL);
		1183	assert(user->prev == NULL);
		1184
		1185	user->handle->object = object;
		1186
		1187	user->prev = NULL;
		1188	user->next = object->users;
		1189
		1190	if (object->users != NULL)
		1191	object->users->prev = user;
		1192	object->users = user;
		1193
		1194	#ifdef LLCACHE_TRACE
		1195	LOG(("Adding user %p to %p", user, object));
		1196	#endif
		1197
		1198	return NSERROR_OK;
		1199	}
		1200
		1201	/**
		1202	* Handle FETCH_REDIRECT event
		1203	*
		1204	* \param object Object being redirected
		1205	* \param target Target of redirect (may be relative)
		1206	* \param replacement Pointer to location to receive replacement object
		1207	* \return NSERROR_OK on success, appropriate error otherwise
		1208	*/
		1209	static nserror llcache_fetch_redirect(llcache_object object, const char target,
		1210	llcache_object **replacement)
		1211	{
		1212	nserror error;
		1213	llcache_object *dest;
		1214	llcache_object_user user, next;
		1215	const llcache_post_data *post = object->fetch.post;
		1216	nsurl *url;
		1217	lwc_string *scheme;
		1218	lwc_string *object_scheme;
		1219	bool match;
		1220	/* Extract HTTP response code from the fetch object */
		1221	long http_code = fetch_http_code(object->fetch.fetch);
		1222
		1223	/* Abort fetch for this object */
		1224	fetch_abort(object->fetch.fetch);
		1225	object->fetch.fetch = NULL;
		1226
		1227	/* Invalidate the cache control data */
		1228	llcache_invalidate_cache_control_data(object);
		1229
		1230	/* And mark it complete */
		1231	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1232
		1233	/* Forcibly stop redirecting if we've followed too many redirects */
		1234	#define REDIRECT_LIMIT 10
		1235	if (object->fetch.redirect_count > REDIRECT_LIMIT) {
		1236	llcache_event event;
		1237
		1238	LOG(("Too many nested redirects"));
		1239
		1240	event.type = LLCACHE_EVENT_ERROR;
		1241	event.data.msg = messages_get("BadRedirect");
		1242
		1243	return llcache_send_event_to_users(object, &event);
		1244	}
		1245	#undef REDIRECT_LIMIT
		1246
		1247	/* Make target absolute */
		1248	error = nsurl_join(object->url, target, &url);
		1249	if (error != NSERROR_OK)
		1250	return error;
		1251
		1252	/* Reject attempts to redirect from unvalidated to validated schemes
		1253	* A "validated" scheme is one over which we have some guarantee that
		1254	* the source is trustworthy. */
		1255	object_scheme = nsurl_get_component(object->url, NSURL_SCHEME);
		1256	scheme = nsurl_get_component(url, NSURL_SCHEME);
		1257
		1258	/* resource: and about: are allowed to redirect anywhere */
		1259	if ((lwc_string_isequal(object_scheme, llcache_resource_lwc,
		1260	&match) == lwc_error_ok && match == false) &&
		1261	(lwc_string_isequal(object_scheme, llcache_about_lwc,
		1262	&match) == lwc_error_ok && match == false)) {
		1263	/* file, about and resource are not valid redirect targets */
		1264	if ((lwc_string_isequal(object_scheme, llcache_file_lwc,
		1265	&match) == lwc_error_ok && match == true) \|\|
		1266	(lwc_string_isequal(object_scheme, llcache_about_lwc,
		1267	&match) == lwc_error_ok && match == true) \|\|
		1268	(lwc_string_isequal(object_scheme, llcache_resource_lwc,
		1269	&match) == lwc_error_ok && match == true)) {
		1270	lwc_string_unref(object_scheme);
		1271	lwc_string_unref(scheme);
		1272	nsurl_unref(url);
		1273	return NSERROR_OK;
		1274	}
		1275	}
		1276
		1277	lwc_string_unref(scheme);
		1278	lwc_string_unref(object_scheme);
		1279
		1280	/* Bail out if we've no way of handling this URL */
		1281	if (fetch_can_fetch(url) == false) {
		1282	nsurl_unref(url);
		1283	return NSERROR_OK;
		1284	}
		1285
		1286	if (http_code == 301 \|\| http_code == 302 \|\| http_code == 303) {
		1287	/* 301, 302, 303 redirects are all unconditional GET requests */
		1288	post = NULL;
		1289	} else if (http_code != 307 \|\| post != NULL) {
		1290	/** \todo 300, 305, 307 with POST */
		1291	nsurl_unref(url);
		1292	return NSERROR_OK;
		1293	}
		1294
		1295	/* Attempt to fetch target URL */
		1296	error = llcache_object_retrieve(url, object->fetch.flags,
		1297	object->fetch.referer, post,
		1298	object->fetch.redirect_count + 1, &dest);
		1299
		1300	/* No longer require url */
		1301	nsurl_unref(url);
		1302
		1303	if (error != NSERROR_OK)
		1304	return error;
		1305
		1306	/* Move user(s) to replacement object */
		1307	for (user = object->users; user != NULL; user = next) {
		1308	next = user->next;
		1309
		1310	llcache_object_remove_user(object, user);
		1311	llcache_object_add_user(dest, user);
		1312	}
		1313
		1314	/* Dest is now our object */
		1315	*replacement = dest;
		1316
		1317	return NSERROR_OK;
		1318	}
		1319
		1320	/**
		1321	* Update an object's cache state
		1322	*
		1323	* \param object Object to update cache for
		1324	* \return NSERROR_OK.
		1325	*/
		1326	static nserror llcache_object_cache_update(llcache_object *object)
		1327	{
		1328	if (object->cache.date == 0)
		1329	object->cache.date = time(NULL);
		1330
		1331	return NSERROR_OK;
		1332	}
		1333
		1334	/**
		1335	* Handle FETCH_NOTMODIFIED event
		1336	*
		1337	* \param object Object to process
		1338	* \param replacement Pointer to location to receive replacement object
		1339	* \return NSERROR_OK.
		1340	*/
		1341	static nserror llcache_fetch_notmodified(llcache_object *object,
		1342	llcache_object **replacement)
		1343	{
		1344	/* There may be no candidate if the server erroneously responded
		1345	* to an unconditional request with a 304 Not Modified response.
		1346	* In this case, we simply retain the initial object, having
		1347	* invalidated it and marked it as complete.
		1348	*/
		1349	if (object->candidate != NULL) {
		1350	llcache_object_user user, next;
		1351
		1352	/* Move user(s) to candidate content */
		1353	for (user = object->users; user != NULL; user = next) {
		1354	next = user->next;
		1355
		1356	llcache_object_remove_user(object, user);
		1357	llcache_object_add_user(object->candidate, user);
		1358	}
		1359
		1360	/* Candidate is no longer a candidate for us */
		1361	object->candidate->candidate_count--;
		1362
		1363	/* Clone our cache control data into the candidate */
		1364	llcache_object_clone_cache_data(object, object->candidate,
		1365	false);
		1366	/* Bring candidate's cache data up to date */
		1367	llcache_object_cache_update(object->candidate);
		1368	/* Revert no-cache to normal, if required */
		1369	if (object->candidate->cache.no_cache ==
		1370	LLCACHE_VALIDATE_ONCE) {
		1371	object->candidate->cache.no_cache =
		1372	LLCACHE_VALIDATE_FRESH;
		1373	}
		1374
		1375	/* Candidate is now our object */
		1376	*replacement = object->candidate;
		1377	object->candidate = NULL;
		1378	} else {
		1379	/* There was no candidate: retain object */
		1380	*replacement = object;
		1381	}
		1382
		1383	/* Ensure fetch has stopped */
		1384	fetch_abort(object->fetch.fetch);
		1385	object->fetch.fetch = NULL;
		1386
		1387	/* Invalidate our cache-control data */
		1388	llcache_invalidate_cache_control_data(object);
		1389
		1390	/* Mark it complete */
		1391	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1392
		1393	/* Old object will be flushed from the cache on the next poll */
		1394
		1395	return NSERROR_OK;
		1396	}
		1397
		1398	/**
		1399	* Process a chunk of fetched data
		1400	*
		1401	* \param object Object being fetched
		1402	* \param data Data to process
		1403	* \param len Byte length of data
		1404	* \return NSERROR_OK on success, appropriate error otherwise.
		1405	*/
		1406	static nserror llcache_fetch_process_data(llcache_object object, const uint8_t data,
		1407	size_t len)
		1408	{
		1409	/* Resize source buffer if it's too small */
		1410	if (object->source_len + len >= object->source_alloc) {
		1411	const size_t new_len = object->source_len + len + 64 * 1024;
		1412	uint8_t *temp = realloc(object->source_data, new_len);
		1413	if (temp == NULL)
		1414	return NSERROR_NOMEM;
		1415
		1416	object->source_data = temp;
		1417	object->source_alloc = new_len;
		1418	}
		1419
		1420	/* Append this data chunk to source buffer */
		1421	memcpy(object->source_data + object->source_len, data, len);
		1422	object->source_len += len;
		1423
		1424	return NSERROR_OK;
		1425	}
		1426
		1427	/**
		1428	* Handle a query response
		1429	*
		1430	* \param proceed Whether to proceed with fetch
		1431	* \param cbpw Our context for query
		1432	* \return NSERROR_OK on success, appropriate error otherwise
		1433	*/
		1434	static nserror llcache_query_handle_response(bool proceed, void *cbpw)
		1435	{
		1436	llcache_event event;
		1437	llcache_object *object = cbpw;
		1438
		1439	object->fetch.outstanding_query = false;
		1440
		1441	/* Refetch, using existing fetch parameters, if client allows us to */
		1442	if (proceed)
		1443	return llcache_object_refetch(object);
		1444
		1445	/* Invalidate cache-control data */
		1446	llcache_invalidate_cache_control_data(object);
		1447
		1448	/* Mark it complete */
		1449	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1450
		1451	/* Inform client(s) that object fetch failed */
		1452	event.type = LLCACHE_EVENT_ERROR;
		1453	/** \todo More appropriate error message */
		1454	event.data.msg = messages_get("FetchFailed");
		1455
		1456	return llcache_send_event_to_users(object, &event);
		1457	}
		1458
		1459	/**
		1460	* Handle an authentication request
		1461	*
		1462	* \param object Object being fetched
		1463	* \param realm Authentication realm
		1464	* \return NSERROR_OK on success, appropriate error otherwise.
		1465	*/
		1466	static nserror llcache_fetch_auth(llcache_object object, const char realm)
		1467	{
		1468	const char *auth;
		1469	nserror error = NSERROR_OK;
		1470
		1471	/* Abort fetch for this object */
		1472	fetch_abort(object->fetch.fetch);
		1473	object->fetch.fetch = NULL;
		1474
		1475	/* Invalidate cache-control data */
		1476	llcache_invalidate_cache_control_data(object);
		1477
		1478	/* Destroy headers */
		1479	llcache_destroy_headers(object);
		1480
		1481	/* If there was no realm, then default to the URL */
		1482	/** \todo If there was no WWW-Authenticate header, use response body */
		1483	if (realm == NULL)
		1484	realm = nsurl_access(object->url);
		1485
		1486	auth = urldb_get_auth_details(object->url, realm);
		1487
		1488	if (auth == NULL \|\| object->fetch.tried_with_auth == true) {
		1489	/* No authentication details, or tried what we had, so ask */
		1490	object->fetch.tried_with_auth = false;
		1491
		1492	if (llcache->query_cb != NULL) {
		1493	llcache_query query;
		1494
		1495	/* Emit query for authentication details */
		1496	query.type = LLCACHE_QUERY_AUTH;
		1497	query.url = object->url;
		1498	query.data.auth.realm = realm;
		1499
		1500	object->fetch.outstanding_query = true;
		1501
		1502	error = llcache->query_cb(&query, llcache->query_cb_pw,
		1503	llcache_query_handle_response, object);
		1504	} else {
		1505	llcache_event event;
		1506
		1507	/* Mark object complete */
		1508	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1509
		1510	/* Inform client(s) that object fetch failed */
		1511	event.type = LLCACHE_EVENT_ERROR;
		1512	/** \todo More appropriate error message */
		1513	event.data.msg = messages_get("FetchFailed");
		1514
		1515	error = llcache_send_event_to_users(object, &event);
		1516	}
		1517	} else {
		1518	/* Flag that we've tried to refetch with credentials, so
		1519	* that if the fetch fails again, we ask the user again */
		1520	object->fetch.tried_with_auth = true;
		1521	error = llcache_object_refetch(object);
		1522	}
		1523
		1524	return error;
		1525	}
		1526
		1527	/**
		1528	* Handle a TLS certificate verification failure
		1529	*
		1530	* \param object Object being fetched
		1531	* \param certs Certificate chain
		1532	* \param num Number of certificates in chain
		1533	* \return NSERROR_OK on success, appropriate error otherwise
		1534	*/
		1535	static nserror llcache_fetch_cert_error(llcache_object *object,
		1536	const struct ssl_cert_info *certs, size_t num)
		1537	{
		1538	nserror error = NSERROR_OK;
		1539
		1540	/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
		1541	object->fetch.fetch = NULL;
		1542
		1543	/* Invalidate cache-control data */
		1544	llcache_invalidate_cache_control_data(object);
		1545
		1546	if (llcache->query_cb != NULL) {
		1547	llcache_query query;
		1548
		1549	/* Emit query for TLS */
		1550	query.type = LLCACHE_QUERY_SSL;
		1551	query.url = object->url;
		1552	query.data.ssl.certs = certs;
		1553	query.data.ssl.num = num;
		1554
		1555	object->fetch.outstanding_query = true;
		1556
		1557	error = llcache->query_cb(&query, llcache->query_cb_pw,
		1558	llcache_query_handle_response, object);
		1559	} else {
		1560	llcache_event event;
		1561
		1562	/* Mark object complete */
		1563	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1564
		1565	/* Inform client(s) that object fetch failed */
		1566	event.type = LLCACHE_EVENT_ERROR;
		1567	/** \todo More appropriate error message */
		1568	event.data.msg = messages_get("FetchFailed");
		1569
		1570	error = llcache_send_event_to_users(object, &event);
		1571	}
		1572
		1573	return error;
		1574	}
		1575
		1576	/**
		1577	* Handle a TLS connection setup failure
		1578	*
		1579	* \param object Object being fetched
		1580	* \return NSERROR_OK on success, appropriate error otherwise
		1581	*/
		1582	static nserror llcache_fetch_ssl_error(llcache_object *object)
		1583	{
		1584	nserror error = NSERROR_OK;
		1585
		1586	/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
		1587	object->fetch.fetch = NULL;
		1588
		1589	/* Invalidate cache-control data */
		1590	llcache_invalidate_cache_control_data(object);
		1591
		1592	if (object->fetch.tried_with_tls_downgrade == true) {
		1593	/* Have already tried to downgrade, so give up */
		1594	llcache_event event;
		1595
		1596	/* Mark object complete */
		1597	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1598
		1599	/* Inform client(s) that object fetch failed */
		1600	event.type = LLCACHE_EVENT_ERROR;
		1601	/** \todo More appropriate error message */
		1602	event.data.msg = messages_get("FetchFailed");
		1603
		1604	error = llcache_send_event_to_users(object, &event);
		1605	} else {
		1606	/* Flag that we've tried to downgrade, so that if the
		1607	* fetch fails again, we give up */
		1608	object->fetch.tried_with_tls_downgrade = true;
		1609	error = llcache_object_refetch(object);
		1610	}
		1611
		1612	return error;
		1613	}
		1614
		1615	/**
		1616	* Handler for fetch events
		1617	*
		1618	* \param msg Fetch event
		1619	* \param p Our private data
		1620	*/
		1621	static void llcache_fetch_callback(const fetch_msg msg, void p)
		1622	{
		1623	nserror error = NSERROR_OK;
		1624	llcache_object *object = p;
		1625	llcache_event event;
4821	ashmew2	1626	__menuet__debug_out("Inside llcache_fetch_callback\n");
3584	sourcerer	1627
		1628	#ifdef LLCACHE_TRACE
		1629	LOG(("Fetch event %d for %p", msg->type, object));
		1630	#endif
		1631
		1632	switch (msg->type) {
		1633	case FETCH_HEADER:
		1634	/* Received a fetch header */
		1635	object->fetch.state = LLCACHE_FETCH_HEADERS;
		1636
		1637	error = llcache_fetch_process_header(object,
		1638	msg->data.header_or_data.buf,
		1639	msg->data.header_or_data.len);
		1640	break;
		1641
		1642	/* 3xx responses */
		1643	case FETCH_REDIRECT:
		1644	/* Request resulted in a redirect */
		1645
		1646	/* Release candidate, if any */
		1647	if (object->candidate != NULL) {
		1648	object->candidate->candidate_count--;
		1649	object->candidate = NULL;
		1650	}
		1651
		1652	error = llcache_fetch_redirect(object,
		1653	msg->data.redirect, &object);
		1654	break;
		1655	case FETCH_NOTMODIFIED:
		1656	/* Conditional request determined that cached object is fresh */
		1657	error = llcache_fetch_notmodified(object, &object);
		1658	break;
		1659
		1660	/* Normal 2xx state machine */
		1661	case FETCH_DATA:
		1662	/* Received some data */
		1663	if (object->fetch.state != LLCACHE_FETCH_DATA) {
		1664	/* On entry into this state, check if we need to
		1665	* invalidate the cache control data. We are guaranteed
		1666	* to have received all response headers.
		1667	*
		1668	* There are two cases in which we want to suppress
		1669	* cacheing of an object:
		1670	*
		1671	* 1) The HTTP response code is not 200 or 203
		1672	* 2) The request URI had a query string and the
		1673	* response headers did not provide an explicit
		1674	* object expiration time.
		1675	*/
		1676	long http_code = fetch_http_code(object->fetch.fetch);
		1677
		1678	if ((http_code != 200 && http_code != 203) \|\|
		1679	(object->has_query &&
		1680	(object->cache.max_age == INVALID_AGE &&
		1681	object->cache.expires == 0))) {
		1682	/* Invalidate cache control data */
		1683	llcache_invalidate_cache_control_data(object);
		1684	}
		1685
		1686	/* Release candidate, if any */
		1687	if (object->candidate != NULL) {
		1688	object->candidate->candidate_count--;
		1689	object->candidate = NULL;
		1690	}
		1691	}
		1692
		1693	object->fetch.state = LLCACHE_FETCH_DATA;
		1694
		1695	error = llcache_fetch_process_data(object,
		1696	msg->data.header_or_data.buf,
		1697	msg->data.header_or_data.len);
		1698	break;
		1699	case FETCH_FINISHED:
		1700	/* Finished fetching */
		1701	{
		1702	uint8_t *temp;
		1703
		1704	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1705	object->fetch.fetch = NULL;
		1706
		1707	/* Shrink source buffer to required size */
		1708	temp = realloc(object->source_data,
		1709	object->source_len);
		1710	/* If source_len is 0, then temp may be NULL */
		1711	if (temp != NULL \|\| object->source_len == 0) {
		1712	object->source_data = temp;
		1713	object->source_alloc = object->source_len;
		1714	}
		1715
		1716	llcache_object_cache_update(object);
		1717	}
		1718	break;
		1719
		1720	/* Out-of-band information */
		1721	case FETCH_ERROR:
		1722	/* An error occurred while fetching */
		1723	/* The fetch has has already been cleaned up by the fetcher */
		1724	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1725	object->fetch.fetch = NULL;
		1726
		1727	/* Release candidate, if any */
		1728	if (object->candidate != NULL) {
		1729	object->candidate->candidate_count--;
		1730	object->candidate = NULL;
		1731	}
		1732
		1733	/* Invalidate cache control data */
		1734	llcache_invalidate_cache_control_data(object);
		1735
		1736	/** \todo Consider using errorcode for something */
		1737
		1738	event.type = LLCACHE_EVENT_ERROR;
		1739	event.data.msg = msg->data.error;
		1740
		1741	error = llcache_send_event_to_users(object, &event);
		1742
		1743	break;
		1744	case FETCH_PROGRESS:
		1745	/* Progress update */
		1746	event.type = LLCACHE_EVENT_PROGRESS;
		1747	event.data.msg = msg->data.progress;
		1748
		1749	error = llcache_send_event_to_users(object, &event);
		1750
		1751	break;
		1752
		1753	/* Events requiring action */
		1754	case FETCH_AUTH:
		1755	/* Need Authentication */
		1756
		1757	/* Release candidate, if any */
		1758	if (object->candidate != NULL) {
		1759	object->candidate->candidate_count--;
		1760	object->candidate = NULL;
		1761	}
		1762
		1763	error = llcache_fetch_auth(object, msg->data.auth.realm);
		1764	break;
		1765	case FETCH_CERT_ERR:
		1766	/* Something went wrong when validating TLS certificates */
		1767
		1768	/* Release candidate, if any */
		1769	if (object->candidate != NULL) {
		1770	object->candidate->candidate_count--;
		1771	object->candidate = NULL;
		1772	}
		1773
		1774	error = llcache_fetch_cert_error(object,
		1775	msg->data.cert_err.certs,
		1776	msg->data.cert_err.num_certs);
		1777	break;
		1778	case FETCH_SSL_ERR:
		1779	/* TLS connection setup failed */
		1780
		1781	/* Release candidate, if any */
		1782	if (object->candidate != NULL) {
		1783	object->candidate->candidate_count--;
		1784	object->candidate = NULL;
		1785	}
		1786
		1787	error = llcache_fetch_ssl_error(object);
		1788	break;
		1789	}
		1790
		1791	/* Deal with any errors reported by event handlers */
		1792	if (error != NSERROR_OK) {
		1793	if (object->fetch.fetch != NULL) {
		1794	fetch_abort(object->fetch.fetch);
		1795	object->fetch.fetch = NULL;
		1796
		1797	/* Invalidate cache control data */
		1798	llcache_invalidate_cache_control_data(object);
		1799
		1800	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		1801	}
4821	ashmew2	1802	__menuet__debug_out("Returning from llcache_fetch_callback. (err != NS_OK)\n");
3584	sourcerer	1803	return;
		1804	}
4821	ashmew2	1805	__menuet__debug_out("Returning from llcache_fetch_callback.(err = NS_OK)\n");
3584	sourcerer	1806	}
		1807
		1808	/**
		1809	* Find a user of a low-level cache object
		1810	*
		1811	* \param handle External cache handle to search for
		1812	* \return Pointer to corresponding user, or NULL if not found
		1813	*/
		1814	static llcache_object_user llcache_object_find_user(const llcache_handle handle)
		1815	{
		1816	llcache_object_user *user;
		1817
		1818	assert(handle->object != NULL);
		1819
		1820	for (user = handle->object->users; user != NULL; user = user->next) {
		1821	if (user->handle == handle)
		1822	break;
		1823	}
		1824
		1825	return user;
		1826	}
		1827
		1828	/**
		1829	* Remove a low-level cache object from a cache list
		1830	*
		1831	* \param object Object to remove
		1832	* \param list List to remove from
		1833	* \return NSERROR_OK
		1834	*/
		1835	static nserror llcache_object_remove_from_list(llcache_object *object,
		1836	llcache_object **list)
		1837	{
		1838	if (object == *list)
		1839	*list = object->next;
		1840	else
		1841	object->prev->next = object->next;
		1842
		1843	if (object->next != NULL)
		1844	object->next->prev = object->prev;
		1845
		1846	return NSERROR_OK;
		1847	}
		1848
		1849	/**
		1850	* Determine if a low-level cache object resides in a given list
		1851	*
		1852	* \param object Object to search for
		1853	* \param list List to search in
		1854	* \return True if object resides in list, false otherwise
		1855	*/
		1856	static bool llcache_object_in_list(const llcache_object *object,
		1857	const llcache_object *list)
		1858	{
		1859	while (list != NULL) {
		1860	if (list == object)
		1861	break;
		1862
		1863	list = list->next;
		1864	}
		1865
		1866	return list != NULL;
		1867	}
		1868
		1869	/**
		1870	* Notify users of an object's current state
		1871	*
		1872	* \param object Object to notify users about
		1873	* \return NSERROR_OK on success, appropriate error otherwise
		1874	*/
		1875	static nserror llcache_object_notify_users(llcache_object *object)
		1876	{
		1877	nserror error;
		1878	llcache_object_user user, next_user;
		1879	llcache_event event;
		1880
		1881	#ifdef LLCACHE_TRACE
		1882	bool emitted_notify = false;
		1883	#endif
		1884
		1885	/**
		1886	* State transitions and event emission for users.
		1887	* Rows: user state. Cols: object state.
		1888	*
		1889	* User\Obj INIT HEADERS DATA COMPLETE
		1890	* INIT - T T* T*
		1891	* HEADERS - - T T*
		1892	* DATA - - M T
		1893	* COMPLETE - - - -
		1894	*
		1895	* T => transition user to object state
		1896	* M => no transition required, but may need to emit event
		1897	*
		1898	* The transitions marked with an asterisk can be removed by moving
		1899	* the user context into the subsequent state and then reevaluating.
		1900	*
		1901	* Events are issued as follows:
		1902	*
		1903	* HAD_HEADERS: on transition from HEADERS -> DATA state
		1904	* HAD_DATA : in DATA state, whenever there's new source data
		1905	* DONE : on transition from DATA -> COMPLETE state
		1906	*/
		1907
		1908	for (user = object->users; user != NULL; user = next_user) {
		1909	/* Emit necessary events to bring the user up-to-date */
		1910	llcache_handle *handle = user->handle;
		1911	const llcache_fetch_state objstate = object->fetch.state;
		1912
		1913	/* Flag that this user is the current iteration target
		1914	* in case the client attempts to destroy it underneath us */
		1915	user->iterator_target = true;
		1916
		1917	/* A note on the computation of next_user:
		1918	*
		1919	* Within this loop, we may make a number of calls to
		1920	* client code. Our contract with clients is that they
		1921	* can do whatever they like from within their callback
		1922	* handlers. This is so that we limit the pain of
		1923	* reentrancy to this module alone.
		1924	*
		1925	* One of the things a client can do from within its
		1926	* callback handler is to remove users from this object's
		1927	* user list. In the common case, the user they attempt
		1928	* to remove is the current iteration target, and we
		1929	* already protect against that causing problems here.
		1930	* However, no such protection exists if the client
		1931	* attempts to remove other users from this object's
		1932	* user list.
		1933	*
		1934	* Therefore, we cannot compute next_user up-front
		1935	* and expect it to remain valid across calls to
		1936	* client code (as the identity of the next user
		1937	* in the list may change underneath us). Instead,
		1938	* we must compute next_user at the point where we
		1939	* are about to cause another iteration of this loop
		1940	* (i.e. at the very end, and also at the points where
		1941	* continue is used)
		1942	*/
		1943
		1944	#ifdef LLCACHE_TRACE
		1945	if (handle->state != objstate) {
		1946	if (emitted_notify == false) {
		1947	LOG(("Notifying users of %p", object));
		1948	emitted_notify = true;
		1949	}
		1950
		1951	LOG(("User %p state: %d Object state: %d",
		1952	user, handle->state, objstate));
		1953	}
		1954	#endif
		1955
		1956	/* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */
		1957	if (handle->state == LLCACHE_FETCH_INIT &&
		1958	objstate > LLCACHE_FETCH_INIT) {
		1959	handle->state = LLCACHE_FETCH_HEADERS;
		1960	}
		1961
		1962	/* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */
		1963	if (handle->state == LLCACHE_FETCH_HEADERS &&
		1964	objstate > LLCACHE_FETCH_HEADERS) {
		1965	handle->state = LLCACHE_FETCH_DATA;
		1966
		1967	/* Emit HAD_HEADERS event */
		1968	event.type = LLCACHE_EVENT_HAD_HEADERS;
		1969
		1970	error = handle->cb(handle, &event, handle->pw);
		1971
		1972	if (user->queued_for_delete) {
		1973	next_user = user->next;
		1974	llcache_object_remove_user(object, user);
		1975	llcache_object_user_destroy(user);
		1976
		1977	if (error != NSERROR_OK)
		1978	return error;
		1979
		1980	continue;
		1981	} else if (error == NSERROR_NEED_DATA) {
		1982	/* User requested replay */
		1983	handle->state = LLCACHE_FETCH_HEADERS;
		1984
		1985	/* Continue with the next user -- we'll
		1986	* reemit the event next time round */
		1987	user->iterator_target = false;
		1988	next_user = user->next;
		1989	continue;
		1990	} else if (error != NSERROR_OK) {
		1991	user->iterator_target = false;
		1992	return error;
		1993	}
		1994	}
		1995
		1996	/* User: DATA, Obj: DATA, COMPLETE, more source available */
		1997	if (handle->state == LLCACHE_FETCH_DATA &&
		1998	objstate >= LLCACHE_FETCH_DATA &&
		1999	object->source_len > handle->bytes) {
		2000	size_t orig_handle_read;
		2001
		2002	/* Construct HAD_DATA event */
		2003	event.type = LLCACHE_EVENT_HAD_DATA;
		2004	event.data.data.buf =
		2005	object->source_data + handle->bytes;
		2006	event.data.data.len =
		2007	object->source_len - handle->bytes;
		2008
		2009	/* Update record of last byte emitted */
		2010	if (object->fetch.flags &
		2011	LLCACHE_RETRIEVE_STREAM_DATA) {
		2012	/* Streaming, so reset to zero to
		2013	* minimise amount of cached source data.
		2014	* Additionally, we don't support replay
		2015	* when streaming. */
		2016	orig_handle_read = 0;
		2017	handle->bytes = object->source_len = 0;
		2018	} else {
		2019	orig_handle_read = handle->bytes;
		2020	handle->bytes = object->source_len;
		2021	}
		2022
		2023	/* Emit event */
		2024	error = handle->cb(handle, &event, handle->pw);
		2025	if (user->queued_for_delete) {
		2026	next_user = user->next;
		2027	llcache_object_remove_user(object, user);
		2028	llcache_object_user_destroy(user);
		2029
		2030	if (error != NSERROR_OK)
		2031	return error;
		2032
		2033	continue;
		2034	} else if (error == NSERROR_NEED_DATA) {
		2035	/* User requested replay */
		2036	handle->bytes = orig_handle_read;
		2037
		2038	/* Continue with the next user -- we'll
		2039	* reemit the data next time round */
		2040	user->iterator_target = false;
		2041	next_user = user->next;
		2042	continue;
		2043	} else if (error != NSERROR_OK) {
		2044	user->iterator_target = false;
		2045	return error;
		2046	}
		2047	}
		2048
		2049	/* User: DATA, Obj: COMPLETE => User->COMPLETE */
		2050	if (handle->state == LLCACHE_FETCH_DATA &&
		2051	objstate > LLCACHE_FETCH_DATA) {
		2052	handle->state = LLCACHE_FETCH_COMPLETE;
		2053
		2054	/* Emit DONE event */
		2055	event.type = LLCACHE_EVENT_DONE;
		2056
		2057	error = handle->cb(handle, &event, handle->pw);
		2058	if (user->queued_for_delete) {
		2059	next_user = user->next;
		2060	llcache_object_remove_user(object, user);
		2061	llcache_object_user_destroy(user);
		2062
		2063	if (error != NSERROR_OK)
		2064	return error;
		2065
		2066	continue;
		2067	} else if (error == NSERROR_NEED_DATA) {
		2068	/* User requested replay */
		2069	handle->state = LLCACHE_FETCH_DATA;
		2070
		2071	/* Continue with the next user -- we'll
		2072	* reemit the event next time round */
		2073	user->iterator_target = false;
		2074	next_user = user->next;
		2075	continue;
		2076	} else if (error != NSERROR_OK) {
		2077	user->iterator_target = false;
		2078	return error;
		2079	}
		2080	}
		2081
		2082	/* No longer the target of an iterator */
		2083	user->iterator_target = false;
		2084
		2085	next_user = user->next;
		2086	}
		2087
		2088	return NSERROR_OK;
		2089	}
		2090
		2091	/**
		2092	* Make a snapshot of the current state of an llcache_object.
		2093	*
		2094	* This has the side-effect of the new object being non-cacheable,
		2095	* also not-fetching and not a candidate for any other object.
		2096	*
		2097	* Also note that this new object has no users and at least one
		2098	* should be assigned to it before llcache_clean is entered or it
		2099	* will be immediately cleaned up.
		2100	*
		2101	* \param object The object to take a snapshot of
		2102	* \param snapshot Pointer to receive snapshot of \a object
		2103	* \return NSERROR_OK on success, appropriate error otherwise
		2104	*/
		2105	static nserror llcache_object_snapshot(llcache_object *object,
		2106	llcache_object **snapshot)
		2107	{
		2108	llcache_object *newobj;
		2109	nserror error;
		2110
		2111	error = llcache_object_new(object->url, &newobj);
		2112
		2113	if (error != NSERROR_OK)
		2114	return error;
		2115
		2116	newobj->has_query = object->has_query;
		2117
		2118	newobj->source_alloc = newobj->source_len = object->source_len;
		2119
		2120	if (object->source_len > 0) {
		2121	newobj->source_data = malloc(newobj->source_alloc);
		2122	if (newobj->source_data == NULL) {
		2123	llcache_object_destroy(newobj);
		2124	return NSERROR_NOMEM;
		2125	}
		2126	memcpy(newobj->source_data, object->source_data,
		2127	newobj->source_len);
		2128	}
		2129
		2130	if (object->num_headers > 0) {
		2131	newobj->headers = calloc(sizeof(llcache_header),
		2132	object->num_headers);
		2133	if (newobj->headers == NULL) {
		2134	llcache_object_destroy(newobj);
		2135	return NSERROR_NOMEM;
		2136	}
		2137	while (newobj->num_headers < object->num_headers) {
		2138	llcache_header *nh =
		2139	&(newobj->headers[newobj->num_headers]);
		2140	llcache_header *oh =
		2141	&(object->headers[newobj->num_headers]);
		2142	newobj->num_headers += 1;
		2143	nh->name = strdup(oh->name);
		2144	nh->value = strdup(oh->value);
		2145	if (nh->name == NULL \|\| nh->value == NULL) {
		2146	llcache_object_destroy(newobj);
		2147	return NSERROR_NOMEM;
		2148	}
		2149	}
		2150	}
		2151
		2152	newobj->fetch.state = LLCACHE_FETCH_COMPLETE;
		2153
		2154	*snapshot = newobj;
		2155
		2156	return NSERROR_OK;
		2157	}
		2158
		2159
		2160	/******************************************************************************
		2161	* Public API *
		2162	******************************************************************************/
		2163
		2164	/**
		2165	* Attempt to clean the cache
		2166	*/
		2167	/* Exported interface documented in llcache.h */
		2168	void llcache_clean(void)
		2169	{
		2170	llcache_object object, next;
		2171	uint32_t llcache_size = 0;
		2172	int remaining_lifetime;
		2173
		2174	#ifdef LLCACHE_TRACE
		2175	LOG(("Attempting cache clean"));
		2176	#endif
		2177
		2178	/* Candidates for cleaning are (in order of priority):
		2179	*
		2180	* 1) Uncacheable objects with no users
		2181	* 2) Stale cacheable objects with no users or pending fetches
		2182	* 3) Fresh cacheable objects with no users or pending fetches
		2183	*/
		2184
		2185	/* 1) Uncacheable objects with no users or fetches */
		2186	for (object = llcache->uncached_objects; object != NULL; object = next) {
		2187	next = object->next;
		2188
		2189	/* The candidate count of uncacheable objects is always 0 */
		2190	if ((object->users == NULL) &&
		2191	(object->candidate_count == 0) &&
		2192	(object->fetch.fetch == NULL) &&
		2193	(object->fetch.outstanding_query == false)) {
		2194	#ifdef LLCACHE_TRACE
		2195	LOG(("Found victim %p", object));
		2196	#endif
		2197	llcache_object_remove_from_list(object,
		2198	&llcache->uncached_objects);
		2199	llcache_object_destroy(object);
		2200	} else {
		2201	llcache_size += object->source_len + sizeof(*object);
		2202	}
		2203	}
		2204
		2205	/* 2) Stale cacheable objects with no users or pending fetches */
		2206	for (object = llcache->cached_objects; object != NULL; object = next) {
		2207	next = object->next;
		2208
		2209	remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache);
		2210
		2211	if ((object->users == NULL) &&
		2212	(object->candidate_count == 0) &&
		2213	(object->fetch.fetch == NULL) &&
		2214	(object->fetch.outstanding_query == false)) {
		2215
		2216	if (remaining_lifetime > 0) {
		2217	/* object is fresh */
		2218	llcache_size += object->source_len + sizeof(*object);
		2219	} else {
		2220	/* object is not fresh */
		2221	#ifdef LLCACHE_TRACE
		2222	LOG(("Found stale cacheable object (%p) with no users or pending fetches", object));
		2223	#endif
		2224	llcache_object_remove_from_list(object,
		2225	&llcache->cached_objects);
		2226	llcache_object_destroy(object);
		2227	}
		2228	} else {
		2229	llcache_size += object->source_len + sizeof(*object);
		2230	}
		2231	}
		2232
		2233	/* 3) Fresh cacheable objects with no users or pending
		2234	* fetches, only if the cache exceeds the configured size.
		2235	*/
		2236	if (llcache->limit < llcache_size) {
		2237	for (object = llcache->cached_objects; object != NULL;
		2238	object = next) {
		2239	next = object->next;
		2240
		2241	if ((object->users == NULL) &&
		2242	(object->candidate_count == 0) &&
		2243	(object->fetch.fetch == NULL) &&
		2244	(object->fetch.outstanding_query == false)) {
		2245	#ifdef LLCACHE_TRACE
		2246	LOG(("Found victim %p", object));
		2247	#endif
		2248	llcache_size -=
		2249	object->source_len + sizeof(*object);
		2250
		2251	llcache_object_remove_from_list(object,
		2252	&llcache->cached_objects);
		2253	llcache_object_destroy(object);
		2254	}
		2255	}
		2256	}
		2257
		2258	#ifdef LLCACHE_TRACE
		2259	LOG(("Size: %u", llcache_size));
		2260	#endif
		2261
		2262	}
		2263
		2264	/* See llcache.h for documentation */
		2265	nserror
		2266	llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit)
		2267	{
		2268	llcache = calloc(1, sizeof(struct llcache_s));
		2269	if (llcache == NULL) {
		2270	return NSERROR_NOMEM;
		2271	}
		2272
		2273	llcache->query_cb = cb;
		2274	llcache->query_cb_pw = pw;
		2275	llcache->limit = llcache_limit;
		2276
		2277	/* Create static scheme strings */
		2278	if (lwc_intern_string("file", SLEN("file"),
		2279	&llcache_file_lwc) != lwc_error_ok)
		2280	return NSERROR_NOMEM;
		2281
		2282	if (lwc_intern_string("about", SLEN("about"),
		2283	&llcache_about_lwc) != lwc_error_ok)
		2284	return NSERROR_NOMEM;
		2285
		2286	if (lwc_intern_string("resource", SLEN("resource"),
		2287	&llcache_resource_lwc) != lwc_error_ok)
		2288	return NSERROR_NOMEM;
		2289
		2290	LOG(("llcache initialised with a limit of %d bytes", llcache_limit));
		2291
		2292	return NSERROR_OK;
		2293	}
		2294
		2295	/* See llcache.h for documentation */
		2296	void llcache_finalise(void)
		2297	{
		2298	llcache_object object, next;
		2299
		2300	/* Clean uncached objects */
		2301	for (object = llcache->uncached_objects; object != NULL; object = next) {
		2302	llcache_object_user user, next_user;
		2303
		2304	next = object->next;
		2305
		2306	for (user = object->users; user != NULL; user = next_user) {
		2307	next_user = user->next;
		2308
		2309	if (user->handle != NULL)
		2310	free(user->handle);
		2311
		2312	free(user);
		2313	}
		2314
		2315	/* Fetch system has already been destroyed */
		2316	object->fetch.fetch = NULL;
		2317
		2318	llcache_object_destroy(object);
		2319	}
		2320
		2321	/* Clean cached objects */
		2322	for (object = llcache->cached_objects; object != NULL; object = next) {
		2323	llcache_object_user user, next_user;
		2324
		2325	next = object->next;
		2326
		2327	for (user = object->users; user != NULL; user = next_user) {
		2328	next_user = user->next;
		2329
		2330	if (user->handle != NULL)
		2331	free(user->handle);
		2332
		2333	free(user);
		2334	}
		2335
		2336	/* Fetch system has already been destroyed */
		2337	object->fetch.fetch = NULL;
		2338
		2339	llcache_object_destroy(object);
		2340	}
		2341
		2342	/* Unref static scheme lwc strings */
		2343	lwc_string_unref(llcache_file_lwc);
		2344	lwc_string_unref(llcache_about_lwc);
		2345	lwc_string_unref(llcache_resource_lwc);
		2346
		2347	free(llcache);
		2348	llcache = NULL;
		2349	}
		2350
		2351	/* See llcache.h for documentation */
		2352	nserror llcache_poll(void)
		2353	{
		2354	llcache_object *object;
		2355
		2356	fetch_poll();
		2357
		2358	/* Catch new users up with state of objects */
		2359	for (object = llcache->cached_objects; object != NULL;
		2360	object = object->next) {
		2361	llcache_object_notify_users(object);
		2362	}
		2363
		2364	for (object = llcache->uncached_objects; object != NULL;
		2365	object = object->next) {
		2366	llcache_object_notify_users(object);
		2367	}
		2368
		2369	return NSERROR_OK;
		2370	}
		2371
		2372	/* See llcache.h for documentation */
		2373	nserror llcache_handle_retrieve(nsurl *url, uint32_t flags,
		2374	nsurl referer, const llcache_post_data post,
		2375	llcache_handle_callback cb, void *pw,
		2376	llcache_handle **result)
		2377	{
		2378	nserror error;
		2379	llcache_object_user *user;
		2380	llcache_object *object;
		2381
		2382	/* Can we fetch this URL at all? */
		2383	if (fetch_can_fetch(url) == false)
		2384	return NSERROR_NO_FETCH_HANDLER;
		2385
		2386	/* Create a new object user */
		2387	error = llcache_object_user_new(cb, pw, &user);
		2388	if (error != NSERROR_OK)
		2389	return error;
		2390
		2391	/* Retrieve a suitable object from the cache,
		2392	* creating a new one if needed. */
		2393	error = llcache_object_retrieve(url, flags, referer, post, 0, &object);
		2394	if (error != NSERROR_OK) {
		2395	llcache_object_user_destroy(user);
		2396	return error;
		2397	}
		2398
		2399	/* Add user to object */
		2400	llcache_object_add_user(object, user);
		2401
		2402	*result = user->handle;
		2403
		2404	return NSERROR_OK;
		2405	}
		2406
		2407	/* See llcache.h for documentation */
		2408	nserror llcache_handle_change_callback(llcache_handle *handle,
		2409	llcache_handle_callback cb, void *pw)
		2410	{
		2411	handle->cb = cb;
		2412	handle->pw = pw;
		2413
		2414	return NSERROR_OK;
		2415	}
		2416
		2417	/* See llcache.h for documentation */
		2418	nserror llcache_handle_release(llcache_handle *handle)
		2419	{
		2420	nserror error = NSERROR_OK;
		2421	llcache_object *object = handle->object;
		2422	llcache_object_user *user = llcache_object_find_user(handle);
		2423
		2424	assert(user != NULL);
		2425
		2426	if (user->iterator_target) {
		2427	/* Can't remove / delete user object if it's
		2428	* the target of an iterator */
		2429	user->queued_for_delete = true;
		2430	} else {
		2431	/* Remove the user from the object and destroy it */
		2432	error = llcache_object_remove_user(object, user);
		2433	if (error == NSERROR_OK) {
		2434	error = llcache_object_user_destroy(user);
		2435	}
		2436	}
		2437
		2438	return error;
		2439	}
		2440
		2441	/* See llcache.h for documentation */
		2442	nserror llcache_handle_clone(llcache_handle handle, llcache_handle *result)
		2443	{
		2444	nserror error;
		2445	llcache_object_user *newuser;
		2446
		2447	error = llcache_object_user_new(handle->cb, handle->pw, &newuser);
		2448	if (error == NSERROR_OK) {
		2449	llcache_object_add_user(handle->object, newuser);
		2450	newuser->handle->state = handle->state;
		2451	*result = newuser->handle;
		2452	}
		2453
		2454	return error;
		2455	}
		2456
		2457	/* See llcache.h for documentation */
		2458	nserror llcache_handle_abort(llcache_handle *handle)
		2459	{
		2460	llcache_object_user *user = llcache_object_find_user(handle);
		2461	llcache_object object = handle->object, newobject;
		2462	nserror error = NSERROR_OK;
		2463	bool all_alone = true;
		2464
		2465	/* Determine if we are the only user */
		2466	if (user->prev != NULL)
		2467	all_alone = false;
		2468	if (user->next != NULL)
		2469	all_alone = false;
		2470
		2471	if (all_alone == false) {
		2472	/* We must snapshot this object */
		2473	error = llcache_object_snapshot(object, &newobject);
		2474	if (error != NSERROR_OK)
		2475	return error;
		2476
		2477	/* Move across to the new object */
		2478	if (user->iterator_target) {
		2479	/* User is current iterator target, clone it */
		2480	llcache_object_user *newuser =
		2481	calloc(1, sizeof(llcache_object_user));
		2482	if (newuser == NULL) {
		2483	llcache_object_destroy(newobject);
		2484	return NSERROR_NOMEM;
		2485	}
		2486
		2487	/* Move handle across to clone */
		2488	newuser->handle = user->handle;
		2489	user->handle = NULL;
		2490
		2491	/* Mark user as needing deletion */
		2492	user->queued_for_delete = true;
		2493
		2494	llcache_object_add_user(newobject, newuser);
		2495	} else {
		2496	llcache_object_remove_user(object, user);
		2497	llcache_object_add_user(newobject, user);
		2498	}
		2499
		2500	/* Add new object to uncached list */
		2501	llcache_object_add_to_list(newobject,
		2502	&llcache->uncached_objects);
		2503	} else {
		2504	/* We're the only user, so abort any fetch in progress */
		2505	if (object->fetch.fetch != NULL) {
		2506	fetch_abort(object->fetch.fetch);
		2507	object->fetch.fetch = NULL;
		2508	}
		2509
		2510	object->fetch.state = LLCACHE_FETCH_COMPLETE;
		2511
		2512	/* Invalidate cache control data */
		2513	llcache_invalidate_cache_control_data(object);
		2514	}
		2515
		2516	return error;
		2517	}
		2518
		2519	/* See llcache.h for documentation */
		2520	nserror llcache_handle_force_stream(llcache_handle *handle)
		2521	{
		2522	llcache_object_user *user = llcache_object_find_user(handle);
		2523	llcache_object *object = handle->object;
		2524
		2525	/* Cannot stream if there are multiple users */
		2526	if (user->prev != NULL \|\| user->next != NULL)
		2527	return NSERROR_OK;
		2528
		2529	/* Forcibly uncache this object */
		2530	if (llcache_object_in_list(object, llcache->cached_objects)) {
		2531	llcache_object_remove_from_list(object,
		2532	&llcache->cached_objects);
		2533	llcache_object_add_to_list(object, &llcache->uncached_objects);
		2534	}
		2535
		2536	object->fetch.flags \|= LLCACHE_RETRIEVE_STREAM_DATA;
		2537
		2538	return NSERROR_OK;
		2539	}
		2540
		2541	/* See llcache.h for documentation */
		2542	nserror llcache_handle_invalidate_cache_data(llcache_handle *handle)
		2543	{
		2544	if (handle->object != NULL && handle->object->fetch.fetch == NULL &&
		2545	handle->object->cache.no_cache ==
		2546	LLCACHE_VALIDATE_FRESH) {
		2547	handle->object->cache.no_cache = LLCACHE_VALIDATE_ONCE;
		2548	}
		2549
		2550	return NSERROR_OK;
		2551	}
		2552
		2553	/* See llcache.h for documentation */
		2554	nsurl llcache_handle_get_url(const llcache_handle handle)
		2555	{
		2556	return handle->object != NULL ? handle->object->url : NULL;
		2557	}
		2558
		2559	/* See llcache.h for documentation */
		2560	const uint8_t llcache_handle_get_source_data(const llcache_handle handle,
		2561	size_t *size)
		2562	{
		2563	*size = handle->object != NULL ? handle->object->source_len : 0;
		2564
		2565	return handle->object != NULL ? handle->object->source_data : NULL;
		2566	}
		2567
		2568	/* See llcache.h for documentation */
		2569	const char llcache_handle_get_header(const llcache_handle handle,
		2570	const char *key)
		2571	{
		2572	const llcache_object *object = handle->object;
		2573	size_t i;
		2574
		2575	if (object == NULL)
		2576	return NULL;
		2577
		2578	/* About as trivial as possible */
		2579	for (i = 0; i < object->num_headers; i++) {
		2580	if (strcasecmp(key, object->headers[i].name) == 0)
		2581	return object->headers[i].value;
		2582	}
		2583
		2584	return NULL;
		2585	}
		2586
		2587	/* See llcache.h for documentation */
		2588	bool llcache_handle_references_same_object(const llcache_handle *a,
		2589	const llcache_handle *b)
		2590	{
		2591	return a->object == b->object;
		2592	}
		2593

Subversion Repositories Kolibri OS

(root)/contrib/network/netsurf/netsurf/content/llcache.c – Rev 4821