Subversion Repositories Kolibri OS

/*

 * Copyright 2009 John-Mark Bell 

 *

 * This file is part of NetSurf, http://www.netsurf-browser.org/

 *

 * NetSurf is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; version 2 of the License.

 *

 * NetSurf is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see .

 */

/** \file

 * High-level resource cache (implementation)

 */

#include 

#include 

#include 

#include "content/content.h"

#include "content/hlcache.h"

#include "content/mimesniff.h"

#include "utils/http.h"

#include "utils/log.h"

#include "utils/messages.h"

#include "utils/ring.h"

#include "utils/schedule.h"

#include "utils/url.h"

#include "utils/utils.h"

typedef struct hlcache_entry hlcache_entry;

typedef struct hlcache_retrieval_ctx hlcache_retrieval_ctx;

/** High-level cache retrieval context */

struct hlcache_retrieval_ctx {

	struct hlcache_retrieval_ctx *r_prev; /**< Previous retrieval context in the ring */

	struct hlcache_retrieval_ctx *r_next; /**< Next retrieval context in the ring */

	llcache_handle *llcache;	/**< Low-level cache handle */

	hlcache_handle *handle;		/**< High-level handle for object */

	uint32_t flags;			/**< Retrieval flags */

	content_type accepted_types;	/**< Accepted types */

	hlcache_child_context child;	/**< Child context */

	bool migrate_target;		/**< Whether this context is the migration target */

};

/** High-level cache handle */

struct hlcache_handle {

	hlcache_entry *entry;		/**< Pointer to cache entry */

	hlcache_handle_callback cb;	/**< Client callback */

	void *pw;			/**< Client data */

};

/** Entry in high-level cache */

struct hlcache_entry {

	struct content *content;	/**< Pointer to associated content */

	hlcache_entry *next;		/**< Next sibling */

	hlcache_entry *prev;		/**< Previous sibling */

};

/** Current state of the cache.

 *

 * Global state of the cache.

 */

struct hlcache_s {

	struct hlcache_parameters params;

	/** List of cached content objects */

	hlcache_entry *content_list;

	/** Ring of retrieval contexts */

	hlcache_retrieval_ctx *retrieval_ctx_ring;

	/* statsistics */

	unsigned int hit_count;

	unsigned int miss_count;

};

/** high level cache state */

static struct hlcache_s *hlcache = NULL;

static void hlcache_clean(void *ignored);

static nserror hlcache_llcache_callback(llcache_handle *handle,

		const llcache_event *event, void *pw);

static nserror hlcache_migrate_ctx(hlcache_retrieval_ctx *ctx,

		lwc_string *effective_type);

static bool hlcache_type_is_acceptable(lwc_string *mime_type,

		content_type accepted_types, content_type *computed_type);

static nserror hlcache_find_content(hlcache_retrieval_ctx *ctx,

		lwc_string *effective_type);

static void hlcache_content_callback(struct content *c,

		content_msg msg, union content_msg_data data, void *pw);

/******************************************************************************

 * Public API								      *

 ******************************************************************************/

nserror

hlcache_initialise(const struct hlcache_parameters *hlcache_parameters)

{

	nserror ret;

	hlcache = calloc(1, sizeof(struct hlcache_s));

	if (hlcache == NULL) {

		return NSERROR_NOMEM;

	}

	ret = llcache_initialise(hlcache_parameters->cb,

				 hlcache_parameters->cb_ctx,

				 hlcache_parameters->limit);

	if (ret != NSERROR_OK) {

		free(hlcache);

		hlcache = NULL;

		return ret;

	}

	hlcache->params = *hlcache_parameters;

	/* Schedule the cache cleanup */

	schedule(hlcache->params.bg_clean_time / 10, hlcache_clean, NULL);

	return NSERROR_OK;

}

/* See hlcache.h for documentation */

void hlcache_stop(void)

{

	/* Remove the hlcache_clean schedule */

	schedule_remove(hlcache_clean, NULL);

}

/* See hlcache.h for documentation */

void hlcache_finalise(void)

{

	uint32_t num_contents, prev_contents;

	hlcache_entry *entry;

	hlcache_retrieval_ctx *ctx, *next;

	/* Obtain initial count of contents remaining */

	for (num_contents = 0, entry = hlcache->content_list;

			entry != NULL; entry = entry->next) {

		num_contents++;

	}

	LOG(("%d contents remain before cache drain", num_contents));

	/* Drain cache */

	do {

		prev_contents = num_contents;

		hlcache_clean(NULL);

		for (num_contents = 0, entry = hlcache->content_list;

				entry != NULL; entry = entry->next) {

			num_contents++;

		}

	} while (num_contents > 0 && num_contents != prev_contents);

	LOG(("%d contents remaining:", num_contents));

	for (entry = hlcache->content_list; entry != NULL; entry = entry->next) {

		hlcache_handle entry_handle = { entry, NULL, NULL };

		if (entry->content != NULL) {

			LOG(("	%p : %s (%d users)", entry,

			     nsurl_access(hlcache_handle_get_url(&entry_handle)), content_count_users(entry->content)));

		} else {

			LOG(("	%p", entry));

		}

	}

	/* Clean up retrieval contexts */

	if (hlcache->retrieval_ctx_ring != NULL) {

		ctx = hlcache->retrieval_ctx_ring;

		do {

			next = ctx->r_next;

			if (ctx->llcache != NULL)

				llcache_handle_release(ctx->llcache);

			if (ctx->handle != NULL)

				free(ctx->handle);

			if (ctx->child.charset != NULL)

				free((char *) ctx->child.charset);

			free(ctx);

			ctx = next;

		} while (ctx != hlcache->retrieval_ctx_ring);

		hlcache->retrieval_ctx_ring = NULL;

	}

	LOG(("hit/miss %d/%d", hlcache->hit_count, hlcache->miss_count));

	free(hlcache);

	hlcache = NULL;

	LOG(("Finalising low-level cache"));

	llcache_finalise();

}

/* See hlcache.h for documentation */

nserror hlcache_poll(void)

{

	llcache_poll();

	return NSERROR_OK;

}

/* See hlcache.h for documentation */

nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags,

		nsurl *referer, llcache_post_data *post,

		hlcache_handle_callback cb, void *pw,

		hlcache_child_context *child,

		content_type accepted_types, hlcache_handle **result)

{

	hlcache_retrieval_ctx *ctx;

	nserror error;

	assert(cb != NULL);

	ctx = calloc(1, sizeof(hlcache_retrieval_ctx));

	if (ctx == NULL)

		return NSERROR_NOMEM;

	ctx->handle = calloc(1, sizeof(hlcache_handle));

	if (ctx->handle == NULL) {

		free(ctx);

		return NSERROR_NOMEM;

	}

	if (child != NULL) {

		if (child->charset != NULL) {

			ctx->child.charset = strdup(child->charset);

			if (ctx->child.charset == NULL) {

				free(ctx->handle);

				free(ctx);

				return NSERROR_NOMEM;

			}

		}

		ctx->child.quirks = child->quirks;

	}

	ctx->flags = flags;

	ctx->accepted_types = accepted_types;

	ctx->handle->cb = cb;

	ctx->handle->pw = pw;

	error = llcache_handle_retrieve(url, flags, referer, post,

			hlcache_llcache_callback, ctx,

			&ctx->llcache);

	if (error != NSERROR_OK) {

		free((char *) ctx->child.charset);

		free(ctx->handle);

		free(ctx);

		return error;

	}

	RING_INSERT(hlcache->retrieval_ctx_ring, ctx);

	*result = ctx->handle;

	return NSERROR_OK;

}

/* See hlcache.h for documentation */

nserror hlcache_handle_release(hlcache_handle *handle)

{

	if (handle->entry != NULL) {

		content_remove_user(handle->entry->content,

				hlcache_content_callback, handle);

	} else {

		RING_ITERATE_START(struct hlcache_retrieval_ctx,

				   hlcache->retrieval_ctx_ring,

				   ictx) {

			if (ictx->handle == handle &&

					ictx->migrate_target == false) {

				/* This is the nascent context for us,

				 * so abort the fetch */

				llcache_handle_abort(ictx->llcache);

				llcache_handle_release(ictx->llcache);

				/* Remove us from the ring */

				RING_REMOVE(hlcache->retrieval_ctx_ring, ictx);

				/* Throw us away */

				free((char *) ictx->child.charset);

				free(ictx);

				/* And stop */

				RING_ITERATE_STOP(hlcache->retrieval_ctx_ring,

						ictx);

			}

		} RING_ITERATE_END(hlcache->retrieval_ctx_ring, ictx);

	}

	handle->cb = NULL;

	handle->pw = NULL;

	free(handle);

	return NSERROR_OK;

}

/* See hlcache.h for documentation */

struct content *hlcache_handle_get_content(const hlcache_handle *handle)

{

	assert(handle != NULL);

	if (handle->entry != NULL)

		return handle->entry->content;

	return NULL;

}

/* See hlcache.h for documentation */

nserror hlcache_handle_abort(hlcache_handle *handle)

{

	struct hlcache_entry *entry = handle->entry;

	struct content *c;

	if (entry == NULL) {

		/* This handle is not yet associated with a cache entry.

		 * The implication is that the fetch for the handle has

		 * not progressed to the point where the entry can be

		 * created. */

		RING_ITERATE_START(struct hlcache_retrieval_ctx,

				   hlcache->retrieval_ctx_ring,

				   ictx) {

			if (ictx->handle == handle &&

					ictx->migrate_target == false) {

				/* This is the nascent context for us,

				 * so abort the fetch */

				llcache_handle_abort(ictx->llcache);

				llcache_handle_release(ictx->llcache);

				/* Remove us from the ring */

				RING_REMOVE(hlcache->retrieval_ctx_ring, ictx);

				/* Throw us away */

				free((char *) ictx->child.charset);

				free(ictx);

				/* And stop */

				RING_ITERATE_STOP(hlcache->retrieval_ctx_ring,

						ictx);

			}

		} RING_ITERATE_END(hlcache->retrieval_ctx_ring, ictx);

		return NSERROR_OK;

	}

	c = entry->content;

	if (content_count_users(c) > 1) {

		/* We are not the only user of 'c' so clone it. */

		struct content *clone = content_clone(c);

		if (clone == NULL)

			return NSERROR_NOMEM;

		entry = calloc(sizeof(struct hlcache_entry), 1);

		if (entry == NULL) {

			content_destroy(clone);

			return NSERROR_NOMEM;

		}

		if (content_add_user(clone,

				hlcache_content_callback, handle) == false) {

			content_destroy(clone);

			free(entry);

			return NSERROR_NOMEM;

		}

		content_remove_user(c, hlcache_content_callback, handle);

		entry->content = clone;

		handle->entry = entry;

		entry->prev = NULL;

		entry->next = hlcache->content_list;

		if (hlcache->content_list != NULL)

			hlcache->content_list->prev = entry;

		hlcache->content_list = entry;

		c = clone;

	}

	return content_abort(c);

}

/* See hlcache.h for documentation */

nserror hlcache_handle_replace_callback(hlcache_handle *handle,

		hlcache_handle_callback cb, void *pw)

{

	handle->cb = cb;

	handle->pw = pw;

	return NSERROR_OK;

}

nserror hlcache_handle_clone(hlcache_handle *handle, hlcache_handle **result)

{

	*result = NULL;

	return NSERROR_CLONE_FAILED;

}

/* See hlcache.h for documentation */

nsurl *hlcache_handle_get_url(const hlcache_handle *handle)

{

	nsurl *result = NULL;

	assert(handle != NULL);

	if (handle->entry != NULL) {

		result = content_get_url(handle->entry->content);

	} else {

		RING_ITERATE_START(struct hlcache_retrieval_ctx,

				   hlcache->retrieval_ctx_ring,

				   ictx) {

			if (ictx->handle == handle) {

				/* This is the nascent context for us */

				result = llcache_handle_get_url(ictx->llcache);

				/* And stop */

				RING_ITERATE_STOP(hlcache->retrieval_ctx_ring,

						ictx);

			}

		} RING_ITERATE_END(hlcache->retrieval_ctx_ring, ictx);

	}

	return result;

}

/******************************************************************************

 * High-level cache internals						      *

 ******************************************************************************/

/**

 * Attempt to clean the cache

 */

void hlcache_clean(void *ignored)

{

	hlcache_entry *entry, *next;

	for (entry = hlcache->content_list; entry != NULL; entry = next) {

		next = entry->next;

		if (entry->content == NULL)

			continue;

		if (content__get_status(entry->content) ==

				CONTENT_STATUS_LOADING)

			continue;

		if (content_count_users(entry->content) != 0)

			continue;

		/** \todo This is over-zealous: all unused contents

		 * will be immediately destroyed. Ideally, we want to

		 * purge all unused contents that are using stale

		 * source data, and enough fresh contents such that

		 * the cache fits in the configured cache size limit.

		 */

		/* Remove entry from cache */

		if (entry->prev == NULL)

			hlcache->content_list = entry->next;

		else

			entry->prev->next = entry->next;

		if (entry->next != NULL)

			entry->next->prev = entry->prev;

		/* Destroy content */

		content_destroy(entry->content);

		/* Destroy entry */

		free(entry);

	}

	/* Attempt to clean the llcache */

	llcache_clean();

	/* Re-schedule ourselves */

	schedule(hlcache->params.bg_clean_time / 10, hlcache_clean, NULL);

}

/**

 * Handler for low-level cache events

 *

 * \param handle  Handle for which event is issued

 * \param event	  Event data

 * \param pw	  Pointer to client-specific data

 * \return NSERROR_OK on success, appropriate error otherwise

 */

nserror hlcache_llcache_callback(llcache_handle *handle,

		const llcache_event *event, void *pw)

{

	hlcache_retrieval_ctx *ctx = pw;

	lwc_string *effective_type = NULL;

	nserror error;

	assert(ctx->llcache == handle);

	switch (event->type) {

	case LLCACHE_EVENT_HAD_HEADERS:

		error = mimesniff_compute_effective_type(handle, NULL, 0,

				ctx->flags & HLCACHE_RETRIEVE_SNIFF_TYPE,

				ctx->accepted_types == CONTENT_IMAGE,

				&effective_type);

		if (error == NSERROR_OK || error == NSERROR_NOT_FOUND) {

			/* If the sniffer was successful or failed to find

			 * a Content-Type header when sniffing was

			 * prohibited, we must migrate the retrieval context. */

			error = hlcache_migrate_ctx(ctx, effective_type);

			if (effective_type != NULL)

				lwc_string_unref(effective_type);

		}

		/* No need to report that we need data:

		 * we'll get some anyway if there is any */

		if (error == NSERROR_NEED_DATA)

			error = NSERROR_OK;

		return error;

		break;

	case LLCACHE_EVENT_HAD_DATA:

		error = mimesniff_compute_effective_type(handle,

				event->data.data.buf, event->data.data.len,

				ctx->flags & HLCACHE_RETRIEVE_SNIFF_TYPE,

				ctx->accepted_types == CONTENT_IMAGE,

				&effective_type);

		if (error != NSERROR_OK) {

			assert(0 && "MIME sniff failed with data");

		}

		error = hlcache_migrate_ctx(ctx, effective_type);

		lwc_string_unref(effective_type);

		return error;

		break;

	case LLCACHE_EVENT_DONE:

		/* DONE event before we could determine the effective MIME type.

		 */

		error = mimesniff_compute_effective_type(handle,

				NULL, 0, false, false, &effective_type);

		if (error == NSERROR_OK) {

			error = hlcache_migrate_ctx(ctx, effective_type);

			lwc_string_unref(effective_type);

			return error;

		}

		if (ctx->handle->cb != NULL) {

			hlcache_event hlevent;

			hlevent.type = CONTENT_MSG_ERROR;

			hlevent.data.error = messages_get("BadType");

			ctx->handle->cb(ctx->handle, &hlevent, ctx->handle->pw);

		}

		break;

	case LLCACHE_EVENT_ERROR:

		if (ctx->handle->cb != NULL) {

			hlcache_event hlevent;

			hlevent.type = CONTENT_MSG_ERROR;

			hlevent.data.error = event->data.msg;

			ctx->handle->cb(ctx->handle, &hlevent, ctx->handle->pw);

		}

		break;

	case LLCACHE_EVENT_PROGRESS:

		break;

	}

	return NSERROR_OK;

}

/**

 * Migrate a retrieval context into its final destination content

 *

 * \param ctx             Context to migrate

 * \param effective_type  The effective MIME type of the content, or NULL

 * \return NSERROR_OK on success,

 *         NSERROR_NEED_DATA on success where data is needed,

 *         appropriate error otherwise

 */

nserror hlcache_migrate_ctx(hlcache_retrieval_ctx *ctx,

		lwc_string *effective_type)

{

	content_type type = CONTENT_NONE;

	nserror error = NSERROR_OK;

	ctx->migrate_target = true;

	if (effective_type != NULL &&

			hlcache_type_is_acceptable(effective_type,

			ctx->accepted_types, &type)) {

		error = hlcache_find_content(ctx, effective_type);

		if (error != NSERROR_OK && error != NSERROR_NEED_DATA) {

			if (ctx->handle->cb != NULL) {

				hlcache_event hlevent;

				hlevent.type = CONTENT_MSG_ERROR;

				hlevent.data.error = messages_get("MiscError");

				ctx->handle->cb(ctx->handle, &hlevent,

						ctx->handle->pw);

			}

			llcache_handle_abort(ctx->llcache);

			llcache_handle_release(ctx->llcache);

		}

	} else if (type == CONTENT_NONE &&

			(ctx->flags & HLCACHE_RETRIEVE_MAY_DOWNLOAD)) {

		/* Unknown type, and we can download, so convert */

		llcache_handle_force_stream(ctx->llcache);

		if (ctx->handle->cb != NULL) {

			hlcache_event hlevent;

			hlevent.type = CONTENT_MSG_DOWNLOAD;

			hlevent.data.download = ctx->llcache;

			ctx->handle->cb(ctx->handle, &hlevent,

					ctx->handle->pw);

		}

		/* Ensure caller knows we need data */

		error = NSERROR_NEED_DATA;

	} else {

		/* Unacceptable type: report error */

		if (ctx->handle->cb != NULL) {

			hlcache_event hlevent;

			hlevent.type = CONTENT_MSG_ERROR;

			hlevent.data.error = messages_get("UnacceptableType");

			ctx->handle->cb(ctx->handle, &hlevent,

					ctx->handle->pw);

		}

		llcache_handle_abort(ctx->llcache);

		llcache_handle_release(ctx->llcache);

	}

	ctx->migrate_target = false;

	/* No longer require retrieval context */

	RING_REMOVE(hlcache->retrieval_ctx_ring, ctx);

	free((char *) ctx->child.charset);

	free(ctx);

	return error;

}

/**

 * Determine if the specified MIME type is acceptable

 *

 * \param mime_type       MIME type to consider

 * \param accepted_types  Array of acceptable types, or NULL for any

 * \param computed_type	  Pointer to location to receive computed type of object

 * \return True if the type is acceptable, false otherwise

 */

bool hlcache_type_is_acceptable(lwc_string *mime_type,

		content_type accepted_types, content_type *computed_type)

{

	content_type type;

	type = content_factory_type_from_mime_type(mime_type);

	*computed_type = type;

	return ((accepted_types & type) != 0);

}

/**

 * Find a content for the high-level cache handle

 *

 * \param ctx             High-level cache retrieval context

 * \param effective_type  Effective MIME type of content

 * \return NSERROR_OK on success,

 *         NSERROR_NEED_DATA on success where data is needed,

 *         appropriate error otherwise

 *

 * \pre handle::state == HLCACHE_HANDLE_NEW

 * \pre Headers must have been received for associated low-level handle

 * \post Low-level handle is either released, or associated with new content

 * \post High-level handle is registered with content

 */

nserror hlcache_find_content(hlcache_retrieval_ctx *ctx,

		lwc_string *effective_type)

{

	hlcache_entry *entry;

	hlcache_event event;

	nserror error = NSERROR_OK;

	/* Search list of cached contents for a suitable one */

	for (entry = hlcache->content_list; entry != NULL; entry = entry->next) {

		hlcache_handle entry_handle = { entry, NULL, NULL };

		const llcache_handle *entry_llcache;

		if (entry->content == NULL)

			continue;

		/* Ignore contents in the error state */

		if (content_get_status(&entry_handle) == CONTENT_STATUS_ERROR)

			continue;

		/* Ensure that content is shareable */

		if (content_is_shareable(entry->content) == false)

			continue;

		/* Ensure that quirks mode is acceptable */

		if (content_matches_quirks(entry->content,

				ctx->child.quirks) == false)

			continue;

		/* Ensure that content uses same low-level object as

		 * low-level handle */

		entry_llcache = content_get_llcache_handle(entry->content);

		if (llcache_handle_references_same_object(entry_llcache,

				ctx->llcache))

			break;

	}

	if (entry == NULL) {

		/* No existing entry, so need to create one */

		entry = malloc(sizeof(hlcache_entry));

		if (entry == NULL)

			return NSERROR_NOMEM;

		/* Create content using llhandle */

		entry->content = content_factory_create_content(ctx->llcache,

				ctx->child.charset, ctx->child.quirks,

				effective_type);

		if (entry->content == NULL) {

			free(entry);

			return NSERROR_NOMEM;

		}

		/* Insert into cache */

		entry->prev = NULL;

		entry->next = hlcache->content_list;

		if (hlcache->content_list != NULL)

			hlcache->content_list->prev = entry;

		hlcache->content_list = entry;

		/* Signal to caller that we created a content */

		error = NSERROR_NEED_DATA;

		hlcache->miss_count++;

	} else {

		/* Found a suitable content: no longer need low-level handle */

		llcache_handle_release(ctx->llcache);

		hlcache->hit_count++;

	}

	/* Associate handle with content */

	if (content_add_user(entry->content,

			hlcache_content_callback, ctx->handle) == false)

		return NSERROR_NOMEM;

	/* Associate cache entry with handle */

	ctx->handle->entry = entry;

	/* Catch handle up with state of content */

	if (ctx->handle->cb != NULL) {

		content_status status = content_get_status(ctx->handle);

		if (status == CONTENT_STATUS_LOADING) {

			event.type = CONTENT_MSG_LOADING;

			ctx->handle->cb(ctx->handle, &event, ctx->handle->pw);

		} else if (status == CONTENT_STATUS_READY) {

			event.type = CONTENT_MSG_LOADING;

			ctx->handle->cb(ctx->handle, &event, ctx->handle->pw);

			if (ctx->handle->cb != NULL) {

				event.type = CONTENT_MSG_READY;

				ctx->handle->cb(ctx->handle, &event,

						ctx->handle->pw);

			}

		} else if (status == CONTENT_STATUS_DONE) {

			event.type = CONTENT_MSG_LOADING;

			ctx->handle->cb(ctx->handle, &event, ctx->handle->pw);

			if (ctx->handle->cb != NULL) {

				event.type = CONTENT_MSG_READY;

				ctx->handle->cb(ctx->handle, &event,

						ctx->handle->pw);

			}

			if (ctx->handle->cb != NULL) {

				event.type = CONTENT_MSG_DONE;

				ctx->handle->cb(ctx->handle, &event,

						ctx->handle->pw);

			}

		}

	}

	return error;

}

/**

 * Veneer between content callback API and hlcache callback API

 *

 * \param c	Content to emit message for

 * \param msg	Message to emit

 * \param data	Data for message

 * \param pw	Pointer to private data (hlcache_handle)

 */

void hlcache_content_callback(struct content *c, content_msg msg,

		union content_msg_data data, void *pw)

{

	hlcache_handle *handle = pw;

	hlcache_event event;

	nserror error = NSERROR_OK;

	event.type = msg;

	event.data = data;

	if (handle->cb != NULL)

		error = handle->cb(handle, &event, handle->pw);

	if (error != NSERROR_OK)

		LOG(("Error in callback: %d", error));

}