WebSVN – Kolibri OS – Blame – /contrib/network/netsurf/netsurf/content/fetchers/curl.c

Rev	Author	Line No.	Line
3584	sourcerer	1	/*
		2	* Copyright 2006 Daniel Silverstone
		3	* Copyright 2007 James Bursa
		4	* Copyright 2003 Phil Mellor
		5	*
		6	* This file is part of NetSurf.
		7	*
		8	* NetSurf is free software; you can redistribute it and/or modify
		9	* it under the terms of the GNU General Public License as published by
		10	* the Free Software Foundation; version 2 of the License.
		11	*
		12	* NetSurf is distributed in the hope that it will be useful,
		13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
		14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	* GNU General Public License for more details.
		16	*
		17	* You should have received a copy of the GNU General Public License
		18	* along with this program. If not, see .
		19	*/
		20
		21	/** \file
		22	* Fetching of data from a URL (implementation).
		23	*
		24	* This implementation uses libcurl's 'multi' interface.
		25	*
		26	*
		27	* The CURL handles are cached in the curl_handle_ring. There are at most
		28	* ::max_cached_fetch_handles in this ring.
		29	*/
		30
5043	ashmew2	31	/*
		32	TODO:
		33	Have a double linked list containing all the handles on work that needs to be done
		34	Inside fetch_poll_curl, do work on it.
		35	Let the overall structure remain intact
		36	*/
4224	sourcerer	37
3584	sourcerer	38	#include
		39	#include
		40	#include
		41	#include
5043	ashmew2	42	#include
3584	sourcerer	43	#include
		44	#include
		45	#include
		46	#include
		47
		48	#include
		49
		50	#include "utils/config.h"
5043	ashmew2	51	/* #include */
3584	sourcerer	52	#include "content/fetch.h"
		53	#include "content/fetchers/curl.h"
		54	#include "content/urldb.h"
		55	#include "desktop/netsurf.h"
		56	#include "desktop/options.h"
		57	#include "utils/log.h"
		58	#include "utils/messages.h"
		59	#include "utils/schedule.h"
		60	#include "utils/utils.h"
		61	#include "utils/ring.h"
		62	#include "utils/useragent.h"
		63
		64	/* BIG FAT WARNING: This is here because curl doesn't give you an FD to
		65	* poll on, until it has processed a bit of the handle. So we need schedules
		66	* in order to make this work.
		67	*/
		68	#include
		69
5043	ashmew2	70	#include "http_msg.h"
		71	#include "http.h"
		72
		73	/**********************************************************************
		74	******This section added for resolving compile errors*************/
		75	#define CURL_ERROR_SIZE 100
		76	/Definitions for CURL EASY Codes/
		77
		78	#define CURLE_OK 0
		79	#define CURLE_PARTIAL_FILE 18
		80	#define CURLE_WRITE_ERROR 23
		81	#define CURLE_SSL_CONNECT_ERROR 35
		82
		83	/Definitions for CURL MULTI Codes/
		84
		85	#define CURLM_OK 0
		86	#define CURLM_CALL_MULTI_PERFORM -1
		87	#define CURLM_FAILED 123123
		88	/* KOSH stands for KolibriOS HTTP :) */
		89
		90	int FLAG_HTTP11 = 1 << 0;
		91	int FLAG_GOT_HEADER = 1 << 1;
		92	int FLAG_GOT_ALL_DATA = 1 << 2;
		93	int FLAG_CONTENT_LENGTH = 1 << 3;
		94	int FLAG_CHUNKED = 1 << 4;
		95	int FLAG_CONNECTED = 1 << 5;
		96
		97	/* ERROR flags go into the upper word */
		98	int FLAG_INVALID_HEADER = 1 << 16;
		99	int FLAG_NO_RAM = 1 << 17;
		100	int FLAG_SOCKET_ERROR = 1 << 18;
		101	int FLAG_TIMEOUT_ERROR = 1 << 19;
		102	int FLAG_TRANSFER_FAILED = 1 << 20;
		103
		104
		105	struct KOSHcode {
		106	long code;
		107	};
		108
		109	struct KOSHMcode {
		110	long code;
		111	};
		112
		113	struct KOSHMsg {
		114	char *msg;
		115	};
		116
		117	typedef struct KOSHcode KOSHcode;
		118	typedef struct KOSHMcode KOSHMcode;
		119	struct kosh_infotype {
		120	int type;
		121	};
		122	typedef struct kosh_infotype kosh_infotype;
		123
		124	struct curl_slist {
		125	char *data;
		126	struct curl_slist *next;
		127	};
		128
		129	/**********************************************************************/
		130
3584	sourcerer	131	/* uncomment this to use scheduler based calling
		132	#define FETCHER_CURLL_SCHEDULED 1
		133	*/
		134
5043	ashmew2	135	/** SSL certificate info */
		136	/* struct cert_info { */
		137	/* X509 cert; /\< Pointer to certificate \/ */
		138	/* long err; /\*< OpenSSL error code \/ */
		139	/* }; */
3584	sourcerer	140
5043	ashmew2	141	/** Information for a single fetch. */
		142	struct curl_fetch_info {
		143	struct fetch fetch_handle; /< The fetch handle we're parented by. /
		144	struct http_msg * curl_handle; /*< cURL handle if being fetched, or 0. /
		145	bool had_headers; /*< Headers have been processed. /
		146	bool abort; /*< Abort requested. /
		147	bool stopped; /*< Download stopped on purpose. /
		148	bool only_2xx; /*< Only HTTP 2xx responses acceptable. /
		149	bool downgrade_tls; /*< Downgrade to TLS <= 1.0 /
		150	nsurl url; /< URL of this fetch. /
		151	lwc_string host; /< The hostname of this fetch. /
		152	struct curl_slist headers; /< List of request headers. /
		153	char location; /< Response Location header, or 0. /
		154	unsigned long content_length; /*< Response Content-Length, or 0. /
		155	char cookie_string; /< Cookie string for this fetch /
		156	char realm; /< HTTP Auth Realm /
		157	char post_urlenc; /< Url encoded POST string, or 0. /
		158	long http_code; /*< HTTP result code from cURL. /
		159	struct curl_httppost post_multipart; /< Multipart post data, or 0. /
		160	/* #define MAX_CERTS 10 */
		161	/* struct cert_info cert_data[MAX_CERTS]; /\*< HTTPS certificate data \/ */
		162	unsigned int last_progress_update; /*< Time of last progress update /
		163	};
3584	sourcerer	164
5043	ashmew2	165	struct cache_handle {
		166	struct http_msg handle; /< The cached struct http_msg handle /
		167	lwc_string host; /< The host for which this handle is cached /
3584	sourcerer	168
5043	ashmew2	169	struct cache_handle r_prev; /< Previous cached handle in ring. /
		170	struct cache_handle r_next; /< Next cached handle in ring. /
		171	};
3616	sourcerer	172
5043	ashmew2	173	struct fetch_info_slist {
		174	struct curl_fetch_info fetch_info; / this fetch_info contains the same handle as the struct */
		175	struct http_msg *handle;
		176	bool fetch_curl_header_called;
		177	struct fetch_info_slist *next;
3584	sourcerer	178	};
		179
		180
5043	ashmew2	181	struct fetch_info_slist fetch_curl_multi; /< Global cURL multi handle. /
3584	sourcerer	182
5043	ashmew2	183	/** Curl handle with default options set; not used for transfers. */
		184	static struct http_msg *fetch_blank_curl;
		185	static struct cache_handle curl_handle_ring = 0; /< Ring of cached handles /
		186	static int curl_fetchers_registered = 0;
		187	static bool curl_with_openssl;
		188
		189	static char fetch_error_buffer[CURL_ERROR_SIZE]; /*< Error buffer for cURL. /
		190	static char fetch_proxy_userpwd[100]; /*< Proxy authentication details. /
		191
		192	static bool fetch_curl_initialise(lwc_string *scheme);
		193	static void fetch_curl_finalise(lwc_string *scheme);
		194	static bool fetch_curl_can_fetch(const nsurl *url);
3584	sourcerer	195	static void * fetch_curl_setup(struct fetch parent_fetch, nsurl url,
		196	bool only_2xx, bool downgrade_tls, const char *post_urlenc,
		197	const struct fetch_multipart_data *post_multipart,
5043	ashmew2	198	const char **headers);
		199	static bool fetch_curl_start(void *vfetch);
		200	static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch,
		201	struct http_msg *handle);
		202	static struct http_msg fetch_curl_get_handle(lwc_string host);
		203	static void fetch_curl_cache_handle(struct http_msg handle, lwc_string host);
		204	static KOSHcode fetch_curl_set_options(struct curl_fetch_info *f);
		205	/* static KOSHcode fetch_curl_sslctxfun(CURL curl_handle, void _sslctx, */
		206	/* void p); /
		207	static void fetch_curl_abort(void *vf);
		208	static void fetch_curl_stop(struct fetch_info_slist *f);
		209	static void fetch_curl_free(void *f);
		210	static void fetch_curl_poll(lwc_string *scheme_ignored);
		211	static void fetch_curl_done(struct fetch_info_slist *curl_handle);
		212	static int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
		213	double ultotal, double ulnow);
		214	static int fetch_curl_ignore_debug(struct http_msg *handle,
		215	kosh_infotype type,
		216	char *data,
		217	size_t size,
		218	void *userptr);
		219	static size_t fetch_curl_data(void *_f);
		220	/* static size_t fetch_curl_header(char data, size_t size, size_t nmemb, /
		221	/* void _f); /
		222	void fetch_curl_header(void *_f);
		223	static bool fetch_curl_process_headers(struct curl_fetch_info *f);
		224	static struct curl_httppost *fetch_curl_post_convert(
		225	const struct fetch_multipart_data *control);
3584	sourcerer	226
5043	ashmew2	227	/* static int fetch_curl_verify_callback(int preverify_ok, */
		228	/* X509_STORE_CTX x509_ctx); /
		229	/* static int fetch_curl_cert_verify_callback(X509_STORE_CTX x509_ctx, /
		230	/* void parm); /
3584	sourcerer	231
5043	ashmew2	232	/************Functions added for replacing curl's provided functionality **********/
		233	struct curl_slist curl_slist_append(struct curl_slist list, const char * string );
		234	void curl_slist_free_all(struct curl_slist *);
		235	struct fetch_info_slist curl_multi_remove_handle(struct fetch_info_slist multi_handle, struct curl_fetch_info *fetch_to_delete);
		236	struct http_msg * curl_easy_init(void);
		237	void curl_easy_cleanup(struct http_msg *handle);
		238	int curl_multi_add_handle(struct fetch_info_slist *multi_handle, struct curl_fetch_info new_fetch_info);
3616	sourcerer	239
3584	sourcerer	240	/**
		241	* Initialise the fetcher.
		242	*
		243	* Must be called once before any other function.
		244	*/
		245
		246	void fetch_curl_register(void)
		247	{
5043	ashmew2	248	/* KOSHcode code; */
		249	/* curl_version_info_data data; /
		250	int i;
		251	lwc_string *scheme;
		252
		253	LOG(("curl_version (no CURL XD)"));
3584	sourcerer	254
5043	ashmew2	255	const struct fetcher_operation_table fetcher_ops = {
		256	.initialise = fetch_curl_initialise,
		257	.acceptable = fetch_curl_can_fetch,
		258	.setup = fetch_curl_setup,
		259	.start = fetch_curl_start,
		260	.abort = fetch_curl_abort,
		261	.free = fetch_curl_free,
		262	.poll = fetch_curl_poll,
		263	.finalise = fetch_curl_finalise
		264	};
		265
3584	sourcerer	266
5043	ashmew2	267	/* code = curl_global_init(CURL_GLOBAL_ALL); */
		268	/* if (code != CURLE_OK) */
		269	/* die("Failed to initialise the fetch module " */
		270	/* "(curl_global_init failed)."); */
3584	sourcerer	271
5043	ashmew2	272	/TODO : Put the init function for our global queue here /
		273	/* What do we need for an init? Just setting the list to NULL should be enough.
		274	We might track the number of nodes in the list, but that's not required since we usean SLL.
		275	*/
		276	/* fetch_curl_multi = curl_multi_init(); */
		277	/* if (!fetch_curl_multi) */
		278	/* die("Failed to initialise the fetch module " */
		279	/* "(curl_multi_init failed)."); */
3584	sourcerer	280
5043	ashmew2	281	/* Create a curl easy handle with the options that are common to all
		282	fetches. */
		283
		284	/HTTP Library initiated using volatime asm code in http.c /
		285	DBG("Calling curl_easy_init\n");
		286	/* fetch_blank_curl = curl_easy_init(); */
		287
		288	/* if (!fetch_blank_curl) */
		289	/* { */
		290	/* DBG("fetch_blank_curl is NULL"); */
		291	/* die("Failed to initialise the fetch module " */
		292	/* "(curl_easy_init failed)."); */
		293	/* } */
		294	/* else */
		295	/* DBG("fetch_blank_curl is usable."); */
3584	sourcerer	296
5043	ashmew2	297	fetch_blank_curl = NULL;
		298
		299	/* TODO: The SETOPT calls set up the parameters for the curl handle.
		300	Since we don't want to use curl, these are of no use, but our native handle
		301	should consider all these fields while being set up for proper functioning
		302	*/
		303
		304	/* #undef SETOPT */
		305	/* #define SETOPT(option, value) \ */
		306	/* code = curl_easy_setopt(fetch_blank_curl, option, value); \ */
		307	/* if (code != CURLE_OK) \ */
		308	/* goto curl_easy_setopt_failed; */
		309
		310	/* if (verbose_log) { */
		311	/* SETOPT(CURLOPT_VERBOSE, 1); */
		312	/* } else { */
		313	/* SETOPT(CURLOPT_VERBOSE, 0); */
		314	/* } */
		315	/* SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer); */
		316	/* if (nsoption_bool(suppress_curl_debug)) */
		317	/* SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore_debug); */
		318	/* SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data); */
		319	/* SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header); / / Calling fetch_curl_header inside fetch_curl_process_headers */
		320	/* SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress); / / TODO: Add this with httplib somehow */
		321	/* SETOPT(CURLOPT_NOPROGRESS, 0); */
		322	/* SETOPT(CURLOPT_USERAGENT, user_agent_string()); */
		323	/* SETOPT(CURLOPT_ENCODING, "gzip"); */
		324	/* SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L); */
		325	/* SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); */
		326	/* SETOPT(CURLOPT_NOSIGNAL, 1L); */
		327	/* SETOPT(CURLOPT_CONNECTTIMEOUT, 30L); */
		328
		329	/* if (nsoption_charp(ca_bundle) && */
		330	/* strcmp(nsoption_charp(ca_bundle), "")) { */
		331	/* LOG(("ca_bundle: '%s'", nsoption_charp(ca_bundle))); */
		332	/* SETOPT(CURLOPT_CAINFO, nsoption_charp(ca_bundle)); */
		333	/* } */
		334	/* if (nsoption_charp(ca_path) && strcmp(nsoption_charp(ca_path), "")) { */
		335	/* LOG(("ca_path: '%s'", nsoption_charp(ca_path))); */
		336	/* SETOPT(CURLOPT_CAPATH, nsoption_charp(ca_path)); */
		337	/* } */
		338
		339	/TODO: Useless for now, no SSL Support/
		340
		341	/* /\* Detect whether the SSL CTX function API works \/ /
		342	/* curl_with_openssl = true; */
		343	/* code = curl_easy_setopt(fetch_blank_curl, */
		344	/* CURLOPT_SSL_CTX_FUNCTION, NULL); */
		345	/* if (code != CURLE_OK) { */
		346	/* curl_with_openssl = false; */
		347	/* } */
		348
		349	/* /\* LOG(("cURL %slinked against openssl", curl_with_openssl ? "" : "not ")); \/ /
		350
		351	/* /\* cURL initialised okay, register the fetchers \/ /
		352
		353	/* data = curl_version_info(CURLVERSION_NOW); */
		354
		355	/*TODO: We strictly want to deal with only http as a protocol right now, this stuff can come
		356	handy later, so it shall sit here for a while XD
		357	Removing the for loop for a single http fetcher setup scheme.
		358	*/
		359
		360	/* for (i = 0; data->protocols[i]; i++) {
		361	if (strcmp(data->protocols[i], "http") == 0) {
		362	if (lwc_intern_string("http", SLEN("http"),
		363	&scheme) != lwc_error_ok) {
		364	die("Failed to initialise the fetch module "
		365	"(couldn't intern \"http\").");
		366	}
		367
		368	} else if (strcmp(data->protocols[i], "https") == 0) {
		369	if (lwc_intern_string("https", SLEN("https"),
		370	&scheme) != lwc_error_ok) {
		371	die("Failed to initialise the fetch module "
		372	"(couldn't intern \"https\").");
		373	}
		374
		375	} else {
		376	// Ignore non-http(s) protocols
		377	continue;
3584	sourcerer	378	}
		379
5043	ashmew2	380	if (!fetch_add_fetcher(scheme,
		381	fetch_curl_initialise,
3584	sourcerer	382	fetch_curl_can_fetch,
		383	fetch_curl_setup,
		384	fetch_curl_start,
		385	fetch_curl_abort,
5043	ashmew2	386	sfetch_curl_free,
3584	sourcerer	387	#ifdef FETCHER_CURLL_SCHEDULED
		388	NULL,
		389	#else
		390	fetch_curl_poll,
		391	#endif
		392	fetch_curl_finalise)) {
5043	ashmew2	393	LOG(("Unable to register cURL fetcher for %s",
		394	data->protocols[i]));
3584	sourcerer	395	}
5043	ashmew2	396	}
		397	*/
3584	sourcerer	398
5043	ashmew2	399	/* if (lwc_intern_string("http", SLEN("http"), */
		400	/* &scheme) != lwc_error_ok) { */
		401	/* die("Failed to initialise the fetch module " */
		402	/* "(couldn't intern \"http\")."); */
		403	/* } */
		404
		405	/* if (!fetch_add_fetcher(scheme, */
		406	/* fetch_curl_initialise, */
		407	/* fetch_curl_can_fetch, */
		408	/* fetch_curl_setup, */
		409	/* fetch_curl_start, */
		410	/* fetch_curl_abort, */
		411	/* fetch_curl_free, */
		412	/* #ifdef FETCHER_CURLL_SCHEDULED */
		413	/* NULL, */
		414	/* #else */
		415	/* fetch_curl_poll, */
		416	/* #endif */
		417	/* fetch_curl_finalise)) { */
		418	/* LOG(("Unable to register cURL fetcher for %s", */
		419	/* "http")); */
		420	/* } */
		421	/* DBG("fetch_curl_register returning."); */
		422	return;
		423
		424	curl_easy_setopt_failed:
		425	die("Failed to initialise the fetch module "
		426	"(curl_easy_setopt failed).");
3584	sourcerer	427	}
		428
		429
		430	/**
		431	* Initialise a cURL fetcher.
		432	*/
		433
5043	ashmew2	434	/* Seems to not need any work right now, curl_fetchers_registered variable seems handy*/
3584	sourcerer	435	bool fetch_curl_initialise(lwc_string *scheme)
		436	{
5043	ashmew2	437	LOG(("Initialise cURL fetcher for %s", lwc_string_data(scheme)));
		438	curl_fetchers_registered++;
3584	sourcerer	439	return true; /* Always succeeds */
		440	}
		441
		442
		443	/**
5043	ashmew2	444	* Finalise a cURL fetcher
3584	sourcerer	445	*/
		446
		447	void fetch_curl_finalise(lwc_string *scheme)
		448	{
5043	ashmew2	449	struct cache_handle *h;
		450	curl_fetchers_registered--;
		451	LOG(("Finalise cURL fetcher %s", lwc_string_data(scheme)));
		452	if (curl_fetchers_registered == 0) {
		453	/* KOSHMcode codem; */
		454	int codem;
		455	/* All the fetchers have been finalised. */
		456	LOG(("All cURL fetchers finalised, closing down cURL"));
		457
		458	/* TODO: Add any clean up functions for httplib here. */
		459	/* curl_easy_cleanup now contains http_free() */
		460
		461	/* curl_easy_cleanup(fetch_blank_curl); */
		462
		463	/* codem = curl_multi_cleanup(fetch_curl_multi); */
		464	/* if (codem != CURLM_OK) */
		465	/* LOG(("curl_multi_cleanup failed: ignoring")); */
		466
		467	/* curl_global_cleanup(); */
		468	}
		469
		470	/* Free anything remaining in the cached curl handle ring */
		471	while (curl_handle_ring != NULL) {
		472	h = curl_handle_ring;
		473	RING_REMOVE(curl_handle_ring, h);
		474	lwc_string_unref(h->host);
		475	}
3584	sourcerer	476	}
		477
5043	ashmew2	478	bool fetch_curl_can_fetch(const nsurl *url)
3584	sourcerer	479	{
5043	ashmew2	480	return nsurl_has_component(url, NSURL_HOST);
3584	sourcerer	481	}
		482
		483	/**
		484	* Start fetching data for the given URL.
		485	*
		486	* The function returns immediately. The fetch may be queued for later
		487	* processing.
		488	*
		489	* A pointer to an opaque struct curl_fetch_info is returned, which can be
		490	* passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory
		491	* is exhausted (or some other fatal error occurred).
		492	*
		493	* The caller must supply a callback function which is called when anything
		494	* interesting happens. The callback function is first called with msg
		495	* FETCH_HEADER, with the header in data, then one or more times
		496	* with FETCH_DATA with some data for the url, and finally with
		497	* FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred:
		498	* data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER,
		499	* FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL.
		500	*
		501	* Some private data can be passed as the last parameter to fetch_start, and
		502	* callbacks will contain this.
		503	*/
		504
5043	ashmew2	505	void * fetch_curl_setup(struct fetch parent_fetch, nsurl url,
		506	bool only_2xx, bool downgrade_tls, const char *post_urlenc,
3584	sourcerer	507	const struct fetch_multipart_data *post_multipart,
5043	ashmew2	508	const char **headers)
3584	sourcerer	509	{
5043	ashmew2	510	struct curl_fetch_info *fetch;
		511	struct curl_slist *slist;
		512	int i;
		513
		514	fetch = malloc(sizeof (*fetch));
3584	sourcerer	515
5043	ashmew2	516	if (fetch == NULL)
		517	{
		518	LOG(("Fetch was NULL. Aborting fetch_curl_setup"));
		519	return 0;
		520	}
		521	fetch->fetch_handle = parent_fetch;
		522
		523	LOG(("fetch %p, url '%s'", fetch, nsurl_access(url)));
		524
		525	/* construct a new fetch structure */
		526	fetch->curl_handle = NULL;
		527	fetch->had_headers = false;
		528	fetch->abort = false;
		529	fetch->stopped = false;
		530	fetch->only_2xx = only_2xx;
		531	fetch->downgrade_tls = downgrade_tls;
		532	fetch->headers = NULL;
		533	fetch->url = nsurl_ref(url);
		534	fetch->host = nsurl_get_component(url, NSURL_HOST);
		535	fetch->location = NULL;
		536	fetch->content_length = 0;
		537	fetch->http_code = 0;
		538	fetch->cookie_string = NULL;
		539	fetch->realm = NULL;
		540	fetch->post_urlenc = NULL;
		541	fetch->post_multipart = NULL;
		542
		543	if (post_urlenc)
		544	{
		545	LOG(("post_urlenc is not NULL : %s.\n", post_urlenc));
		546	fetch->post_urlenc = strdup(post_urlenc);
		547	}
		548	else if (post_multipart)
		549	{
		550	LOG(("post_multipart is not NULL : %x.\n", post_multipart));
		551	/TODO: Need a post converter here, shouldn't be large though/
		552	fetch->post_multipart = fetch_curl_post_convert(post_multipart);
		553	}
		554	/* memset(fetch->cert_data, 0, sizeof(fetch->cert_data)); */
		555	fetch->last_progress_update = 0;
		556
		557	if (fetch->host == NULL \|\|
		558	(post_multipart != NULL && fetch->post_multipart == NULL) \|\|
		559	(post_urlenc != NULL && fetch->post_urlenc == NULL))
		560	goto failed;
		561
		562	/* TODO : Write an implementation for curl_slist_append for adding and appending fields to header*/
		563	/* This is done for now */
		564
		565	/* #define APPEND(list, value) \ */
		566	/* slist = curl_slist_append(list, value); \ */
		567	/* if (slist == NULL) \ */
		568	/* goto failed; \ */
		569	/* list = slist; */
3616	sourcerer	570
5043	ashmew2	571	/*TODO : This section will need some work because we don't use curl headers but use the ones
		572	provided by http.obj which does not necessarily include the same prefed headers
		573	*/
		574	/* remove curl default headers */
		575	/* APPEND(fetch->headers, "Pragma:"); */
3584	sourcerer	576
5043	ashmew2	577	/* when doing a POST libcurl sends Expect: 100-continue" by default
		578	* which fails with lighttpd, so disable it (see bug 1429054) */
		579	/* APPEND(fetch->headers, "Expect:"); */
3584	sourcerer	580
5043	ashmew2	581	/* if ((nsoption_charp(accept_language) != NULL) && */
		582	/* (nsoption_charp(accept_language)[0] != '\0')) { */
		583	/* char s[80]; */
		584	/* snprintf(s, sizeof s, "Accept-Language: %s, ;q=0.1", /
		585	/* nsoption_charp(accept_language)); */
		586	/* s[sizeof s - 1] = 0; */
		587	/* APPEND(fetch->headers, s); */
		588	/* } */
		589
		590	/* if (nsoption_charp(accept_charset) != NULL && */
		591	/* nsoption_charp(accept_charset)[0] != '\0') { */
		592	/* char s[80]; */
		593	/* snprintf(s, sizeof s, "Accept-Charset: %s, ;q=0.1", /
		594	/* nsoption_charp(accept_charset)); */
		595	/* s[sizeof s - 1] = 0; */
		596	/* APPEND(fetch->headers, s); */
		597	/* } */
		598
		599	/* if (nsoption_bool(do_not_track) == true) { */
		600	/* APPEND(fetch->headers, "DNT: 1"); */
		601	/* } */
		602
		603	/* /\* And add any headers specified by the caller \/ /
		604	/* for (i = 0; headers[i] != NULL; i++) { */
		605	/* APPEND(fetch->headers, headers[i]); */
		606	/* } */
		607
		608	return fetch;
		609
		610	failed:
		611	if (fetch->host != NULL)
		612	lwc_string_unref(fetch->host);
		613
		614	nsurl_unref(fetch->url);
		615	free(fetch->post_urlenc);
		616	/* TOOD: Figure out a way to deal with post data. */
		617	/* if (fetch->post_multipart) */
		618	/* curl_formfree(fetch->post_multipart); */
		619	curl_slist_free_all(fetch->headers);
		620	free(fetch);
		621	return NULL;
		622	}
		623
		624
		625	/**
		626	* Dispatch a single job
		627	*/
		628	bool fetch_curl_start(void *vfetch)
		629	{
		630
		631	struct curl_fetch_info fetch = (struct curl_fetch_info)vfetch;
		632	DBG("Inside fetch_curl_start\n");
		633	return fetch_curl_initiate_fetch(fetch,
		634	fetch_curl_get_handle(fetch->host));
		635	}
		636
		637
		638	/**
		639	* Initiate a fetch from the queue.
		640	*
		641	* Called with a fetch structure and a CURL handle to be used to fetch the
		642	* content.
		643	*
		644	* This will return whether or not the fetch was successfully initiated.
		645	*/
		646
		647	bool fetch_curl_initiate_fetch(struct curl_fetch_info fetch, struct http_msg handle)
		648	{
		649	KOSHcode code;
		650	KOSHMcode codem;
		651	unsigned int wererat; /* Like soUrcerer wanted it :D */
3620	sourcerer	652	char *zz;
		653	int pr;
		654
5043	ashmew2	655	/* fetch->curl_handle = handle; */
		656	/* Don't need to add options to handle from http obj */
		657	/* Initialise the handle */
		658	/* DBG("inside fetch_curl_initiate_fetch()...\n"); */
		659
		660	/* code = fetch_curl_set_options(fetch); */
		661	/* if (code.code != CURLE_OK) { */
		662	/* fetch->curl_handle = 0; */
		663	/* return false; */
		664	/* } */
		665
		666	/* TODO : Write a curl_multi_add_handle alternative which puts the handle in our global queue
		667	for polling later on multiple transfers together*/
		668
		669	/* add to the global curl multi handle */
		670
		671	DBG("inside fetch_curl_initiate_fetch()...\n");
		672
		673	nsurl_get(fetch->url, NSURL_WITH_FRAGMENT, &zz, &pr);
		674
		675	if (zz == NULL) {
		676	fetch_curl_abort(fetch);
		677	return NULL;
		678	}
		679
		680	/TODO : Always clear the flags for the handle here/
		681
		682	if(fetch->post_urlenc)
		683	{
		684	LOG(("http_post on %s with headers: %s", zz, fetch->post_urlenc));
5535	hidnplayr	685	wererat = http_post(zz, NULL, NULL, NULL, "application/x-www-form-urlencoded", strlen(fetch->post_urlenc));
5043	ashmew2	686
		687	if(wererat == 0)
		688	{
		689	LOG(("Error. http_post failed. Aborting fetch.\n"));
		690	fetch_curl_abort(fetch);
3616	sourcerer	691	return NULL;
5043	ashmew2	692	}
		693	else /Send the post request body/
		694	{
		695	int sent = http_send(wererat, fetch->post_urlenc, strlen(fetch->post_urlenc));
		696	LOG(("Sent %d bytes in http_send for %s", sent, fetch->post_urlenc));
		697	}
		698	}
		699	else /* GET Request */
		700	{
		701	LOG(("http_get on URL : %s", zz));
5535	hidnplayr	702	wererat = http_get(zz, NULL, NULL, NULL); /* Initiates the GET on the handle we want to initiate for */
5043	ashmew2	703
		704	if(wererat == 0) /* http_get failed. Something wrong. Can't do anything here */
		705	{
		706	DBG("Error. http_get failed. Aborting fetch.\n");
		707	fetch_curl_abort(fetch);
		708	return NULL;
		709	}
		710	}
		711
		712	/* Probably check for the older curl_handle here and http_free() or http_disconnect accordingly TODO */
		713
		714	fetch->curl_handle = (struct http_msg )wererat; / Adding the http_msg handle to fetch->handle */
		715
		716	LOG(("wererat is %u with flags = %u", wererat, fetch->curl_handle->flags));
		717
		718	codem.code = curl_multi_add_handle(&fetch_curl_multi, fetch);
		719
		720	/* Probably drop the assert and handle this properly, but that's for later */
		721	assert(codem.code == CURLM_OK \|\| codem.code == CURLM_CALL_MULTI_PERFORM);
		722
		723	/* TODO: No idea what this does right now. Shouldn't this be inside an #if macro call? to enable/disable curll scheduling.*/
		724	schedule(1, (schedule_callback_fn)fetch_curl_poll, NULL);
		725
		726	return true;
		727	}
		728
		729
		730	/**
		731	* Find a CURL handle to use to dispatch a job
		732	*/
		733
		734	struct http_msg fetch_curl_get_handle(lwc_string host)
		735	{
		736	struct cache_handle *h;
		737	struct http_msg *ret;
		738
		739	DBG("inside fetch_curl_get_handle()...\n");
		740
		741	RING_FINDBYLWCHOST(curl_handle_ring, h, host);
		742	if (h) {
		743	ret = h->handle;
		744	lwc_string_unref(h->host);
		745	RING_REMOVE(curl_handle_ring, h);
		746	free(h);
		747	} else {
		748	/* ret = curl_easy_duphandle(fetch_blank_curl); */
		749	/* TODO: Verify if this is equivalent to curl_easy_duphandle call above this */
		750	ret = curl_easy_init();
3616	sourcerer	751	}
5043	ashmew2	752
		753	return ret;
		754	}
3584	sourcerer	755
		756
5043	ashmew2	757	/**
		758	* Cache a CURL handle for the provided host (if wanted)
		759	*/
3584	sourcerer	760
5043	ashmew2	761	/TODO : Useful for using a pre existing cached handle for faster lookup/
3584	sourcerer	762
5043	ashmew2	763	void fetch_curl_cache_handle(struct http_msg handle, lwc_string host)
		764	{
		765	struct cache_handle *h = 0;
		766	int c;
3584	sourcerer	767
5043	ashmew2	768	DBG("inside fetch_curl_cache_handle...\n");
		769
		770	RING_FINDBYLWCHOST(curl_handle_ring, h, host);
		771	if (h) {
		772	/TODO: Replace curl_easy_cleanup function for something useful for use with KOSH/
		773	/* Already have a handle cached for this hostname */
		774	curl_easy_cleanup(handle);
		775	return;
		776	}
		777	/* We do not have a handle cached, first up determine if the cache is full */
		778	RING_GETSIZE(struct cache_handle, curl_handle_ring, c);
		779	if (c >= nsoption_int(max_cached_fetch_handles)) {
		780	/* Cache is full, so, we rotate the ring by one and
		781	* replace the oldest handle with this one. We do this
		782	* without freeing/allocating memory (except the
		783	* hostname) and without removing the entry from the
		784	* ring and then re-inserting it, in order to be as
		785	* efficient as we can.
		786	*/
		787	if (curl_handle_ring != NULL) {
		788	h = curl_handle_ring;
		789	curl_handle_ring = h->r_next;
		790	curl_easy_cleanup(h->handle);
		791	h->handle = handle;
		792	lwc_string_unref(h->host);
		793	h->host = lwc_string_ref(host);
		794	} else {
		795	/* Actually, we don't want to cache any handles */
		796	curl_easy_cleanup(handle);
		797	}
		798
		799	return;
		800	}
		801	/* The table isn't full yet, so make a shiny new handle to add to the ring */
		802	h = (struct cache_handle*)malloc(sizeof(struct cache_handle));
		803	h->handle = handle;
		804	h->host = lwc_string_ref(host);
		805	RING_INSERT(curl_handle_ring, h);
3584	sourcerer	806	}
		807
		808
		809	/**
5043	ashmew2	810	* Set options specific for a fetch.
3584	sourcerer	811	*/
5043	ashmew2	812
		813	/*TODO: This function sets up a specific fetch. Need a replacement for SETOPT for setting parameters
		814	in our implementation
		815	*/
		816
		817	KOSHcode
		818	fetch_curl_set_options(struct curl_fetch_info *f)
3584	sourcerer	819	{
5043	ashmew2	820	KOSHcode code;
		821	const char *auth;
		822
		823	DBG("Inside fetch_curl_set_options\n");
		824
		825	/TODO: Replace SETOPT with sane set up of parameters for our handle/
		826
		827	/* #undef SETOPT */
		828	/* #define SETOPT(option, value) { \ */
		829	/* code = curl_easy_setopt(f->curl_handle, option, value); \ */
		830	/* if (code != CURLE_OK) \ */
		831	/* return code; \ */
		832	/* } */
		833
		834	/* SETOPT(CURLOPT_URL, nsurl_access(f->url)); */
		835	/* SETOPT(CURLOPT_PRIVATE, f); */
		836	/* SETOPT(CURLOPT_WRITEDATA, f); */
		837	/* SETOPT(CURLOPT_WRITEHEADER, f); */
		838	/* SETOPT(CURLOPT_PROGRESSDATA, f); */
		839	/* SETOPT(CURLOPT_REFERER, fetch_get_referer_to_send(f->fetch_handle)); */
		840	/* SETOPT(CURLOPT_HTTPHEADER, f->headers); */
		841	/* if (f->post_urlenc) { */
		842	/* SETOPT(CURLOPT_HTTPPOST, NULL); */
		843	/* SETOPT(CURLOPT_HTTPGET, 0L); */
		844	/* SETOPT(CURLOPT_POSTFIELDS, f->post_urlenc); */
		845	/* } else if (f->post_multipart) { */
		846	/* SETOPT(CURLOPT_POSTFIELDS, NULL); */
		847	/* SETOPT(CURLOPT_HTTPGET, 0L); */
		848	/* SETOPT(CURLOPT_HTTPPOST, f->post_multipart); */
		849	/* } else { */
		850	/* SETOPT(CURLOPT_POSTFIELDS, NULL); */
		851	/* SETOPT(CURLOPT_HTTPPOST, NULL); */
		852	/* SETOPT(CURLOPT_HTTPGET, 1L); */
		853	/* } */
		854
		855
		856	/* f->cookie_string = urldb_get_cookie(f->url, true); */
		857
		858
		859	/* if (f->cookie_string) { */
		860	/* SETOPT(CURLOPT_COOKIE, f->cookie_string); */
		861	/* } else { */
		862	/* SETOPT(CURLOPT_COOKIE, NULL); */
		863	/* } */
		864
		865	/* if ((auth = urldb_get_auth_details(f->url, NULL)) != NULL) { */
		866	/* SETOPT(CURLOPT_HTTPAUTH, CURLAUTH_ANY); */
		867	/* SETOPT(CURLOPT_USERPWD, auth); */
		868	/* } else { */
		869	/* SETOPT(CURLOPT_USERPWD, NULL); */
		870	/* } */
		871
		872	/* if (nsoption_bool(http_proxy) && */
		873	/* (nsoption_charp(http_proxy_host) != NULL) && */
		874	/* (strncmp(nsurl_access(f->url), "file:", 5) != 0)) { */
		875	/* SETOPT(CURLOPT_PROXY, nsoption_charp(http_proxy_host)); */
		876	/* SETOPT(CURLOPT_PROXYPORT, (long) nsoption_int(http_proxy_port)); */
		877	/* if (nsoption_int(http_proxy_auth) != OPTION_HTTP_PROXY_AUTH_NONE) { */
		878	/* SETOPT(CURLOPT_PROXYAUTH, */
		879	/* nsoption_int(http_proxy_auth) == */
		880	/* OPTION_HTTP_PROXY_AUTH_BASIC ? */
		881	/* (long) CURLAUTH_BASIC : */
		882	/* (long) CURLAUTH_NTLM); */
		883	/* snprintf(fetch_proxy_userpwd, */
		884	/* sizeof fetch_proxy_userpwd, */
		885	/* "%s:%s", */
		886	/* nsoption_charp(http_proxy_auth_user), */
		887	/* nsoption_charp(http_proxy_auth_pass)); */
		888	/* SETOPT(CURLOPT_PROXYUSERPWD, fetch_proxy_userpwd); */
		889	/* } */
		890	/* } else { */
		891	/* SETOPT(CURLOPT_PROXY, NULL); */
		892	/* } */
		893
		894	/* /\* Disable SSL session ID caching, as some servers can't cope. \/ /
		895	/* SETOPT(CURLOPT_SSL_SESSIONID_CACHE, 0); */
		896
		897	/* if (urldb_get_cert_permissions(f->url)) { */
		898	/* /\* Disable certificate verification \/ /
		899	/* SETOPT(CURLOPT_SSL_VERIFYPEER, 0L); */
		900	/* SETOPT(CURLOPT_SSL_VERIFYHOST, 0L); */
		901	/* if (curl_with_openssl) { */
		902	/* SETOPT(CURLOPT_SSL_CTX_FUNCTION, NULL); */
		903	/* SETOPT(CURLOPT_SSL_CTX_DATA, NULL); */
		904	/* } */
		905	/* } else { */
		906	/* /\* do verification \/ /
		907	/* SETOPT(CURLOPT_SSL_VERIFYPEER, 1L); */
		908	/* SETOPT(CURLOPT_SSL_VERIFYHOST, 2L); */
		909	/* if (curl_with_openssl) { */
		910	/* SETOPT(CURLOPT_SSL_CTX_FUNCTION, fetch_curl_sslctxfun); */
		911	/* SETOPT(CURLOPT_SSL_CTX_DATA, f); */
		912	/* } */
		913	/* } */
		914	code.code = CURLE_OK;
		915	return code;
		916	/* return CURLE_OK; */
3584	sourcerer	917	}
		918
		919
5043	ashmew2	920	/**
		921	* cURL SSL setup callback
		922	*/
		923	/TODO : Commenting this out because of lack of SSL support right now /
3584	sourcerer	924
5043	ashmew2	925	/******************************************************************
		926	KOSHcode
		927	fetch_curl_sslctxfun(CURL curl_handle, void _sslctx, void *parm)
		928	{
		929	struct curl_fetch_info f = (struct curl_fetch_info ) parm;
		930	SSL_CTX *sslctx = _sslctx;
		931	long options = SSL_OP_ALL;
3584	sourcerer	932
5043	ashmew2	933	SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback);
		934	SSL_CTX_set_cert_verify_callback(sslctx, fetch_curl_cert_verify_callback,
		935	parm);
3584	sourcerer	936
5043	ashmew2	937	if (f->downgrade_tls) {
		938	#ifdef SSL_OP_NO_TLSv1_1
		939	// Disable TLS1.1, if the server can't cope with it
		940	options \|= SSL_OP_NO_TLSv1_1;
		941	#endif
		942	}
3584	sourcerer	943
5043	ashmew2	944	#ifdef SSL_OP_NO_TLSv1_2
		945	// Disable TLS1.2, as it causes some servers to stall.
		946	options \|= SSL_OP_NO_TLSv1_2;
		947	#endif
		948
		949	SSL_CTX_set_options(sslctx, options);
		950
		951	return CURLE_OK;
		952	}
		953	******************************************************************/
		954
3584	sourcerer	955	/**
3616	sourcerer	956	* Abort a fetch.
3584	sourcerer	957	*/
5043	ashmew2	958	/* TODO: Seems usable until further action */
3584	sourcerer	959
5043	ashmew2	960	void fetch_curl_abort(void *vf)
3584	sourcerer	961	{
5043	ashmew2	962	struct curl_fetch_info f = (struct curl_fetch_info )vf;
		963	assert(f);
		964	LOG(("fetch %p, url '%s'", f, nsurl_access(f->url)));
3616	sourcerer	965
5043	ashmew2	966	fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, f);
		967
		968	if (f->curl_handle) {
		969	f->abort = true;
		970	} else {
		971	fetch_remove_from_queues(f->fetch_handle);
		972	fetch_free(f->fetch_handle);
		973	}
3584	sourcerer	974	}
		975
		976
		977	/**
5043	ashmew2	978	* Clean up the provided fetch object and free it.
		979	*
		980	* Will prod the queue afterwards to allow pending requests to be initiated.
		981	*/
		982
		983	void fetch_curl_stop(struct fetch_info_slist *node)
		984	{
		985	KOSHMcode codem;
		986
		987	/* TODO: Assert doesn't look like a safe option, but this is probably a fatal condition */
		988
		989	struct curl_fetch_info *f = node->fetch_info;
		990
		991	assert(f);
		992	LOG(("fetch %p, url '%s'", f, nsurl_access(f->url)));
		993
		994	if (f->curl_handle) {
		995	/* remove from curl multi handle */
		996	/TODO: Need a replacement for curl_multi_remove_handle function/
		997	/* LOG(("fetch_curl_multi : %u", fetch_curl_multi)); */
		998	fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, f);
		999	/* assert(codem.code == CURLM_OK); */
		1000	/* Put this curl handle into the cache if wanted. */
		1001	/* TODO: Cache? */
		1002	/* fetch_curl_cache_handle(f->curl_handle, f->host); */
		1003
		1004	if(f && f->curl_handle)
		1005	http_free(f->curl_handle);
		1006
		1007	f->curl_handle = 0;
		1008	}
		1009
		1010	fetch_remove_from_queues(f->fetch_handle);
		1011	LOG(("Returning"));
		1012	}
		1013
		1014
		1015	/**
3616	sourcerer	1016	* Free a fetch structure and associated resources.
3584	sourcerer	1017	*/
5043	ashmew2	1018	/TODO: Except the cert details at the bottom of the function, everything else seems workable/
3584	sourcerer	1019
5043	ashmew2	1020	void fetch_curl_free(void *vf)
3584	sourcerer	1021	{
5043	ashmew2	1022	struct curl_fetch_info f = (struct curl_fetch_info )vf;
		1023	int i;
		1024	DBG("inside fetch_curl_free()..\n");
		1025
		1026	if (f->curl_handle)
		1027	curl_easy_cleanup(f->curl_handle);
		1028
		1029	nsurl_unref(f->url);
		1030	lwc_string_unref(f->host);
		1031	free(f->location);
		1032	free(f->cookie_string);
		1033	free(f->realm);
		1034	if (f->headers)
		1035	curl_slist_free_all(f->headers);
		1036	free(f->post_urlenc);
		1037	/* TODO: Deal with POST data asap */
		1038	/* if (f->post_multipart) */
		1039	/* curl_formfree(f->post_multipart); */
		1040
		1041	/* for (i = 0; i < MAX_CERTS && f->cert_data[i].cert; i++) { */
		1042	/* f->cert_data[i].cert->references--; */
		1043	/* if (f->cert_data[i].cert->references == 0) */
		1044	/* X509_free(f->cert_data[i].cert); */
		1045	/* } */
		1046
		1047	free(f);
3584	sourcerer	1048	}
		1049
5043	ashmew2	1050
		1051	/**
		1052	* Do some work on current fetches.
		1053	*
		1054	* Must be called regularly to make progress on fetches.
		1055	*/
		1056
		1057	/*TODO: This is our slave (from master slave) function that will poll fetches.
		1058	We will maintain our own global ring of handles and let this function poll the entries
		1059	and do some work accordingly. Useful for multiple transfers simultaneously.
		1060	*/
		1061
		1062	void fetch_curl_poll(lwc_string *scheme_ignored)
3616	sourcerer	1063	{
5043	ashmew2	1064	int running, queue;
		1065	KOSHMcode codem;
4821	ashmew2	1066
5043	ashmew2	1067	if(!fetch_curl_multi)
		1068	LOG(("fetch_curl_multi is NULL"));
		1069	else
		1070	{
		1071	struct fetch_info_slist *temp = fetch_curl_multi;
		1072
		1073	curl_multi_perform(fetch_curl_multi);
		1074
		1075	while(temp)
		1076	{
		1077	struct fetch_info_slist *new_temp = temp->next;
3584	sourcerer	1078
5043	ashmew2	1079	/* Check if the headers were received. thanks hidnplayr :P */
		1080	if ((temp->handle->flags & FLAG_GOT_HEADER) && (!temp->fetch_curl_header_called))
		1081	{
		1082	fetch_curl_header(temp->fetch_info);
		1083	temp->fetch_curl_header_called = true;
		1084	}
		1085
		1086	if(temp->handle->flags & FLAG_GOT_ALL_DATA) /* FLAG_GOT_ALL_DATA is set */
		1087	{
		1088	/* DBG(("calling fetch_curl_data")); */
		1089	/* LOG(("content in handle is : %s", temp->handle->content_ptr)); */
		1090	fetch_curl_data(temp->fetch_info);
		1091	fetch_curl_done(temp);
		1092	fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, temp->fetch_info);
		1093	}
		1094
		1095	/Add Error FLAG handle here TODO/
		1096
		1097	temp = new_temp;
		1098	}
		1099	}
		1100	/* TODO: Add more flags here */
		1101
		1102	/*TODO: Handle various conditions here, and set the status code accordinpgly when
		1103	calling fetch_curL_done
		1104	*/
		1105
		1106	/* TODO: Probably decide the condition of the fetch here */
		1107
		1108	/* The whole data recieved is shown by FLAG_GOT_ALL_DATA that is 1 SHL 2, meaning 4. Check for it right here. */
		1109
		1110	/* process curl results */
		1111	/TODO: Needs to be replaced , no idea how to do it right now /
		1112	/* Go through each http_msg handle from http.obj and check if it's done yet or not ,
		1113	using the return value from http_receive. If done, remove it. Else let it stay.
		1114	*/
		1115	/*TODO: This has been commented to figure out linker errors.
		1116	Uncomment this and combine this with the above chunk toget the main process loop
		1117	*/
		1118	/* curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); */
		1119	/* while (curl_msg) { */
		1120	/* switch (curl_msg->msg) { */
		1121	/* case CURLMSG_DONE: */
		1122	/* fetch_curl_done(curl_msg->easy_handle, */
		1123	/* curl_msg->data.result); */
		1124	/* break; */
		1125	/* default: */
		1126	/* break; */
		1127	/* } */
		1128	/* curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); */
		1129	/* } */
		1130
		1131	#ifdef FETCHER_CURLL_SCHEDULED
		1132	if (running != 0) {
		1133	schedule(1, (schedule_callback_fn)fetch_curl_poll, fetch_curl_poll);
		1134	}
		1135	#endif
		1136	/* LOG(("Returning froms fetch_curl_poll\n")); */
3616	sourcerer	1137	}
3584	sourcerer	1138
5043	ashmew2	1139
		1140	/**
		1141	* Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
		1142	*
		1143	* \param curl_handle curl easy handle of fetch
		1144	*/
		1145
		1146	/* TODO: curl_easy_getinfo needs a replacement for getting the status of things around
		1147	SSL stuff needs to go away , as usual.
		1148	*/
		1149	void fetch_curl_done(struct fetch_info_slist *node)
3584	sourcerer	1150	{
3616	sourcerer	1151	fetch_msg msg;
5043	ashmew2	1152	bool finished = false;
		1153	bool error = false;
		1154	bool cert = false;
		1155	bool abort_fetch;
		1156	struct curl_fetch_info *f = node->fetch_info;
		1157	char _hideous_hack = (char ) (void *) &f;
		1158	KOSHcode code;
		1159	int result = CURLE_OK;
3584	sourcerer	1160
5043	ashmew2	1161	/* TODO: Remove this definition and get a better replacement for CURLINFO_PRIVATE */
		1162
		1163	/* struct cert_info certs[MAX_CERTS]; */
		1164	/* memset(certs, 0, sizeof(certs)); */
3584	sourcerer	1165
5043	ashmew2	1166	/* find the structure associated with this fetch */
		1167	/* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */
		1168	/* TODO: Do we really need curl_easy_getinfo? Our library struct provides us with all of this info already */
		1169	/* code.code = curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, _hideous_hack); */
		1170	/* assert(code.code == CURLE_OK); */
3584	sourcerer	1171
5043	ashmew2	1172	abort_fetch = f->abort;
		1173	LOG(("done %s", nsurl_access(f->url)));
4821	ashmew2	1174
5043	ashmew2	1175	if (abort_fetch == false && (result == CURLE_OK \|\|
		1176	(result == CURLE_WRITE_ERROR && f->stopped == false))) {
		1177	/* fetch completed normally or the server fed us a junk gzip
		1178	* stream (usually in the form of garbage at the end of the
		1179	* stream). Curl will have fed us all but the last chunk of
		1180	* decoded data, which is sad as, if we'd received the last
		1181	* chunk, too, we'd be able to render the whole object.
		1182	* As is, we'll just have to accept that the end of the
		1183	* object will be truncated in this case and leave it to
		1184	* the content handlers to cope. */
		1185	if (f->stopped \|\|
		1186	(!f->had_headers &&
		1187	fetch_curl_process_headers(f)))
		1188	; /* redirect with no body or similar */
		1189	else
		1190	finished = true;
		1191	} else if (result == CURLE_PARTIAL_FILE) {
		1192	/* CURLE_PARTIAL_FILE occurs if the received body of a
		1193	* response is smaller than that specified in the
		1194	* Content-Length header. */
		1195	if (!f->had_headers && fetch_curl_process_headers(f))
		1196	; /* redirect with partial body, or similar */
		1197	else {
		1198	finished = true;
		1199	}
		1200	} else if (result == CURLE_WRITE_ERROR && f->stopped) {
		1201	/* CURLE_WRITE_ERROR occurs when fetch_curl_data
		1202	* returns 0, which we use to abort intentionally */
		1203	;
		1204	/* } else if (result == CURLE_SSL_PEER_CERTIFICATE \|\| */
		1205	/* result == CURLE_SSL_CACERT) { */
		1206	/* memcpy(certs, f->cert_data, sizeof(certs)); */
		1207	/* memset(f->cert_data, 0, sizeof(f->cert_data)); */
		1208	/* cert = true; */
		1209	} else {
		1210	LOG(("Unknown cURL response code %d", result));
		1211	error = true;
		1212	}
3584	sourcerer	1213
5043	ashmew2	1214	fetch_curl_stop(node);
		1215
		1216	if (abort_fetch)
		1217	; /* fetch was aborted: no callback */
		1218	else if (finished) {
		1219	msg.type = FETCH_FINISHED;
		1220	__menuet__debug_out("Calling FETCH_FINISHED callback inside fetch_curl_data\n");
		1221	fetch_send_callback(&msg, f->fetch_handle);
		1222	/* } else if (cert) { */
		1223	/* int i; */
		1224	/* BIO mem; /
		1225	/* BUF_MEM buf; /
		1226	/* /\* struct ssl_cert_info ssl_certs[MAX_CERTS]; \/ /
		1227
		1228	/* for (i = 0; i < MAX_CERTS && certs[i].cert; i++) { */
		1229	/* ssl_certs[i].version = */
		1230	/* X509_get_version(certs[i].cert); */
		1231
		1232	/* mem = BIO_new(BIO_s_mem()); */
		1233	/* ASN1_TIME_print(mem, */
		1234	/* X509_get_notBefore(certs[i].cert)); */
		1235	/* BIO_get_mem_ptr(mem, &buf); */
		1236	/* (void) BIO_set_close(mem, BIO_NOCLOSE); */
		1237	/* BIO_free(mem); */
		1238	/* snprintf(ssl_certs[i].not_before, */
		1239	/* min(sizeof ssl_certs[i].not_before, */
		1240	/* (unsigned) buf->length + 1), */
		1241	/* "%s", buf->data); */
		1242	/* BUF_MEM_free(buf); */
		1243
		1244	/* mem = BIO_new(BIO_s_mem()); */
		1245	/* ASN1_TIME_print(mem, */
		1246	/* X509_get_notAfter(certs[i].cert)); */
		1247	/* BIO_get_mem_ptr(mem, &buf); */
		1248	/* (void) BIO_set_close(mem, BIO_NOCLOSE); */
		1249	/* BIO_free(mem); */
		1250	/* snprintf(ssl_certs[i].not_after, */
		1251	/* min(sizeof ssl_certs[i].not_after, */
		1252	/* (unsigned) buf->length + 1), */
		1253	/* "%s", buf->data); */
		1254	/* BUF_MEM_free(buf); */
		1255
		1256	/* ssl_certs[i].sig_type = */
		1257	/* X509_get_signature_type(certs[i].cert); */
		1258	/* ssl_certs[i].serial = */
		1259	/* ASN1_INTEGER_get( */
		1260	/* X509_get_serialNumber(certs[i].cert)); */
		1261	/* mem = BIO_new(BIO_s_mem()); */
		1262	/* X509_NAME_print_ex(mem, */
		1263	/* X509_get_issuer_name(certs[i].cert), */
		1264	/* 0, XN_FLAG_SEP_CPLUS_SPC \| */
		1265	/* XN_FLAG_DN_REV \| XN_FLAG_FN_NONE); */
		1266	/* BIO_get_mem_ptr(mem, &buf); */
		1267	/* (void) BIO_set_close(mem, BIO_NOCLOSE); */
		1268	/* BIO_free(mem); */
		1269	/* snprintf(ssl_certs[i].issuer, */
		1270	/* min(sizeof ssl_certs[i].issuer, */
		1271	/* (unsigned) buf->length + 1), */
		1272	/* "%s", buf->data); */
		1273	/* BUF_MEM_free(buf); */
		1274
		1275	/* mem = BIO_new(BIO_s_mem()); */
		1276	/* X509_NAME_print_ex(mem, */
		1277	/* X509_get_subject_name(certs[i].cert), */
		1278	/* 0, XN_FLAG_SEP_CPLUS_SPC \| */
		1279	/* XN_FLAG_DN_REV \| XN_FLAG_FN_NONE); */
		1280	/* BIO_get_mem_ptr(mem, &buf); */
		1281	/* (void) BIO_set_close(mem, BIO_NOCLOSE); */
		1282	/* BIO_free(mem); */
		1283	/* snprintf(ssl_certs[i].subject, */
		1284	/* min(sizeof ssl_certs[i].subject, */
		1285	/* (unsigned) buf->length + 1), */
		1286	/* "%s", buf->data); */
		1287	/* BUF_MEM_free(buf); */
		1288
		1289	/* ssl_certs[i].cert_type = */
		1290	/* X509_certificate_type(certs[i].cert, */
		1291	/* X509_get_pubkey(certs[i].cert)); */
		1292
		1293	/* /\* and clean up \/ /
		1294	/* certs[i].cert->references--; */
		1295	/* if (certs[i].cert->references == 0) */
		1296	/* X509_free(certs[i].cert); */
		1297	/* } */
		1298
		1299	/* msg.type = FETCH_CERT_ERR; */
		1300	/* msg.data.cert_err.certs = ssl_certs; */
		1301	/* msg.data.cert_err.num_certs = i; */
		1302	/* fetch_send_callback(&msg, f->fetch_handle); */
		1303	} else if (error) {
		1304	if (result != CURLE_SSL_CONNECT_ERROR) {
		1305	msg.type = FETCH_ERROR;
		1306	msg.data.error = fetch_error_buffer;
		1307	} else {
		1308	msg.type = FETCH_SSL_ERR;
		1309	}
		1310	fetch_send_callback(&msg, f->fetch_handle);
		1311	}
		1312
		1313	fetch_free(f->fetch_handle);
		1314	LOG(("Returning"));
3616	sourcerer	1315	}
3584	sourcerer	1316
5043	ashmew2	1317
		1318	/**
		1319	* Callback function for fetch progress.
		1320	*/
		1321
		1322	/* TODO: Useful for showing the fetch's progress. Need to figure out a way to hook this up with http.obj
		1323	More of an interface feature, but it'll be nice to have in a browser.
		1324	*/
		1325
		1326	int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
		1327	double ultotal, double ulnow)
3584	sourcerer	1328	{
5043	ashmew2	1329	static char fetch_progress_buffer[256]; /*< Progress buffer for cURL /
		1330	struct curl_fetch_info f = (struct curl_fetch_info ) clientp;
		1331	unsigned int time_now_cs;
3616	sourcerer	1332	fetch_msg msg;
3584	sourcerer	1333
5043	ashmew2	1334	DBG("inside fetch_curl_progress()..\n");
3584	sourcerer	1335
5043	ashmew2	1336	if (f->abort)
		1337	return 0;
3584	sourcerer	1338
5043	ashmew2	1339	msg.type = FETCH_PROGRESS;
		1340	msg.data.progress = fetch_progress_buffer;
3584	sourcerer	1341
5043	ashmew2	1342	/* Rate limit each fetch's progress notifications to 2 a second */
		1343	#define UPDATES_PER_SECOND 2
		1344	#define UPDATE_DELAY_CS (100 / UPDATES_PER_SECOND)
		1345	time_now_cs = wallclock();
		1346	if (time_now_cs - f->last_progress_update < UPDATE_DELAY_CS)
		1347	return 0;
		1348	f->last_progress_update = time_now_cs;
		1349	#undef UPDATE_DELAY_CS
		1350	#undef UPDATES_PERS_SECOND
3584	sourcerer	1351
5043	ashmew2	1352	if (dltotal > 0) {
		1353	snprintf(fetch_progress_buffer, 255,
		1354	messages_get("Progress"),
		1355	human_friendly_bytesize(dlnow),
		1356	human_friendly_bytesize(dltotal));
		1357	fetch_send_callback(&msg, f->fetch_handle);
		1358	} else {
		1359	snprintf(fetch_progress_buffer, 255,
		1360	messages_get("ProgressU"),
		1361	human_friendly_bytesize(dlnow));
		1362	fetch_send_callback(&msg, f->fetch_handle);
		1363	}
3584	sourcerer	1364
5043	ashmew2	1365	return 0;
		1366	}
3616	sourcerer	1367
5043	ashmew2	1368
		1369	/**
		1370	* Ignore everything given to it.
		1371	*
		1372	* Used to ignore cURL debug.
		1373	*/
		1374
		1375	/TODO: No idea what it does exactly, so let it be like it was/
		1376
		1377	int fetch_curl_ignore_debug(struct http_msg *handle,
		1378	kosh_infotype type,
		1379	char *data,
		1380	size_t size,
		1381	void *userptr)
		1382	{
		1383	return 0;
3584	sourcerer	1384	}
		1385
5043	ashmew2	1386	void send_header_callbacks(char header, unsigned int header_length, struct curl_fetch_info f)
		1387	{
		1388	fetch_msg msg;
		1389	int newline = 0;
		1390	int i;
3584	sourcerer	1391
5043	ashmew2	1392	msg.type = FETCH_HEADER;
		1393
		1394	for(i = 0;i < header_length; i++)
		1395	{
		1396	if(header[i] == '\n')
		1397	{
		1398	msg.data.header_or_data.len = i - newline;
		1399	msg.data.header_or_data.buf = (const uint8_t *) (header + newline);
		1400	/* LOG(("buf inside send_header_cb is : %.s\n", i - newline, header+newline)); /
		1401
		1402	newline = i+1;
		1403	fetch_send_callback(&msg, f->fetch_handle);
		1404	}
		1405	}
		1406	}
		1407
		1408	/**
		1409	* Callback function for cURL.
		1410	*/
		1411	/TODO: Seems okay for now /
		1412
		1413	size_t fetch_curl_data(void *_f)
		1414	{
		1415	struct curl_fetch_info *f = _f;
		1416	char *data = f->curl_handle->content_ptr;
		1417	KOSHcode code;
3616	sourcerer	1418	fetch_msg msg;
3584	sourcerer	1419
5043	ashmew2	1420	DBG("inside fetch_curl_data()..\n");
4224	sourcerer	1421
5043	ashmew2	1422	/* if(f->curl_handle) */
		1423	/* LOG(("curl_handle is not NULL\n")); */
		1424	/* else */
		1425	/* LOG(("curl_handle is NULL\n")); */
4224	sourcerer	1426
5043	ashmew2	1427	LOG(("Will be Setting HTTP Code to : %u\n", f->curl_handle->status));
		1428
		1429	/* ensure we only have to get this information once */
		1430	if (!f->http_code)
		1431	{
		1432	/* TODO: For extracting the http response code of what happened in case we don't already have that.
		1433	http_msg struct should have this info available for query.
4224	sourcerer	1434
5043	ashmew2	1435	code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, */
		1436	/* &f->http_code); */
		1437	__menuet__debug_out("f->http_code is 0\n");
		1438	LOG(("f->http_code was 0\n"));
4821	ashmew2	1439
5043	ashmew2	1440	f->http_code = f->curl_handle->status;
		1441	fetch_set_http_code(f->fetch_handle, f->http_code);
4224	sourcerer	1442
5043	ashmew2	1443	/* assert(code.code == CURLE_OK); */
		1444	}
		1445	else
		1446	{
		1447	f->http_code = f->curl_handle->status;
		1448	fetch_set_http_code(f->fetch_handle, f->http_code);
		1449	}
4224	sourcerer	1450
5043	ashmew2	1451	__menuet__debug_out("fetch_curl_data: ");
		1452
		1453	LOG(("fetch->http_code is : %ld\n", f->http_code));
4224	sourcerer	1454
5043	ashmew2	1455	/* ignore body if this is a 401 reply by skipping it and reset
		1456	the HTTP response code to enable follow up fetches */
4224	sourcerer	1457
5043	ashmew2	1458	if (f->http_code == 401)
		1459	{
		1460	f->http_code = 0;
		1461	return;
		1462	/* return size * nmemb; */
		1463	}
3584	sourcerer	1464
5043	ashmew2	1465	if (f->abort \|\| (!f->had_headers && fetch_curl_process_headers(f))) {
		1466	__menuet__debug_out("Setting f->stopped = true\n");
		1467	f->stopped = true;
		1468	return 0;
		1469	}
3584	sourcerer	1470
5043	ashmew2	1471	/* send data to the caller */
3584	sourcerer	1472
5043	ashmew2	1473	LOG(("Inside fetch_curl_data, http_code is : %li", f->http_code));
3616	sourcerer	1474
5043	ashmew2	1475	if(f->http_code == 200)
		1476	{
		1477	send_header_callbacks(&f->curl_handle->header, f->curl_handle->header_length, f);
		1478	LOG(("Finished sending header callbacks\n"));
3584	sourcerer	1479
5043	ashmew2	1480	if (f->abort) {
		1481	f->stopped = true;
		1482	return 0;
		1483	}
		1484	}
		1485	else
		1486	LOG(("Error, http_code is not 200 but is : %u", f->http_code));
4821	ashmew2	1487
5043	ashmew2	1488	msg.type = FETCH_DATA;
		1489	msg.data.header_or_data.buf = (const uint8_t *) data;
		1490	msg.data.header_or_data.len = (size_t)f->curl_handle->content_received;
		1491	/* LOG(("FETCH_DATA with buf = %s and length = %u", msg.data.header_or_data.buf, msg.data.header_or_data.len)); */
		1492	fetch_send_callback(&msg, f->fetch_handle);
4821	ashmew2	1493
5043	ashmew2	1494	/* __menuet__debug_out("After Calling callback_send_fetch\n in fetch_curl_data"); */
3584	sourcerer	1495
5043	ashmew2	1496	if (f->abort) {
		1497	f->stopped = true;
		1498	return 0;
3616	sourcerer	1499	}
3584	sourcerer	1500
5043	ashmew2	1501	/* __menuet__debug_out("Exiting fetch_curl_data"); */
		1502	/* return size * nmemb; */
		1503	}
3584	sourcerer	1504
5043	ashmew2	1505	/**
		1506	Convertor function for converting a header field to its dedicated buffer
		1507	Also terminates with a NULL character so that the string is safe for further use.
		1508	field_name: Append the field name: to the generated string. Pass NULL for no field name.
		1509	source -> Refers to the original data which needs to be copied into dest.
		1510	dest -> destination. This will be allocated using malloc in the function as size is determined here.1
		1511
		1512	dest uses a double pointer in order to allocate storage for the original pointer and not it's temporary copy.
		1513
		1514	**/
		1515
		1516	void convert_to_asciiz(char field_name, char source, char **dest)
		1517	{
		1518	char *i;
		1519
		1520	if(source == NULL)
		1521	return;
		1522
		1523	if(field_name == NULL)
		1524	{
		1525	for(i = source; !isspace(*i); i++);
		1526
		1527	dest = (char )malloc(i - source + 1); /* Allocate a big enough buffer with +1 for NULL character */
		1528	strncpy(dest, source, i - source); / Copy to buffer */
		1529	(*dest)[i - source] = '\0';
		1530	}
		1531	else
		1532	{
		1533	char *temp;
		1534	for(i = source; !isspace(*i); i++);
		1535
		1536	dest = (char )malloc(i - source + 1 + strlen(field_name) + 2); /* Allocate a big enough buffer with +1 for NULL character */
		1537	strcpy(*dest, field_name);
		1538	temp = *dest + strlen(field_name);
		1539	*temp = ':';
		1540	temp++;
		1541	*temp = ' ';
		1542	temp++;
		1543
		1544	strncpy(temp, source, i - source); /* Copy to buffer */
		1545	temp[i - source] = '\0';
		1546	}
		1547
3584	sourcerer	1548	}
		1549
		1550	/**
5043	ashmew2	1551	* Callback function for headers.
3584	sourcerer	1552	*
5043	ashmew2	1553	* See RFC 2616 4.2.
3584	sourcerer	1554	*/
		1555
5043	ashmew2	1556	/TODO: Seems okay for now /
		1557	/* Called when the headers have been received. A callback should be sent for each header line and not the entire thing at once*/
		1558
		1559	void fetch_curl_header(void _f) / Change type to curl_fetch_infO? TODO*/
3584	sourcerer	1560	{
5043	ashmew2	1561	struct curl_fetch_info *f = _f;
		1562	struct http_msg *handle = f->curl_handle;
		1563	char realm = NULL; /Remove me ? TODO*/
		1564	char *cookie = NULL;
		1565	char *content_length = NULL;
		1566	char *content_type = NULL;
		1567	int realm_start;
		1568	int i;
		1569	fetch_msg msg;
		1570
		1571	/* size = nmemb; / /* ???? */
		1572
		1573	__menuet__debug_out("inside fetch_curl_header()..\n");
		1574
		1575	if (f->abort) {
		1576	f->stopped = true;
		1577	return;
		1578	}
		1579
		1580	f->http_code = handle->status;
		1581	fetch_set_http_code(f->fetch_handle, f->http_code);
		1582
		1583	LOG(("fetch->http_code is : %ld\n", f->http_code));
		1584
		1585	convert_to_asciiz(NULL,http_find_header_field(f->curl_handle, "location"), &f->location);
		1586	convert_to_asciiz("content-length", http_find_header_field(f->curl_handle, "content-length"), &content_length);
		1587	convert_to_asciiz("set-cookie", http_find_header_field(f->curl_handle, "set-cookie"), &cookie);
3616	sourcerer	1588
5043	ashmew2	1589	f->content_length = atol(content_length);
3584	sourcerer	1590
5043	ashmew2	1591	if(cookie)
		1592	fetch_set_cookie(f->fetch_handle, cookie+12);
		1593	return;
		1594	/* if(f->had_headers) */
		1595	/* __menuet__debug_out("curl_fetch_data BEFORE: Had headers is true!\n"); */
		1596	/* else */
		1597	/* __menuet__debug_out("curl_fetch_data BEFORE: Had headers is false!\n"); */
		1598
		1599	/* LOG(("Calling fetch_send_callback from fetch_curl_header")); */
		1600	/* fetch_send_callback(&msg, f->fetch_handle); */
		1601	/* LOG(("AFTER Calling fetch_send_callback from fetch_curl_header")); */
3584	sourcerer	1602
5043	ashmew2	1603	/* if(f->had_headers) */
		1604	/* __menuet__debug_out("curl_fetch_data : Had headers is true!\n"); */
		1605	/* else */
		1606	/* __menuet__debug_out("curl_fetch_data : Had headers is false!\n"); */
3584	sourcerer	1607
5043	ashmew2	1608	/* Remember to use lower case names for header field names for http.obj */
		1609	/* We extract only these fields */
		1610
		1611	/* convert_to_asciiz("content-length", http_find_header_field(f->curl_handle, "content-length"), &content_length); */
		1612	/* convert_to_asciiz("content-type", http_find_header_field(f->curl_handle, "content-type"), &content_type); */
		1613
		1614	/* TODO: Uncomment following line and add more fields if required later */
		1615	/* convert_to_asciiz("www-authenticate", http_find_header_field(f->curl_handle, "www-authenticate"), &realm); */
		1616
		1617	/* LOG(("The &header is : %s", &(f->curl_handle->header))); */
		1618
		1619	/* if(f->location) */
		1620	/* { */
		1621	/* LOG(("Setting data buf to %s with length %d", f->location, strlen(f->location))); */
		1622	/* msg.type = FETCH_HEADER; */
		1623	/* msg.data.header_or_data.buf = (const uint8_t ) f->location; /
		1624	/* msg.data.header_or_data.len = strlen(f->location); */
		1625	/* fetch_send_callback(&msg, f->fetch_handle); */
		1626	/* } */
		1627
		1628	/* if(content_type) */
		1629	/* { */
		1630	/* LOG(("Setting data buf to %s with length %d", content_type, strlen(content_type))); */
		1631	/* f->content_length = atoi(content_type); */
		1632	/* msg.type = FETCH_HEADER; */
		1633	/* msg.data.header_or_data.buf = (const uint8_t ) content_type; /
		1634	/* msg.data.header_or_data.len = strlen(content_type); */
		1635	/* fetch_send_callback(&msg, f->fetch_handle); */
		1636	/* } */
		1637
		1638	/* if(content_length) */
		1639	/* { */
		1640	/* f->content_length = atoi(content_length); */
		1641	/* msg.type = FETCH_HEADER; */
		1642	/* msg.data.header_or_data.buf = (const uint8_t ) content_length; /
		1643	/* msg.data.header_or_data.len = strlen(content_length); */
		1644	/* fetch_send_callback(&msg, f->fetch_handle); */
		1645	/* } */
		1646
		1647	/* Set appropriate fetch properties */
		1648	/* if(cookie) */
		1649	/* { */
		1650	/* fetch_set_cookie(f->fetch_handle, cookie); */
		1651	/* msg.type = FETCH_HEADER; */
		1652	/* msg.data.header_or_data.buf = (const uint8_t ) cookie; /
		1653	/* msg.data.header_or_data.len = strlen(cookie); */
		1654	/* fetch_send_callback(&msg, f->fetch_handle); */
		1655	/* } */
		1656
		1657	/* if(realm) /\* Don't worry about this for now , fix it later TODO \/ /
		1658	/* { */
		1659	/* /\* For getting the realm, this was used as an example : ('WWW-Authenticate: Basic realm="My Realm"') \/ /
		1660
		1661	/* for(i = strlen("www-authenticate: "); realm[i]; i++) */
		1662	/* if(realm[i] == '"') { */
		1663	/* realm_start = i+1; */
		1664	/* break; */
		1665	/* } */
		1666
		1667	/* for(i = realm_start ; realm[i]; i++) */
		1668	/* if(realm[i] == '"') { */
		1669	/* realm[i] = '\0'; */
		1670	/* break; */
		1671	/* } */
		1672
		1673	/* f->realm = realm; */
		1674	/* } */
		1675
		1676	/* TODO: call the fetch_callback for www authenticate field and any other fields that will be added here later */
		1677
		1678	/* LOG(("fetch->http_code is ( AT THE END of f_c_header): %ld\n", f->http_code)); */
		1679	/* __menuet__debug_out("Leaving fetch_curl_header\n"); */
		1680
		1681	/* if(f->had_headers) */
		1682	/* __menuet__debug_out("curl_fetch_data : Had headers is true!\n"); */
		1683	/* else */
		1684	/* __menuet__debug_out("curl_fetch_data : Had headers is false!\n"); */
		1685
		1686	/* f->http_code = handle->status; */
		1687	/* fetch_set_http_code(f->fetch_handle, f->http_code); */
		1688
		1689	/* #define SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' \|\| data[i] == '\t'); i++) */
		1690
		1691	/* if (12 < size && strncasecmp(data, "Location:", 9) == 0) { */
		1692	/* /\* extract Location header \/ /
		1693	/* free(f->location); */
		1694	/* f->location = malloc(size); */
		1695	/* if (!f->location) { */
		1696	/* LOG(("malloc failed")); */
		1697	/* return size; */
		1698	/* } */
		1699	/* SKIP_ST(9); */
		1700	/* strncpy(f->location, data + i, size - i); */
		1701	/* f->location[size - i] = '\0'; */
		1702	/* for (i = size - i - 1; i >= 0 && */
		1703	/* (f->location[i] == ' ' \|\| */
		1704	/* f->location[i] == '\t' \|\| */
		1705	/* f->location[i] == '\r' \|\| */
		1706	/* f->location[i] == '\n'); i--) */
		1707	/* f->location[i] = '\0'; */
		1708	/* } else if (15 < size && strncasecmp(data, "Content-Length:", 15) == 0) { */
		1709	/* /\* extract Content-Length header \/ /
		1710	/* SKIP_ST(15); */
		1711	/* if (i < (int)size && '0' <= data[i] && data[i] <= '9') */
		1712	/* f->content_length = atol(data + i); */
		1713	/* } else if (17 < size && strncasecmp(data, "WWW-Authenticate:", 17) == 0) { */
		1714	/* /\* extract the first Realm from WWW-Authenticate header \/ /
		1715	/* SKIP_ST(17); */
		1716
		1717	/* while (i < (int) size - 5 && */
		1718	/* strncasecmp(data + i, "realm", 5)) */
		1719	/* i++; */
		1720	/* while (i < (int) size - 1 && data[++i] != '"') */
		1721	/* /\* \/; /
		1722	/* i++; */
		1723
		1724	/* if (i < (int) size) { */
		1725	/* size_t end = i; */
		1726
		1727	/* while (end < size && data[end] != '"') */
		1728	/* ++end; */
		1729
		1730	/* if (end < size) { */
		1731	/* free(f->realm); */
		1732	/* f->realm = malloc(end - i + 1); */
		1733	/* if (f->realm != NULL) { */
		1734	/* strncpy(f->realm, data + i, end - i); */
		1735	/* f->realm[end - i] = '\0'; */
		1736	/* } */
		1737	/* } */
		1738	/* } */
		1739	/* } else if (11 < size && strncasecmp(data, "Set-Cookie:", 11) == 0) { */
		1740	/* /\* extract Set-Cookie header \/ /
		1741	/* SKIP_ST(11); */
		1742
		1743	/* fetch_set_cookie(f->fetch_handle, &data[i]); */
		1744	/* } */
		1745
		1746	/* return size; */
		1747	/* #undef SKIP_ST */
		1748	}
		1749
		1750	/**
		1751	* Find the status code and content type and inform the caller.
		1752	*
		1753	* Return true if the fetch is being aborted.
		1754	*/
		1755	/TODO: Handling the http status codes here and performing accordingly/
		1756
		1757	bool fetch_curl_process_headers(struct curl_fetch_info *f)
		1758	{
		1759	long http_code;
		1760	KOSHcode code;
		1761	fetch_msg msg;
		1762
		1763	__menuet__debug_out("Setting had_headers to true\n");
		1764
		1765	f->had_headers = true;
		1766
		1767	http_code = f->curl_handle->status;
		1768	LOG(("Inside fetch_curl_process_headers..HTTP CODE : %ld\n", http_code));
		1769
		1770	if (!f->http_code)
		1771	{
		1772	/* TODO: Handle this like another similar piece of code in the file with HTTP_CODE_CURLINFO */
		1773	/* code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, */
		1774	/* &f->http_code); */
		1775	/* Replaced with this : */
		1776	/* Have a fetch_set_http_code here? TODO*/
		1777
		1778	f->http_code = http_code;
		1779	fetch_set_http_code(f->fetch_handle, f->http_code);
		1780	/* assert(code.code == CURLE_OK); */
		1781	}
		1782
		1783	LOG(("HTTP status code %li\n", http_code));
		1784
		1785	if (http_code == 304 && !f->post_urlenc && !f->post_multipart) {
		1786	/* Not Modified && GET request */
		1787	msg.type = FETCH_NOTMODIFIED;
		1788	fetch_send_callback(&msg, f->fetch_handle);
		1789	return true;
		1790	}
		1791
		1792	/* handle HTTP redirects (3xx response codes) */
		1793	if (300 <= http_code && http_code < 400) {
		1794	LOG(("FETCH_REDIRECT, '%s'", f->location));
		1795	msg.type = FETCH_REDIRECT;
		1796	msg.data.redirect = f->location;
		1797	fetch_send_callback(&msg, f->fetch_handle);
		1798	return true;
		1799	}
		1800
		1801	/* handle HTTP 401 (Authentication errors) */
		1802	if (http_code == 401) {
		1803	msg.type = FETCH_AUTH;
		1804	msg.data.auth.realm = f->realm;
		1805	fetch_send_callback(&msg, f->fetch_handle);
		1806	return true;
		1807	}
		1808
		1809	/* handle HTTP errors (non 2xx response codes) */
		1810	if (f->only_2xx && strncmp(nsurl_access(f->url), "http", 4) == 0 &&
		1811	(http_code < 200 \|\| 299 < http_code)) {
		1812	msg.type = FETCH_ERROR;
		1813	DBG("FETCH_ERROR\n");
		1814	msg.data.error = messages_get("Not2xx");
		1815	fetch_send_callback(&msg, f->fetch_handle);
		1816	return true;
		1817	}
		1818
		1819	if (f->abort)
		1820	return true;
		1821
		1822	DBG("Returning false from fetch_curl_process_headers()\n");
		1823
		1824	return false;
		1825	}
		1826
		1827
		1828	/**
		1829	* Convert a list of struct ::fetch_multipart_data to a list of
		1830	* struct curl_httppost for libcurl.
		1831	*/
		1832
		1833	/* TODO: Not sure how to handle multipart data yet, but hopefully it'll be figured out soon */
		1834	/* TODO: Seems like the forms that are being created use sequential fields, so we can probably craft the same */
		1835	/* using post from http,obj */
		1836
		1837	struct curl_httppost *
		1838	fetch_curl_post_convert(const struct fetch_multipart_data *control)
		1839	{
		1840	struct curl_httppost post = 0, last = 0;
		1841	/* TODO: CURLFORMcode code; */
		1842
		1843	DBG("inside fetch_curl_post_convert()..\n");
		1844
		1845	for (; control; control = control->next) {
		1846	if (control->file) {
		1847	char *leafname = 0;
		1848
		1849	leafname = filename_from_path(control->value);
		1850
		1851	if (leafname == NULL)
		1852	continue;
		1853
		1854	/* We have to special case filenames of "", so curl
		1855	* a) actually attempts the fetch and
		1856	* b) doesn't attempt to open the file ""
		1857	*/
		1858	if (control->value[0] == '\0') {
		1859	/* dummy buffer - needs to be static so
		1860	* pointer's still valid when we go out
		1861	* of scope (not that libcurl should be
		1862	* attempting to access it, of course). */
		1863	/* static char buf; */
		1864	/* code = curl_formadd(&post, &last, */
		1865	/* CURLFORM_COPYNAME, control->name, */
		1866	/* CURLFORM_BUFFER, control->value, */
		1867	/* /\* needed, as basename("") == "." \/ /
		1868	/* CURLFORM_FILENAME, "", */
		1869	/* CURLFORM_BUFFERPTR, &buf, */
		1870	/* CURLFORM_BUFFERLENGTH, 0, */
		1871	/* CURLFORM_CONTENTTYPE, */
		1872	/* "application/octet-stream", */
		1873	/* CURLFORM_END); */
		1874	/* if (code != CURL_FORMADD_OK) */
		1875	/* LOG(("curl_formadd: %d (%s)", */
		1876	/* code, control->name)); */
		1877	} else {
		1878	/* char mimetype = fetch_mimetype(control->value); /
		1879	/* code = curl_formadd(&post, &last, */
		1880	/* CURLFORM_COPYNAME, control->name, */
		1881	/* CURLFORM_FILE, control->value, */
		1882	/* CURLFORM_FILENAME, leafname, */
		1883	/* CURLFORM_CONTENTTYPE, */
		1884	/* (mimetype != 0 ? mimetype : "text/plain"), */
		1885	/* CURLFORM_END); */
		1886	/* if (code != CURL_FORMADD_OK) */
		1887	/* LOG(("curl_formadd: %d (%s=%s)", */
		1888	/* code, control->name, */
		1889	/* control->value)); */
		1890	/* free(mimetype); */
		1891	}
		1892	free(leafname);
3616	sourcerer	1893	}
5043	ashmew2	1894	else {
		1895	/* code = curl_formadd(&post, &last, */
		1896	/* CURLFORM_COPYNAME, control->name, */
		1897	/* CURLFORM_COPYCONTENTS, control->value, */
		1898	/* CURLFORM_END); */
		1899	/* if (code != CURL_FORMADD_OK) */
		1900	/* LOG(("curl_formadd: %d (%s=%s)", code, */
		1901	/* control->name, */
		1902	/* control->value)); */
		1903	}
		1904	}
3584	sourcerer	1905
5043	ashmew2	1906	return post;
		1907	}
3584	sourcerer	1908
3616	sourcerer	1909
5043	ashmew2	1910	/**
		1911	* OpenSSL Certificate verification callback
		1912	* Stores certificate details in fetch struct.
		1913	*/
3616	sourcerer	1914
5043	ashmew2	1915	/* TODO: SSL Stuff, useless as of now */
		1916
		1917	/* int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX x509_ctx) /
		1918	/* { */
		1919	/* X509 cert = X509_STORE_CTX_get_current_cert(x509_ctx); /
		1920	/* int depth = X509_STORE_CTX_get_error_depth(x509_ctx); */
		1921	/* int err = X509_STORE_CTX_get_error(x509_ctx); */
		1922	/* struct curl_fetch_info f = X509_STORE_CTX_get_app_data(x509_ctx); /
		1923
		1924	/* /\* save the certificate by incrementing the reference count and */
		1925	/* * keeping a pointer \/ /
		1926	/* if (depth < MAX_CERTS && !f->cert_data[depth].cert) { */
		1927	/* f->cert_data[depth].cert = cert; */
		1928	/* f->cert_data[depth].err = err; */
		1929	/* cert->references++; */
		1930	/* } */
		1931
		1932	/* return preverify_ok; */
		1933	/* } */
		1934
		1935
		1936	/**
		1937	* OpenSSL certificate chain verification callback
		1938	* Verifies certificate chain, setting up context for fetch_curl_verify_callback
		1939	*/
		1940
		1941	/* int fetch_curl_cert_verify_callback(X509_STORE_CTX x509_ctx, void parm) */
		1942	/* { */
		1943	/* int ok; */
		1944
		1945	/* /\* Store fetch struct in context for verify callback \/ /
		1946	/* ok = X509_STORE_CTX_set_app_data(x509_ctx, parm); */
		1947
		1948	/* /\* and verify the certificate chain \/ /
		1949	/* if (ok) */
		1950	/* ok = X509_verify_cert(x509_ctx); */
		1951
		1952	/* return ok; */
		1953	/* } */
		1954
		1955	struct curl_slist curl_slist_append(struct curl_slist list, const char * string )
		1956	{
		1957	struct curl_slist *newnode = NULL;
		1958	DBG("Inside curl_slist_append..\n");
		1959	newnode = malloc(sizeof(struct curl_slist));
		1960
		1961	if(newnode == NULL)
		1962	return NULL;
		1963
		1964	strcpy(newnode->data, string);
		1965
		1966	newnode->next = NULL;
		1967
		1968	if(!list)
		1969	{
		1970	list = newnode;
		1971	}
		1972	else /list isn't null/
		1973	{
		1974	struct curl_slist *temp = list;
		1975
		1976	while(temp->next!=NULL)
		1977	temp = temp->next;
		1978
		1979	temp->next = newnode;
		1980	}
		1981
		1982	return list;
		1983	}
		1984
		1985	void curl_slist_free_all(struct curl_slist *list)
		1986	{
		1987	struct curl_slist *temp = list;
		1988	DBG("Inside curl_slist_free_all..\n");
		1989
		1990	while(list)
		1991	{
		1992	temp = list->next;
		1993	free(list);
		1994	list = temp;
		1995	}
3584	sourcerer	1996	}
		1997
5043	ashmew2	1998	int curl_multi_add_handle(struct fetch_info_slist *multi_handle, struct curl_fetch_info new_fetch)
		1999	{
		2000	DBG("Inside curl_multi_add_handle..Adding handle\n");
3584	sourcerer	2001
5043	ashmew2	2002	if(*multi_handle == NULL)
		2003	{
		2004	struct fetch_info_slist new_node = (struct fetch_info_slist )malloc(sizeof(struct fetch_info_slist));
3584	sourcerer	2005
5043	ashmew2	2006	if(new_node == NULL \|\| new_fetch == NULL) //add failture for malloc here TODO
		2007	return CURLM_FAILED;
		2008
		2009	new_node->fetch_info = new_fetch;
		2010	new_node->handle = new_fetch->curl_handle;
		2011	new_node->fetch_curl_header_called = false;
		2012	*multi_handle = new_node;
		2013	(*multi_handle)->next = NULL;
		2014	}
		2015	else
		2016	{
		2017	struct fetch_info_slist temp = multi_handle;
		2018	struct fetch_info_slist new_node = (struct fetch_info_slist )malloc(sizeof(struct fetch_info_slist));
		2019
		2020	if(new_node == NULL \|\| new_fetch == NULL) //add failture for malloc here TODO
		2021	return CURLM_FAILED;
3584	sourcerer	2022
5043	ashmew2	2023	while(temp->next)
		2024	{
		2025	temp = temp->next;
		2026	}
		2027
		2028	new_node->fetch_info = new_fetch;
		2029	new_node->handle = new_fetch->curl_handle;
		2030	new_node->next = NULL;
		2031	new_node->fetch_curl_header_called = false;
		2032	temp->next = new_node;
		2033	}
		2034
		2035	return CURLM_OK;
		2036	}
		2037
		2038	/* When this function returns, it is assured that the multi list does not contain the node to be deleted.
		2039	If it was, it was deleted. Else, the list is left unchanged
		2040	*/
		2041
		2042	/* This can be sped up a lot by using hash tables or the like for removal to be more speedy :)
		2043	LTG
		2044	*/
		2045
		2046	/* struct fetch_info_slist curl_multi_remove_handle(struct fetch_info_slist multi_handle, struct fetch_info_slist node_to_delete) /
		2047	struct fetch_info_slist curl_multi_remove_handle(struct fetch_info_slist multi_handle, struct curl_fetch_info *fetch_to_delete)
		2048	{
		2049	struct fetch_info_slist *temp = multi_handle;
		2050	char *zz;
		2051	int pr;
		2052
		2053	nsurl_get(fetch_to_delete->url, NSURL_WITH_FRAGMENT, &zz, &pr);
		2054	LOG(("inside curl_multi_remove_handle for %s..\n", zz));
		2055
		2056	if(multi_handle == NULL \|\| fetch_to_delete == NULL)
		2057	return multi_handle;
		2058
		2059	if(temp->fetch_info == fetch_to_delete) /* special case for first node deletion */
		2060	{
		2061	multi_handle = multi_handle->next;
		2062	LOG(("Removed handle\n"));
		2063	/* free(temp); / / Probably shouldnt free. Let other routines free this TODO */
		2064	}
		2065	else /* If the data is present in any consecutive node */
		2066	{
		2067	struct fetch_info_slist *temp2 = multi_handle->next;
		2068
		2069	while(temp2)
		2070	{
		2071	if(temp2->fetch_info == fetch_to_delete)
		2072	{
		2073	temp->next = temp2->next;
		2074	/* free(temp2); / / Shouldnt free node here. Let others handle it (for cache etc). */
		2075	LOG(("Removed handle\n"));
		2076	break;
		2077	}
		2078	else
		2079	{
		2080	temp = temp2;
		2081	temp2 = temp2->next;
		2082	}
		2083	}
		2084	}
		2085	LOG(("Returning"));
		2086	/TODO : http_free should be called here?/
		2087	return multi_handle;
		2088	}
		2089
		2090	/* TODO: Get rid of the curl functions soon DONE*/
		2091
		2092	/* TODO: Actually a function to return a blank handle. The name is misleading right now */
		2093	struct http_msg * curl_easy_init(void)
		2094	{
		2095	struct http_msg *new_handle;
		2096	return new_handle;
		2097	}
		2098
		2099	int curl_multi_perform(struct fetch_info_slist *multi_list)
		2100	{
		2101	struct fetch_info_slist *temp = multi_list;
		2102
		2103	while(temp) {
		2104	http_receive(temp->handle);
		2105	temp = temp->next;
		2106	}
		2107	}
		2108
		2109	void curl_easy_cleanup(struct http_msg *handle)
		2110	{
		2111	http_free(handle);
		2112	}

Subversion Repositories Kolibri OS

(root)/contrib/network/netsurf/netsurf/content/fetchers/curl.c – Rev 5535