0,0 → 1,509 |
/* |
* Copyright 2006 Daniel Silverstone <dsilvers@digital-scurf.org> |
* Copyright 2007 James Bursa <bursa@users.sourceforge.net> |
* Copyright 2003 Phil Mellor <monkeyson@users.sourceforge.net> |
* |
* This file is part of NetSurf. |
* |
* NetSurf is free software; you can redistribute it and/or modify |
* it under the terms of the GNU General Public License as published by |
* the Free Software Foundation; version 2 of the License. |
* |
* NetSurf is distributed in the hope that it will be useful, |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
* GNU General Public License for more details. |
* |
* You should have received a copy of the GNU General Public License |
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
*/ |
|
/** \file |
* Fetching of data from a URL (implementation). |
* |
* This implementation uses libcurl's 'multi' interface. |
* |
* |
* The CURL handles are cached in the curl_handle_ring. There are at most |
* ::max_cached_fetch_handles in this ring. |
*/ |
|
#include <assert.h> |
#include <errno.h> |
#include <inttypes.h> |
#include <stdbool.h> |
#include <string.h> |
#include <strings.h> |
#include <time.h> |
#include <sys/stat.h> |
|
#include <libwapcaplet/libwapcaplet.h> |
|
#include "utils/config.h" |
//#include <openssl/ssl.h> |
#include "content/fetch.h" |
#include "content/fetchers/curl.h" |
#include "content/urldb.h" |
#include "desktop/netsurf.h" |
#include "desktop/options.h" |
#include "utils/log.h" |
#include "utils/messages.h" |
#include "utils/schedule.h" |
#include "utils/utils.h" |
#include "utils/ring.h" |
#include "utils/useragent.h" |
|
/* BIG FAT WARNING: This is here because curl doesn't give you an FD to |
* poll on, until it has processed a bit of the handle. So we need schedules |
* in order to make this work. |
*/ |
#include <desktop/browser.h> |
|
/* uncomment this to use scheduler based calling |
#define FETCHER_CURLL_SCHEDULED 1 |
*/ |
|
typedef int X509_STORE_CTX; |
typedef int X509; |
#define CURL_ERROR_SIZE 0 |
typedef int CURLcode; |
typedef int curl_infotype; |
/** SSL certificate info */ |
struct cert_info { |
X509 *cert; /**< Pointer to certificate */ |
long err; /**< OpenSSL error code */ |
}; |
|
/** Information for a single fetch. */ |
struct curl_fetch_info { |
struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ |
CURL * curl_handle; /**< cURL handle if being fetched, or 0. */ |
bool had_headers; /**< Headers have been processed. */ |
bool abort; /**< Abort requested. */ |
bool stopped; /**< Download stopped on purpose. */ |
bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ |
bool downgrade_tls; /**< Downgrade to TLS <= 1.0 */ |
nsurl *url; /**< URL of this fetch. */ |
lwc_string *host; /**< The hostname of this fetch. */ |
struct curl_slist *headers; /**< List of request headers. */ |
char *location; /**< Response Location header, or 0. */ |
unsigned long content_length; /**< Response Content-Length, or 0. */ |
char *cookie_string; /**< Cookie string for this fetch */ |
char *realm; /**< HTTP Auth Realm */ |
char *post_urlenc; /**< Url encoded POST string, or 0. */ |
long http_code; /**< HTTP result code from cURL. */ |
struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */ |
#define MAX_CERTS 10 |
struct cert_info cert_data[MAX_CERTS]; /**< HTTPS certificate data */ |
unsigned int last_progress_update; /**< Time of last progress update */ |
}; |
|
struct cache_handle { |
CURL *handle; /**< The cached cURL handle */ |
lwc_string *host; /**< The host for which this handle is cached */ |
|
struct cache_handle *r_prev; /**< Previous cached handle in ring. */ |
struct cache_handle *r_next; /**< Next cached handle in ring. */ |
}; |
|
CURLM *fetch_curl_multi; /**< Global cURL multi handle. */ |
/** Curl handle with default options set; not used for transfers. */ |
static CURL *fetch_blank_curl; |
static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */ |
static int curl_fetchers_registered = 0; |
static bool curl_with_openssl; |
|
static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */ |
static char fetch_proxy_userpwd[100]; /**< Proxy authentication details. */ |
|
static bool fetch_curl_initialise(lwc_string *scheme); |
static void fetch_curl_finalise(lwc_string *scheme); |
static bool fetch_curl_can_fetch(const nsurl *url); |
static void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, |
bool only_2xx, bool downgrade_tls, const char *post_urlenc, |
const struct fetch_multipart_data *post_multipart, |
const char **headers); |
static bool fetch_curl_start(void *vfetch); |
static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, |
CURL *handle); |
static CURL *fetch_curl_get_handle(lwc_string *host); |
static void fetch_curl_cache_handle(CURL *handle, lwc_string *host); |
static CURLcode fetch_curl_set_options(struct curl_fetch_info *f); |
static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, |
void *p); |
static void fetch_curl_abort(void *vf); |
static void fetch_curl_stop(struct curl_fetch_info *f); |
static void fetch_curl_free(void *f); |
static void fetch_curl_poll(lwc_string *scheme_ignored); |
static void fetch_curl_done(CURL *curl_handle, CURLcode result); |
static int fetch_curl_progress(void *clientp, double dltotal, double dlnow, |
double ultotal, double ulnow); |
static int fetch_curl_ignore_debug(CURL *handle, |
curl_infotype type, |
char *data, |
size_t size, |
void *userptr); |
static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, |
void *_f); |
static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, |
void *_f); |
static bool fetch_curl_process_headers(struct curl_fetch_info *f); |
static struct curl_httppost *fetch_curl_post_convert( |
const struct fetch_multipart_data *control); |
static int fetch_curl_verify_callback(int preverify_ok, |
X509_STORE_CTX *x509_ctx); |
static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, |
void *parm); |
|
|
/** |
* Initialise the fetcher. |
* |
* Must be called once before any other function. |
*/ |
|
void fetch_curl_register(void) |
{ |
|
lwc_string *scheme; |
|
|
LOG(("curl register\n")); |
|
lwc_intern_string("http", SLEN("http"), &scheme); |
|
if (!fetch_add_fetcher(scheme, |
fetch_curl_initialise, |
fetch_curl_can_fetch, |
fetch_curl_setup, |
fetch_curl_start, |
fetch_curl_abort, |
fetch_curl_free, |
#ifdef FETCHER_CURLL_SCHEDULED |
NULL, |
#else |
fetch_curl_poll, |
#endif |
fetch_curl_finalise)) { |
LOG(("Unable to register cURL fetcher for HTTP")); |
} |
|
|
lwc_intern_string("https", SLEN("https"), &scheme); |
|
if (!fetch_add_fetcher(scheme, |
fetch_curl_initialise, |
fetch_curl_can_fetch, |
fetch_curl_setup, |
fetch_curl_start, |
fetch_curl_abort, |
fetch_curl_free, |
#ifdef FETCHER_CURLL_SCHEDULED |
NULL, |
#else |
fetch_curl_poll, |
#endif |
fetch_curl_finalise)) { |
LOG(("Unable to register cURL fetcher for HTTPS")); |
} |
|
} |
|
|
/** |
* Initialise a cURL fetcher. |
*/ |
|
bool fetch_curl_initialise(lwc_string *scheme) |
{ |
|
LOG(("curl initi lwc\n")); |
return true; /* Always succeeds */ |
} |
|
|
/** |
* Finalise a cURL fetcher |
*/ |
|
void fetch_curl_finalise(lwc_string *scheme) |
{ |
LOG(("curl finali\n")); |
} |
|
bool fetch_curl_can_fetch(const nsurl *url) |
{ |
LOG(("curl can fetch\n")); |
return false; |
} |
|
/** |
* Start fetching data for the given URL. |
* |
* The function returns immediately. The fetch may be queued for later |
* processing. |
* |
* A pointer to an opaque struct curl_fetch_info is returned, which can be |
* passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory |
* is exhausted (or some other fatal error occurred). |
* |
* The caller must supply a callback function which is called when anything |
* interesting happens. The callback function is first called with msg |
* FETCH_HEADER, with the header in data, then one or more times |
* with FETCH_DATA with some data for the url, and finally with |
* FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred: |
* data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER, |
* FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL. |
* |
* Some private data can be passed as the last parameter to fetch_start, and |
* callbacks will contain this. |
*/ |
|
void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, |
bool only_2xx, bool downgrade_tls, const char *post_urlenc, |
const struct fetch_multipart_data *post_multipart, |
const char **headers) |
{ |
LOG(("curl setup\n")); |
|
return; |
} |
|
|
/** |
* Dispatch a single job |
*/ |
bool fetch_curl_start(void *vfetch) |
{ |
LOG(("curl start\n")); |
return 0; |
} |
|
|
/** |
* Initiate a fetch from the queue. |
* |
* Called with a fetch structure and a CURL handle to be used to fetch the |
* content. |
* |
* This will return whether or not the fetch was successfully initiated. |
*/ |
|
bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) |
{ |
LOG(("curl initi fetch\n")); |
return 0; |
} |
|
|
/** |
* Find a CURL handle to use to dispatch a job |
*/ |
|
CURL *fetch_curl_get_handle(lwc_string *host) |
{ |
LOG(("curl get handle\n")); |
return 0; |
} |
|
|
/** |
* Cache a CURL handle for the provided host (if wanted) |
*/ |
|
void fetch_curl_cache_handle(CURL *handle, lwc_string *host) |
{ |
LOG(("curl cache handle\n")); |
} |
|
|
/** |
* Set options specific for a fetch. |
*/ |
|
typedef int CURLcode; |
|
CURLcode |
fetch_curl_set_options(struct curl_fetch_info *f) |
{ |
LOG(("curl set options\n")); |
return -1; |
} |
|
|
/** |
* cURL SSL setup callback |
*/ |
|
CURLcode |
fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) |
{ |
LOG(("curlcodessl\n")); |
|
return -1; |
} |
|
|
/** |
* Abort a fetch. |
*/ |
|
void fetch_curl_abort(void *vf) |
{ |
LOG(("curl abort\n")); |
} |
|
|
/** |
* Clean up the provided fetch object and free it. |
* |
* Will prod the queue afterwards to allow pending requests to be initiated. |
*/ |
|
void fetch_curl_stop(struct curl_fetch_info *f) |
{ |
LOG(("curl stop\n")); |
} |
|
|
/** |
* Free a fetch structure and associated resources. |
*/ |
|
void fetch_curl_free(void *vf) |
{ |
LOG(("curl free\n")); |
} |
|
|
/** |
* Do some work on current fetches. |
* |
* Must be called regularly to make progress on fetches. |
*/ |
|
void fetch_curl_poll(lwc_string *scheme_ignored) |
{ |
LOG(("curl poll\n")); |
} |
|
|
/** |
* Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). |
* |
* \param curl_handle curl easy handle of fetch |
*/ |
|
void fetch_curl_done(CURL *curl_handle, CURLcode result) |
{ |
LOG(("curl done\n")); |
} |
|
|
/** |
* Callback function for fetch progress. |
*/ |
|
int fetch_curl_progress(void *clientp, double dltotal, double dlnow, |
double ultotal, double ulnow) |
{ |
LOG(("curl progress\n")); |
return 0; |
} |
|
|
|
/** |
* Ignore everything given to it. |
* |
* Used to ignore cURL debug. |
*/ |
|
int fetch_curl_ignore_debug(CURL *handle, |
curl_infotype type, |
char *data, |
size_t size, |
void *userptr) |
{ |
LOG(("curl igdebug\n")); |
return 0; |
} |
|
|
/** |
* Callback function for cURL. |
*/ |
|
size_t fetch_curl_data(char *data, size_t size, size_t nmemb, |
void *_f) |
{ |
LOG(("curl callback\n")); |
return 0; |
} |
|
|
/** |
* Callback function for headers. |
* |
* See RFC 2616 4.2. |
*/ |
|
size_t fetch_curl_header(char *data, size_t size, size_t nmemb, |
void *_f) |
|
{ |
LOG(("curl header\n")); |
return 0; |
} |
|
/** |
* Find the status code and content type and inform the caller. |
* |
* Return true if the fetch is being aborted. |
*/ |
|
bool fetch_curl_process_headers(struct curl_fetch_info *f) |
{ |
LOG(("curl proc head\n")); |
return false; |
} |
|
|
/** |
* Convert a list of struct ::fetch_multipart_data to a list of |
* struct curl_httppost for libcurl. |
*/ |
struct curl_httppost * |
fetch_curl_post_convert(const struct fetch_multipart_data *control) |
{ |
struct curl_httppost *post = 0; |
|
LOG(("curl post - FAIL\n")); |
return post; |
} |
|
|
/** |
* OpenSSL Certificate verification callback |
* Stores certificate details in fetch struct. |
*/ |
|
int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx) |
{ |
|
LOG(("curl verify\n")); |
return 0; |
} |
|
|
/** |
* OpenSSL certificate chain verification callback |
* Verifies certificate chain, setting up context for fetch_curl_verify_callback |
*/ |
|
int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) |
{ |
|
LOG(("curl cert verify\n")); |
return 0; |
} |