| 63,99 → 63,38 |
|---|
| #define FETCHER_CURLL_SCHEDULED 1 |
| */ |
| |
| typedef int X509_STORE_CTX; |
| typedef int X509; |
| #define CURL_ERROR_SIZE 0 |
| typedef int CURLcode; |
| typedef int curl_infotype; |
| /** SSL certificate info */ |
| struct cert_info { |
| X509 *cert; /**< Pointer to certificate */ |
| long err; /**< OpenSSL error code */ |
| }; |
| |
| /** Information for a single fetch. */ |
| struct curl_fetch_info { |
| struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ |
| CURL * curl_handle; /**< cURL handle if being fetched, or 0. */ |
| bool had_headers; /**< Headers have been processed. */ |
| bool abort; /**< Abort requested. */ |
| bool stopped; /**< Download stopped on purpose. */ |
| bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ |
| bool downgrade_tls; /**< Downgrade to TLS <= 1.0 */ |
| nsurl *url; /**< URL of this fetch. */ |
| lwc_string *host; /**< The hostname of this fetch. */ |
| struct curl_slist *headers; /**< List of request headers. */ |
| char *location; /**< Response Location header, or 0. */ |
| unsigned long content_length; /**< Response Content-Length, or 0. */ |
| char *cookie_string; /**< Cookie string for this fetch */ |
| char *realm; /**< HTTP Auth Realm */ |
| char *post_urlenc; /**< Url encoded POST string, or 0. */ |
| long http_code; /**< HTTP result code from cURL. */ |
| struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */ |
| #define MAX_CERTS 10 |
| struct cert_info cert_data[MAX_CERTS]; /**< HTTPS certificate data */ |
| unsigned int last_progress_update; /**< Time of last progress update */ |
| }; |
| struct fetch_curl_context { |
| struct fetch_curl_context *r_next, *r_prev; |
| |
| struct cache_handle { |
| CURL *handle; /**< The cached cURL handle */ |
| lwc_string *host; /**< The host for which this handle is cached */ |
| struct fetch *fetchh; /**< Handle for this fetch */ |
| |
| struct cache_handle *r_prev; /**< Previous cached handle in ring. */ |
| struct cache_handle *r_next; /**< Next cached handle in ring. */ |
| bool aborted; /**< Flag indicating fetch has been aborted */ |
| bool locked; /**< Flag indicating entry is already entered */ |
| |
| nsurl *url; /**< The full url the fetch refers to */ |
| char *path; /**< The actual path to be used with open() */ |
| |
| time_t file_etag; /**< Request etag for file (previous st.m_time) */ |
| }; |
| |
| CURLM *fetch_curl_multi; /**< Global cURL multi handle. */ |
| /** Curl handle with default options set; not used for transfers. */ |
| static CURL *fetch_blank_curl; |
| static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */ |
| static int curl_fetchers_registered = 0; |
| static bool curl_with_openssl; |
| static struct fetch_curl_context *ring = NULL; |
| |
| static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */ |
| static char fetch_proxy_userpwd[100]; /**< Proxy authentication details. */ |
| |
| static bool fetch_curl_initialise(lwc_string *scheme); |
| static void fetch_curl_finalise(lwc_string *scheme); |
| static bool fetch_curl_can_fetch(const nsurl *url); |
| static bool fetch_curl_initialise(lwc_string *scheme); //here |
| static void fetch_curl_finalise(lwc_string *scheme); //here |
| static bool fetch_curl_can_fetch(const nsurl *url); //here |
| static void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, |
| bool only_2xx, bool downgrade_tls, const char *post_urlenc, |
| const struct fetch_multipart_data *post_multipart, |
| const char **headers); |
| static bool fetch_curl_start(void *vfetch); |
| static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, |
| CURL *handle); |
| static CURL *fetch_curl_get_handle(lwc_string *host); |
| static void fetch_curl_cache_handle(CURL *handle, lwc_string *host); |
| static CURLcode fetch_curl_set_options(struct curl_fetch_info *f); |
| static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, |
| void *p); |
| static void fetch_curl_abort(void *vf); |
| static void fetch_curl_stop(struct curl_fetch_info *f); |
| static void fetch_curl_free(void *f); |
| static void fetch_curl_poll(lwc_string *scheme_ignored); |
| static void fetch_curl_done(CURL *curl_handle, CURLcode result); |
| static int fetch_curl_progress(void *clientp, double dltotal, double dlnow, |
| double ultotal, double ulnow); |
| static int fetch_curl_ignore_debug(CURL *handle, |
| curl_infotype type, |
| char *data, |
| size_t size, |
| void *userptr); |
| static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, |
| void *_f); |
| static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, |
| void *_f); |
| static bool fetch_curl_process_headers(struct curl_fetch_info *f); |
| static struct curl_httppost *fetch_curl_post_convert( |
| const struct fetch_multipart_data *control); |
| static int fetch_curl_verify_callback(int preverify_ok, |
| X509_STORE_CTX *x509_ctx); |
| static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, |
| void *parm); |
| const char **headers); //here |
| static bool fetch_curl_start(void *vfetch); //here |
| |
| static void fetch_curl_abort(void *vf); //here |
| static void fetch_curl_free(void *f); //here |
| static void fetch_curl_poll(lwc_string *scheme_ignored); //here |
| |
| |
| /** |
| * Initialise the fetcher. |
| * |
| 173,18 → 112,18 |
|---|
| lwc_intern_string("http", SLEN("http"), &scheme); |
| |
| if (!fetch_add_fetcher(scheme, |
| fetch_curl_initialise, |
| fetch_curl_can_fetch, |
| fetch_curl_initialise, //here |
| fetch_curl_can_fetch, //here |
| fetch_curl_setup, |
| fetch_curl_start, |
| fetch_curl_abort, |
| fetch_curl_free, |
| fetch_curl_abort, //here |
| fetch_curl_free, //here |
| #ifdef FETCHER_CURLL_SCHEDULED |
| NULL, |
| #else |
| fetch_curl_poll, |
| fetch_curl_poll, //here |
| #endif |
| fetch_curl_finalise)) { |
| fetch_curl_finalise)) { //here |
| LOG(("Unable to register cURL fetcher for HTTP")); |
| } |
| |
| 231,10 → 170,10 |
|---|
| LOG(("curl finali\n")); |
| } |
| |
| bool fetch_curl_can_fetch(const nsurl *url) |
| static bool fetch_curl_can_fetch(const nsurl *url) |
| { |
| LOG(("curl can fetch\n")); |
| return false; |
| return true; //let's lie a bit |
| } |
| |
| /** |
| 259,251 → 198,240 |
|---|
| * callbacks will contain this. |
| */ |
| |
| void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, |
| bool only_2xx, bool downgrade_tls, const char *post_urlenc, |
| void * fetch_curl_setup (struct fetch *fetchh, |
| nsurl *url, |
| bool only_2xx, |
| bool downgrade_tls, |
| const char *post_urlenc, |
| const struct fetch_multipart_data *post_multipart, |
| const char **headers) |
| { |
| |
| LOG(("curl setup\n")); |
| |
| return; |
| } |
| struct fetch_curl_context *ctx; |
| int i; |
| |
| ctx = calloc(1, sizeof(*ctx)); |
| if (ctx == NULL) |
| return NULL; |
| |
| /** |
| * Dispatch a single job |
| */ |
| bool fetch_curl_start(void *vfetch) |
| { |
| LOG(("curl start\n")); |
| return 0; |
| ctx->path = url_to_path(nsurl_access(url)); |
| if (ctx->path == NULL) { |
| free(ctx); |
| return NULL; |
| } |
| |
| ctx->url = nsurl_ref(url); |
| |
| /** |
| * Initiate a fetch from the queue. |
| * |
| * Called with a fetch structure and a CURL handle to be used to fetch the |
| * content. |
| * |
| * This will return whether or not the fetch was successfully initiated. |
| */ |
| |
| bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) |
| { |
| LOG(("curl initi fetch\n")); |
| return 0; |
| } |
| ctx->fetchh = fetchh; |
| |
| RING_INSERT(ring, ctx); |
| |
| /** |
| * Find a CURL handle to use to dispatch a job |
| */ |
| |
| CURL *fetch_curl_get_handle(lwc_string *host) |
| { |
| LOG(("curl get handle\n")); |
| return 0; |
| return ctx; |
| } |
| |
| |
| /** |
| * Cache a CURL handle for the provided host (if wanted) |
| * Dispatch a single job |
| */ |
| |
| void fetch_curl_cache_handle(CURL *handle, lwc_string *host) |
| bool fetch_curl_start(void *vfetch) |
| { |
| LOG(("curl cache handle\n")); |
| LOG(("curl start\n")); |
| return true; |
| } |
| |
| |
| /** |
| * Set options specific for a fetch. |
| */ |
| |
| typedef int CURLcode; |
| |
| CURLcode |
| fetch_curl_set_options(struct curl_fetch_info *f) |
| { |
| LOG(("curl set options\n")); |
| return -1; |
| } |
| |
| |
| /** |
| * cURL SSL setup callback |
| * Abort a fetch. |
| */ |
| |
| CURLcode |
| fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) |
| void fetch_curl_abort(void *ctx) |
| { |
| LOG(("curlcodessl\n")); |
| struct fetch_curl_context *c = ctx; |
| |
| return -1; |
| /* To avoid the poll loop having to deal with the fetch context |
| * disappearing from under it, we simply flag the abort here. |
| * The poll loop itself will perform the appropriate cleanup. |
| */ |
| c->aborted = true; |
| } |
| |
| |
| /** |
| * Abort a fetch. |
| * Free a fetch structure and associated resources. |
| */ |
| |
| void fetch_curl_abort(void *vf) |
| void fetch_curl_free(void *ctx) |
| { |
| LOG(("curl abort\n")); |
| struct fetch_curl_context *c = ctx; |
| nsurl_unref(c->url); |
| free(c->path); |
| RING_REMOVE(ring, c); |
| free(ctx); |
| } |
| |
| static inline bool fetch_curl_send_callback(const fetch_msg *msg, |
| struct fetch_curl_context *ctx) |
| { |
| ctx->locked = true; |
| fetch_send_callback(msg, ctx->fetchh); |
| ctx->locked = false; |
| |
| /** |
| * Clean up the provided fetch object and free it. |
| * |
| * Will prod the queue afterwards to allow pending requests to be initiated. |
| */ |
| return ctx->aborted; |
| } |
| |
| void fetch_curl_stop(struct curl_fetch_info *f) |
| static bool fetch_curl_send_header(struct fetch_curl_context *ctx, |
| const char *fmt, ...) |
| { |
| LOG(("curl stop\n")); |
| } |
| fetch_msg msg; |
| char header[64]; |
| va_list ap; |
| |
| va_start(ap, fmt); |
| |
| /** |
| * Free a fetch structure and associated resources. |
| */ |
| vsnprintf(header, sizeof header, fmt, ap); |
| |
| void fetch_curl_free(void *vf) |
| { |
| LOG(("curl free\n")); |
| } |
| va_end(ap); |
| |
| msg.type = FETCH_HEADER; |
| msg.data.header_or_data.buf = (const uint8_t *) header; |
| msg.data.header_or_data.len = strlen(header); |
| fetch_curl_send_callback(&msg, ctx); |
| |
| /** |
| * Do some work on current fetches. |
| * |
| * Must be called regularly to make progress on fetches. |
| */ |
| return ctx->aborted; |
| } |
| |
| void fetch_curl_poll(lwc_string *scheme_ignored) |
| static void fetch_curl_process_error(struct fetch_curl_context *ctx, int code) |
| { |
| LOG(("curl poll\n")); |
| } |
| fetch_msg msg; |
| char buffer[1024]; |
| const char *title; |
| char key[8]; |
| |
| /* content is going to return error code */ |
| fetch_set_http_code(ctx->fetchh, code); |
| |
| /** |
| * Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). |
| * |
| * \param curl_handle curl easy handle of fetch |
| */ |
| /* content type */ |
| if (fetch_curl_send_header(ctx, "Content-Type: text/html")) |
| goto fetch_file_process_error_aborted; |
| |
| void fetch_curl_done(CURL *curl_handle, CURLcode result) |
| { |
| LOG(("curl done\n")); |
| } |
| snprintf(key, sizeof key, "HTTP%03d", code); |
| title = messages_get(key); |
| |
| snprintf(buffer, sizeof buffer, "<html><head><title>%s</title></head>" |
| "<body><h1>%s</h1>" |
| "<p>Error %d while fetching file %s</p></body></html>", |
| title, title, code, nsurl_access(ctx->url)); |
| |
| /** |
| * Callback function for fetch progress. |
| */ |
| msg.type = FETCH_DATA; |
| msg.data.header_or_data.buf = (const uint8_t *) buffer; |
| msg.data.header_or_data.len = strlen(buffer); |
| if (fetch_curl_send_callback(&msg, ctx)) |
| goto fetch_file_process_error_aborted; |
| |
| int fetch_curl_progress(void *clientp, double dltotal, double dlnow, |
| double ultotal, double ulnow) |
| { |
| LOG(("curl progress\n")); |
| return 0; |
| msg.type = FETCH_FINISHED; |
| fetch_curl_send_callback(&msg, ctx); |
| |
| fetch_file_process_error_aborted: |
| return; |
| } |
| |
| |
| static void fetch_curl_process(struct fetch_curl_context *ctx) { |
| |
| /** |
| * Ignore everything given to it. |
| * |
| * Used to ignore cURL debug. |
| fetch_msg msg; |
| const char * buf = "<html><body><h1>Hello, file fetcher!</h1></body></html>"; |
| // fetch_curl_process_error(ctx, 501); |
| // return; |
| |
| /* fetch is going to be successful */ |
| fetch_set_http_code(ctx->fetchh, 200); |
| |
| /* Any callback can result in the fetch being aborted. |
| * Therefore, we _must_ check for this after _every_ call to |
| * fetch_file_send_callback(). |
| */ |
| |
| int fetch_curl_ignore_debug(CURL *handle, |
| curl_infotype type, |
| char *data, |
| size_t size, |
| void *userptr) |
| { |
| LOG(("curl igdebug\n")); |
| return 0; |
| } |
| /* content type */ |
| if (fetch_curl_send_header(ctx, "Content-Type: text/html")) |
| goto fetch_file_process_aborted; |
| |
| |
| /** |
| * Callback function for cURL. |
| */ |
| /* main data loop */ |
| |
| size_t fetch_curl_data(char *data, size_t size, size_t nmemb, |
| void *_f) |
| { |
| LOG(("curl callback\n")); |
| return 0; |
| } |
| msg.type = FETCH_DATA; |
| msg.data.header_or_data.buf = (const uint8_t *) buf; |
| msg.data.header_or_data.len = strlen(buf); |
| fetch_curl_send_callback(&msg, ctx); |
| |
| |
| /** |
| * Callback function for headers. |
| * |
| * See RFC 2616 4.2. |
| */ |
| |
| size_t fetch_curl_header(char *data, size_t size, size_t nmemb, |
| void *_f) |
| if (ctx->aborted == false) { |
| msg.type = FETCH_FINISHED; |
| fetch_curl_send_callback(&msg, ctx); |
| } |
| |
| { |
| LOG(("curl header\n")); |
| return 0; |
| fetch_file_process_aborted: |
| return; |
| |
| } |
| |
| |
| /** |
| * Find the status code and content type and inform the caller. |
| * Do some work on current fetches. |
| * |
| * Return true if the fetch is being aborted. |
| * Must be called regularly to make progress on fetches. |
| */ |
| |
| bool fetch_curl_process_headers(struct curl_fetch_info *f) |
| void fetch_curl_poll(lwc_string *scheme_ignored) |
| { |
| LOG(("curl proc head\n")); |
| return false; |
| } |
| LOG(("curl poll\n")); |
| |
| struct fetch_curl_context *c, *next; |
| |
| /** |
| * Convert a list of struct ::fetch_multipart_data to a list of |
| * struct curl_httppost for libcurl. |
| if (ring == NULL) return; |
| |
| /* Iterate over ring, processing each pending fetch */ |
| c = ring; |
| do { |
| /* Ignore fetches that have been flagged as locked. |
| * This allows safe re-entrant calls to this function. |
| * Re-entrancy can occur if, as a result of a callback, |
| * the interested party causes fetch_poll() to be called |
| * again. |
| */ |
| struct curl_httppost * |
| fetch_curl_post_convert(const struct fetch_multipart_data *control) |
| { |
| struct curl_httppost *post = 0; |
| if (c->locked == true) { |
| next = c->r_next; |
| continue; |
| } |
| |
| LOG(("curl post - FAIL\n")); |
| return post; |
| /* Only process non-aborted fetches */ |
| if (c->aborted == false) { |
| /* file fetches can be processed in one go */ |
| fetch_curl_process(c); |
| } |
| |
| /* Compute next fetch item at the last possible moment as |
| * processing this item may have added to the ring. |
| */ |
| next = c->r_next; |
| |
| /** |
| * OpenSSL Certificate verification callback |
| * Stores certificate details in fetch struct. |
| fetch_remove_from_queues(c->fetchh); |
| fetch_free(c->fetchh); |
| |
| /* Advance to next ring entry, exiting if we've reached |
| * the start of the ring or the ring has become empty |
| */ |
| } while ( (c = next) != ring && ring != NULL); |
| |
| int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx) |
| { |
| |
| LOG(("curl verify\n")); |
| return 0; |
| } |
| |
| |
| /** |
| * OpenSSL certificate chain verification callback |
| * Verifies certificate chain, setting up context for fetch_curl_verify_callback |
| */ |
| |
| int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) |
| { |
| |
| LOG(("curl cert verify\n")); |
| return 0; |
| } |