Subversion Repositories Kolibri OS

Rev

Rev 4821 | Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1. /*
  2.  * Copyright 2006 Daniel Silverstone <dsilvers@digital-scurf.org>
  3.  * Copyright 2007 James Bursa <bursa@users.sourceforge.net>
  4.  * Copyright 2003 Phil Mellor <monkeyson@users.sourceforge.net>
  5.  *
  6.  * This file is part of NetSurf.
  7.  *
  8.  * NetSurf is free software; you can redistribute it and/or modify
  9.  * it under the terms of the GNU General Public License as published by
  10.  * the Free Software Foundation; version 2 of the License.
  11.  *
  12.  * NetSurf is distributed in the hope that it will be useful,
  13.  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15.  * GNU General Public License for more details.
  16.  *
  17.  * You should have received a copy of the GNU General Public License
  18.  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19.  */
  20.  
  21. /** \file
  22.  * Fetching of data from a URL (implementation).
  23.  *
  24.  * This implementation uses libcurl's 'multi' interface.
  25.  *
  26.  *
  27.  * The CURL handles are cached in the curl_handle_ring. There are at most
  28.  * ::max_cached_fetch_handles in this ring.
  29.  */
  30.  
  31. /*
  32. TODO:
  33. Have a double linked list containing all the handles on work that needs to be done
  34. Inside fetch_poll_curl, do work on it.
  35. Let the overall structure remain intact
  36. */
  37.  
  38. #include <assert.h>
  39. #include <errno.h>
  40. #include <inttypes.h>
  41. #include <stdbool.h>
  42. #include <stdarg.h>
  43. #include <string.h>
  44. #include <strings.h>
  45. #include <time.h>
  46. #include <sys/stat.h>
  47.  
  48. #include <libwapcaplet/libwapcaplet.h>
  49.  
  50. #include "utils/config.h"
  51. /* #include <openssl/ssl.h> */
  52. #include "content/fetch.h"
  53. #include "content/fetchers/curl.h"
  54. #include "content/urldb.h"
  55. #include "desktop/netsurf.h"
  56. #include "desktop/options.h"
  57. #include "utils/log.h"
  58. #include "utils/messages.h"
  59. #include "utils/schedule.h"
  60. #include "utils/utils.h"
  61. #include "utils/ring.h"
  62. #include "utils/useragent.h"
  63.  
  64. /* BIG FAT WARNING: This is here because curl doesn't give you an FD to
  65.  * poll on, until it has processed a bit of the handle.  So we need schedules
  66.  * in order to make this work.
  67.  */
  68. #include <desktop/browser.h>
  69.  
  70. #include "http_msg.h"
  71. #include "http.h"
  72.  
  73. /**********************************************************************
  74.  ********This section added for resolving compile errors***************/
  75. #define CURL_ERROR_SIZE 100
  76. /*Definitions for CURL EASY Codes*/
  77.  
  78. #define CURLE_OK 0
  79. #define CURLE_PARTIAL_FILE 18
  80. #define CURLE_WRITE_ERROR 23
  81. #define CURLE_SSL_CONNECT_ERROR 35
  82.  
  83. /*Definitions for CURL MULTI Codes*/
  84.  
  85. #define CURLM_OK 0
  86. #define CURLM_CALL_MULTI_PERFORM -1
  87. #define CURLM_FAILED 123123
  88. /* KOSH stands for KolibriOS HTTP :) */
  89.  
  90. int FLAG_HTTP11             = 1 << 0;
  91. int FLAG_GOT_HEADER         = 1 << 1;
  92. int FLAG_GOT_ALL_DATA       = 1 << 2;
  93. int FLAG_CONTENT_LENGTH     = 1 << 3;
  94. int FLAG_CHUNKED            = 1 << 4;
  95. int FLAG_CONNECTED          = 1 << 5;
  96.  
  97. /* ERROR flags go into the upper word */
  98. int FLAG_INVALID_HEADER     = 1 << 16;
  99. int FLAG_NO_RAM             = 1 << 17;
  100. int FLAG_SOCKET_ERROR       = 1 << 18;
  101. int FLAG_TIMEOUT_ERROR      = 1 << 19;
  102. int FLAG_TRANSFER_FAILED    = 1 << 20;
  103.  
  104.  
  105. struct KOSHcode {
  106.   long code;
  107. };
  108.  
  109. struct KOSHMcode {
  110.   long code;
  111. };
  112.  
  113. struct KOSHMsg {
  114.   char *msg;
  115. };
  116.  
  117. typedef struct KOSHcode KOSHcode;
  118. typedef struct KOSHMcode KOSHMcode;
  119. struct kosh_infotype {
  120.   int type;
  121. };
  122. typedef struct kosh_infotype kosh_infotype;
  123.  
  124.  struct curl_slist {
  125.    char *data;
  126.    struct curl_slist *next;
  127.  };
  128.  
  129. /**********************************************************************/
  130.  
  131. /* uncomment this to use scheduler based calling
  132. #define FETCHER_CURLL_SCHEDULED 1
  133. */
  134.  
  135. /** SSL certificate info */
  136. /* struct cert_info { */
  137. /*      X509 *cert;             /\**< Pointer to certificate *\/ */
  138. /*      long err;               /\**< OpenSSL error code *\/ */
  139. /* }; */
  140.  
  141. /** Information for a single fetch. */
  142. struct curl_fetch_info {
  143.         struct fetch *fetch_handle; /**< The fetch handle we're parented by. */
  144.         struct http_msg * curl_handle;  /**< cURL handle if being fetched, or 0. */
  145.         bool had_headers;       /**< Headers have been processed. */
  146.         bool abort;             /**< Abort requested. */
  147.         bool stopped;           /**< Download stopped on purpose. */
  148.         bool only_2xx;          /**< Only HTTP 2xx responses acceptable. */
  149.         bool downgrade_tls;     /**< Downgrade to TLS <= 1.0 */
  150.         nsurl *url;             /**< URL of this fetch. */
  151.         lwc_string *host;       /**< The hostname of this fetch. */
  152.         struct curl_slist *headers;     /**< List of request headers. */
  153.         char *location;         /**< Response Location header, or 0. */
  154.         unsigned long content_length;   /**< Response Content-Length, or 0. */
  155.         char *cookie_string;    /**< Cookie string for this fetch */
  156.         char *realm;            /**< HTTP Auth Realm */
  157.         char *post_urlenc;      /**< Url encoded POST string, or 0. */
  158.         long http_code; /**< HTTP result code from cURL. */
  159.         struct curl_httppost *post_multipart;   /**< Multipart post data, or 0. */
  160. /* #define MAX_CERTS 10 */
  161. /*      struct cert_info cert_data[MAX_CERTS];  /\**< HTTPS certificate data *\/ */
  162.         unsigned int last_progress_update;      /**< Time of last progress update */
  163. };
  164.  
  165. struct cache_handle {
  166.         struct http_msg *handle; /**< The cached struct http_msg handle */
  167.         lwc_string *host;   /**< The host for which this handle is cached */
  168.  
  169.         struct cache_handle *r_prev; /**< Previous cached handle in ring. */
  170.         struct cache_handle *r_next; /**< Next cached handle in ring. */
  171. };
  172.  
  173. struct fetch_info_slist {
  174.   struct curl_fetch_info *fetch_info;  /* this fetch_info contains the same handle as the struct */
  175.   struct http_msg *handle;
  176.   bool fetch_curl_header_called;
  177.   struct fetch_info_slist *next;
  178. };
  179.  
  180.  
  181. struct fetch_info_slist *fetch_curl_multi;              /**< Global cURL multi handle. */
  182.  
  183. /** Curl handle with default options set; not used for transfers. */
  184. static struct http_msg *fetch_blank_curl;
  185. static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */
  186. static int curl_fetchers_registered = 0;
  187. static bool curl_with_openssl;
  188.  
  189. static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */
  190. static char fetch_proxy_userpwd[100];   /**< Proxy authentication details. */
  191.  
  192. static bool fetch_curl_initialise(lwc_string *scheme);
  193. static void fetch_curl_finalise(lwc_string *scheme);
  194. static bool fetch_curl_can_fetch(const nsurl *url);
  195. static void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url,
  196.                  bool only_2xx, bool downgrade_tls, const char *post_urlenc,
  197.                  const struct fetch_multipart_data *post_multipart,
  198.                  const char **headers);
  199. static bool fetch_curl_start(void *vfetch);
  200. static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch,
  201.                 struct http_msg *handle);
  202. static struct http_msg *fetch_curl_get_handle(lwc_string *host);
  203. static void fetch_curl_cache_handle(struct http_msg *handle, lwc_string *host);
  204. static KOSHcode fetch_curl_set_options(struct curl_fetch_info *f);
  205. /* static KOSHcode fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, */
  206. /*                                   void *p); */
  207. static void fetch_curl_abort(void *vf);
  208. static void fetch_curl_stop(struct fetch_info_slist *f);
  209. static void fetch_curl_free(void *f);
  210. static void fetch_curl_poll(lwc_string *scheme_ignored);
  211. static void fetch_curl_done(struct fetch_info_slist *curl_handle);
  212. static int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
  213.                                double ultotal, double ulnow);
  214. static int fetch_curl_ignore_debug(struct http_msg *handle,
  215.                                    kosh_infotype type,
  216.                                    char *data,
  217.                                    size_t size,
  218.                                    void *userptr);
  219. static size_t fetch_curl_data(void *_f);
  220. /* static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, */
  221. /*                              void *_f); */
  222. void fetch_curl_header(void *_f);
  223. static bool fetch_curl_process_headers(struct curl_fetch_info *f);
  224. static struct curl_httppost *fetch_curl_post_convert(
  225.                 const struct fetch_multipart_data *control);
  226.  
  227. /* static int fetch_curl_verify_callback(int preverify_ok, */
  228. /*              X509_STORE_CTX *x509_ctx); */
  229. /* static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, */
  230. /*              void *parm); */
  231.  
  232. /**************Functions added for replacing curl's provided functionality ************/
  233. struct curl_slist *curl_slist_append(struct curl_slist * list, const char * string );
  234. void curl_slist_free_all(struct curl_slist *);
  235. struct fetch_info_slist *curl_multi_remove_handle(struct fetch_info_slist *multi_handle, struct curl_fetch_info *fetch_to_delete);
  236. struct http_msg * curl_easy_init(void);
  237. void curl_easy_cleanup(struct http_msg *handle);
  238. int curl_multi_add_handle(struct fetch_info_slist **multi_handle, struct curl_fetch_info *new_fetch_info);
  239.  
  240. /**
  241.  * Initialise the fetcher.
  242.  *
  243.  * Must be called once before any other function.
  244.  */
  245.  
  246. void fetch_curl_register(void)
  247. {
  248.         /* KOSHcode code; */
  249.         /* curl_version_info_data *data; */
  250.         int i;
  251.         lwc_string *scheme;
  252.        
  253.         LOG(("curl_version (no CURL XD)"));
  254.  
  255.         const struct fetcher_operation_table fetcher_ops = {
  256.           .initialise = fetch_curl_initialise,
  257.           .acceptable = fetch_curl_can_fetch,
  258.           .setup = fetch_curl_setup,
  259.           .start = fetch_curl_start,
  260.           .abort = fetch_curl_abort,
  261.           .free = fetch_curl_free,
  262.           .poll = fetch_curl_poll,
  263.           .finalise = fetch_curl_finalise
  264.         };
  265.        
  266.  
  267.         /* code = curl_global_init(CURL_GLOBAL_ALL); */
  268.         /* if (code != CURLE_OK) */
  269.         /*      die("Failed to initialise the fetch module " */
  270.         /*                      "(curl_global_init failed)."); */
  271.  
  272.         /*TODO : Put the init function for our global queue here */
  273.         /* What do we need for an init? Just setting the list to NULL should be enough.
  274.            We might track the number of nodes in the list, but that's not required since we usean SLL.
  275.         */
  276.         /* fetch_curl_multi = curl_multi_init(); */
  277.         /* if (!fetch_curl_multi) */
  278.         /*      die("Failed to initialise the fetch module " */
  279.         /*                      "(curl_multi_init failed)."); */
  280.  
  281.         /* Create a curl easy handle with the options that are common to all
  282.            fetches. */
  283.        
  284.         /*HTTP Library initiated using volatime asm code in http.c */
  285.         DBG("Calling curl_easy_init\n");
  286.         /* fetch_blank_curl = curl_easy_init();  */
  287.        
  288.         /* if (!fetch_blank_curl) */
  289.         /*   { */
  290.         /*     DBG("fetch_blank_curl is NULL"); */
  291.         /*     die("Failed to initialise the fetch module " */
  292.         /*      "(curl_easy_init failed)."); */
  293.         /*   } */
  294.         /* else */
  295.         /*   DBG("fetch_blank_curl is usable."); */
  296.  
  297.         fetch_blank_curl = NULL;
  298.  
  299.         /* TODO: The SETOPT calls set up the parameters for the curl handle.
  300.            Since we don't want to use curl, these are of no use, but our native handle
  301.            should consider all these fields while being set up for proper functioning
  302.         */
  303.  
  304. /* #undef SETOPT */
  305. /* #define SETOPT(option, value) \ */
  306. /*      code = curl_easy_setopt(fetch_blank_curl, option, value);       \ */
  307. /*      if (code != CURLE_OK)                                           \ */
  308. /*              goto curl_easy_setopt_failed; */
  309.  
  310. /*      if (verbose_log) { */
  311. /*          SETOPT(CURLOPT_VERBOSE, 1); */
  312. /*      } else { */
  313. /*          SETOPT(CURLOPT_VERBOSE, 0); */
  314. /*      } */
  315. /*      SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer); */
  316. /*      if (nsoption_bool(suppress_curl_debug)) */
  317. /*              SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore_debug); */
  318. /*      SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data); */
  319. /*      SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header); */ /* Calling fetch_curl_header inside fetch_curl_process_headers */
  320.         /*      SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress); */ /* TODO: Add this with httplib somehow */
  321. /*      SETOPT(CURLOPT_NOPROGRESS, 0); */
  322. /*      SETOPT(CURLOPT_USERAGENT, user_agent_string()); */
  323. /*      SETOPT(CURLOPT_ENCODING, "gzip"); */
  324. /*      SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L); */
  325. /*      SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); */
  326. /*      SETOPT(CURLOPT_NOSIGNAL, 1L); */
  327. /*      SETOPT(CURLOPT_CONNECTTIMEOUT, 30L); */
  328.  
  329. /*      if (nsoption_charp(ca_bundle) &&  */
  330. /*          strcmp(nsoption_charp(ca_bundle), "")) { */
  331. /*              LOG(("ca_bundle: '%s'", nsoption_charp(ca_bundle))); */
  332. /*              SETOPT(CURLOPT_CAINFO, nsoption_charp(ca_bundle)); */
  333. /*      } */
  334. /*      if (nsoption_charp(ca_path) && strcmp(nsoption_charp(ca_path), "")) { */
  335. /*              LOG(("ca_path: '%s'", nsoption_charp(ca_path))); */
  336. /*              SETOPT(CURLOPT_CAPATH, nsoption_charp(ca_path)); */
  337. /*      } */
  338.  
  339.         /*TODO: Useless for now, no SSL Support*/
  340.  
  341.         /* /\* Detect whether the SSL CTX function API works *\/ */
  342.         /* curl_with_openssl = true; */
  343.         /* code = curl_easy_setopt(fetch_blank_curl,  */
  344.         /*              CURLOPT_SSL_CTX_FUNCTION, NULL); */
  345.         /* if (code != CURLE_OK) { */
  346.         /*      curl_with_openssl = false; */
  347.         /* } */
  348.  
  349.         /* /\* LOG(("cURL %slinked against openssl", curl_with_openssl ? "" : "not ")); *\/ */
  350.  
  351.         /* /\* cURL initialised okay, register the fetchers *\/ */
  352.  
  353.         /* data = curl_version_info(CURLVERSION_NOW); */
  354.  
  355.         /*TODO: We strictly want to deal with only http as a protocol right now, this stuff can come
  356.           handy later, so it shall sit here for a while XD
  357.           Removing the for loop for a single http fetcher setup scheme.
  358.         */
  359.        
  360.         /*      for (i = 0; data->protocols[i]; i++) {
  361.                 if (strcmp(data->protocols[i], "http") == 0) {
  362.                         if (lwc_intern_string("http", SLEN("http"),
  363.                                         &scheme) != lwc_error_ok) {
  364.                                 die("Failed to initialise the fetch module "
  365.                                                 "(couldn't intern \"http\").");
  366.                         }
  367.  
  368.                 } else if (strcmp(data->protocols[i], "https") == 0) {
  369.                         if (lwc_intern_string("https", SLEN("https"),
  370.                                         &scheme) != lwc_error_ok) {
  371.                                 die("Failed to initialise the fetch module "
  372.                                                 "(couldn't intern \"https\").");
  373.                         }
  374.  
  375.                 } else {
  376.                         // Ignore non-http(s) protocols
  377.                         continue;
  378.                 }
  379.  
  380.                 if (!fetch_add_fetcher(scheme,
  381.                 fetch_curl_initialise,
  382.                                 fetch_curl_can_fetch,
  383.                                 fetch_curl_setup,
  384.                                 fetch_curl_start,
  385.                                 fetch_curl_abort,
  386.                                 sfetch_curl_free,
  387. #ifdef FETCHER_CURLL_SCHEDULED
  388.                                        NULL,
  389. #else
  390.                                 fetch_curl_poll,
  391. #endif
  392.                                 fetch_curl_finalise)) {
  393.                         LOG(("Unable to register cURL fetcher for %s",
  394.                                         data->protocols[i]));
  395.                 }
  396.         }
  397. */
  398.  
  399. /*      if (lwc_intern_string("http", SLEN("http"), */
  400. /*                            &scheme) != lwc_error_ok) { */
  401. /*        die("Failed to initialise the fetch module " */
  402. /*            "(couldn't intern \"http\")."); */
  403. /*      } */
  404.        
  405. /*      if (!fetch_add_fetcher(scheme, */
  406. /*                             fetch_curl_initialise, */
  407. /*                             fetch_curl_can_fetch, */
  408. /*                             fetch_curl_setup, */
  409. /*                             fetch_curl_start, */
  410. /*                             fetch_curl_abort, */
  411. /*                             fetch_curl_free, */
  412. /* #ifdef FETCHER_CURLL_SCHEDULED */
  413. /*                             NULL, */
  414. /* #else */
  415. /*                             fetch_curl_poll, */
  416. /* #endif */
  417. /*                             fetch_curl_finalise)) { */
  418. /*        LOG(("Unable to register cURL fetcher for %s", */
  419. /*             "http")); */
  420. /*      }        */
  421. /*      DBG("fetch_curl_register returning.");   */
  422.         return;
  423.        
  424.  curl_easy_setopt_failed:
  425.         die("Failed to initialise the fetch module "
  426.                         "(curl_easy_setopt failed).");
  427. }
  428.  
  429.  
  430. /**
  431.  * Initialise a cURL fetcher.
  432.  */
  433.  
  434. /* Seems to not need any work right now, curl_fetchers_registered variable seems handy*/
  435. bool fetch_curl_initialise(lwc_string *scheme)
  436. {
  437.         LOG(("Initialise cURL fetcher for %s", lwc_string_data(scheme)));
  438.         curl_fetchers_registered++;
  439.         return true; /* Always succeeds */
  440. }
  441.  
  442.  
  443. /**
  444.  * Finalise a cURL fetcher
  445.  */
  446.  
  447. void fetch_curl_finalise(lwc_string *scheme)
  448. {
  449.         struct cache_handle *h;
  450.         curl_fetchers_registered--;
  451.         LOG(("Finalise cURL fetcher %s", lwc_string_data(scheme)));
  452.         if (curl_fetchers_registered == 0) {
  453.                 /* KOSHMcode codem; */
  454.                 int codem;
  455.                 /* All the fetchers have been finalised. */
  456.                 LOG(("All cURL fetchers finalised, closing down cURL"));
  457.  
  458.                 /* TODO: Add any clean up functions for httplib here. */
  459.                 /* curl_easy_cleanup now contains http_free()  */
  460.  
  461.                 /* curl_easy_cleanup(fetch_blank_curl); */
  462.  
  463.                 /* codem = curl_multi_cleanup(fetch_curl_multi); */
  464.                 /* if (codem != CURLM_OK) */
  465.                 /*      LOG(("curl_multi_cleanup failed: ignoring")); */
  466.  
  467.                 /* curl_global_cleanup(); */
  468.         }
  469.  
  470.         /* Free anything remaining in the cached curl handle ring */
  471.         while (curl_handle_ring != NULL) {
  472.                 h = curl_handle_ring;
  473.                 RING_REMOVE(curl_handle_ring, h);
  474.                 lwc_string_unref(h->host);
  475.         }
  476. }
  477.  
  478. bool fetch_curl_can_fetch(const nsurl *url)
  479. {
  480.         return nsurl_has_component(url, NSURL_HOST);
  481. }
  482.  
  483. /**
  484.  * Start fetching data for the given URL.
  485.  *
  486.  * The function returns immediately. The fetch may be queued for later
  487.  * processing.
  488.  *
  489.  * A pointer to an opaque struct curl_fetch_info is returned, which can be
  490.  * passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory
  491.  * is exhausted (or some other fatal error occurred).
  492.  *
  493.  * The caller must supply a callback function which is called when anything
  494.  * interesting happens. The callback function is first called with msg
  495.  * FETCH_HEADER, with the header in data, then one or more times
  496.  * with FETCH_DATA with some data for the url, and finally with
  497.  * FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred:
  498.  * data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER,
  499.  * FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL.
  500.  *
  501.  * Some private data can be passed as the last parameter to fetch_start, and
  502.  * callbacks will contain this.
  503.  */
  504.  
  505. void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url,
  506.                  bool only_2xx, bool downgrade_tls, const char *post_urlenc,
  507.                  const struct fetch_multipart_data *post_multipart,
  508.                         const char **headers)
  509. {
  510.         struct curl_fetch_info *fetch;
  511.         struct  curl_slist *slist;
  512.         int i;
  513.        
  514.         fetch = malloc(sizeof (*fetch));
  515.  
  516.         if (fetch == NULL)
  517.           {
  518.             LOG(("Fetch was NULL. Aborting fetch_curl_setup"));
  519.             return 0;
  520.           }
  521.         fetch->fetch_handle = parent_fetch;
  522.  
  523.         LOG(("fetch %p, url '%s'", fetch, nsurl_access(url)));
  524.  
  525.         /* construct a new fetch structure */
  526.         fetch->curl_handle = NULL;
  527.         fetch->had_headers = false;
  528.         fetch->abort = false;
  529.         fetch->stopped = false;
  530.         fetch->only_2xx = only_2xx;
  531.         fetch->downgrade_tls = downgrade_tls;
  532.         fetch->headers = NULL;
  533.         fetch->url = nsurl_ref(url);
  534.         fetch->host = nsurl_get_component(url, NSURL_HOST);
  535.         fetch->location = NULL;
  536.         fetch->content_length = 0;
  537.         fetch->http_code = 0;
  538.         fetch->cookie_string = NULL;
  539.         fetch->realm = NULL;
  540.         fetch->post_urlenc = NULL;
  541.         fetch->post_multipart = NULL;
  542.  
  543.         if (post_urlenc)
  544.           {
  545.             LOG(("post_urlenc is not NULL : %s.\n", post_urlenc));
  546.             fetch->post_urlenc = strdup(post_urlenc);
  547.           }
  548.         else if (post_multipart)
  549.           {
  550.             LOG(("post_multipart is not NULL : %x.\n", post_multipart));           
  551.           /*TODO: Need a post converter here, shouldn't be large though*/
  552.           fetch->post_multipart = fetch_curl_post_convert(post_multipart);
  553.           }
  554.         /* memset(fetch->cert_data, 0, sizeof(fetch->cert_data)); */
  555.         fetch->last_progress_update = 0;
  556.  
  557.         if (fetch->host == NULL ||
  558.                 (post_multipart != NULL && fetch->post_multipart == NULL) ||
  559.                         (post_urlenc != NULL && fetch->post_urlenc == NULL))
  560.                 goto failed;
  561.  
  562.         /* TODO : Write an implementation for curl_slist_append for adding and appending fields to header*/
  563.         /* This is done for now */
  564.  
  565. /* #define APPEND(list, value) \ */
  566. /*      slist = curl_slist_append(list, value);         \ */
  567. /*      if (slist == NULL)                              \ */
  568. /*              goto failed;                            \ */
  569. /*      list = slist; */
  570.        
  571.         /*TODO : This section will need some work because we don't use curl headers but use the ones
  572.           provided by http.obj which does not necessarily include the same prefed headers
  573.         */
  574.         /* remove curl default headers */
  575.         /* APPEND(fetch->headers, "Pragma:"); */
  576.  
  577.         /* when doing a POST libcurl sends Expect: 100-continue" by default
  578.          * which fails with lighttpd, so disable it (see bug 1429054) */
  579.         /* APPEND(fetch->headers, "Expect:"); */
  580.  
  581.         /* if ((nsoption_charp(accept_language) != NULL) &&  */
  582.         /*     (nsoption_charp(accept_language)[0] != '\0')) { */
  583.         /*      char s[80]; */
  584.         /*      snprintf(s, sizeof s, "Accept-Language: %s, *;q=0.1", */
  585.         /*               nsoption_charp(accept_language)); */
  586.         /*      s[sizeof s - 1] = 0; */
  587.         /*      APPEND(fetch->headers, s); */
  588. /* } */
  589.  
  590. /*      if (nsoption_charp(accept_charset) != NULL &&  */
  591. /*          nsoption_charp(accept_charset)[0] != '\0') { */
  592. /*              char s[80]; */
  593. /*              snprintf(s, sizeof s, "Accept-Charset: %s, *;q=0.1", */
  594. /*                       nsoption_charp(accept_charset)); */
  595. /*              s[sizeof s - 1] = 0; */
  596. /*              APPEND(fetch->headers, s); */
  597. /*      } */
  598.  
  599. /*      if (nsoption_bool(do_not_track) == true) { */
  600. /*              APPEND(fetch->headers, "DNT: 1"); */
  601. /*      } */
  602.  
  603. /*      /\* And add any headers specified by the caller *\/ */
  604. /*      for (i = 0; headers[i] != NULL; i++) { */
  605. /*              APPEND(fetch->headers, headers[i]); */
  606. /*      } */
  607.  
  608.         return fetch;
  609.  
  610. failed:
  611.         if (fetch->host != NULL)
  612.                 lwc_string_unref(fetch->host);
  613.  
  614.         nsurl_unref(fetch->url);
  615.         free(fetch->post_urlenc);
  616.         /* TOOD: Figure out a way to deal with post data. */
  617.         /* if (fetch->post_multipart) */
  618.         /*      curl_formfree(fetch->post_multipart); */
  619.         curl_slist_free_all(fetch->headers);
  620.         free(fetch);
  621.         return NULL;
  622. }
  623.  
  624.  
  625. /**
  626.  * Dispatch a single job
  627.  */
  628. bool fetch_curl_start(void *vfetch)
  629. {
  630.  
  631.         struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch;
  632.         DBG("Inside fetch_curl_start\n");
  633.         return fetch_curl_initiate_fetch(fetch,
  634.                         fetch_curl_get_handle(fetch->host));
  635. }
  636.  
  637.  
  638. /**
  639.  * Initiate a fetch from the queue.
  640.  *
  641.  * Called with a fetch structure and a CURL handle to be used to fetch the
  642.  * content.
  643.  *
  644.  * This will return whether or not the fetch was successfully initiated.
  645.  */
  646.  
  647. bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, struct http_msg *handle)
  648. {
  649.         KOSHcode code;
  650.         KOSHMcode codem;
  651.         unsigned int wererat; /* Like soUrcerer wanted it :D */
  652.         char *zz;
  653.         int pr;
  654.        
  655.         /* fetch->curl_handle = handle; */
  656.         /* Don't need to add options to handle from http obj */
  657.         /* Initialise the handle */
  658.         /* DBG("inside fetch_curl_initiate_fetch()...\n"); */
  659.        
  660.         /* code = fetch_curl_set_options(fetch);  */
  661.         /*  if (code.code != CURLE_OK) {  */
  662.         /*      fetch->curl_handle = 0;  */
  663.         /*      return false;  */
  664.         /*  }  */
  665.  
  666.         /* TODO : Write a curl_multi_add_handle alternative which puts the handle in our global queue
  667.            for polling later on multiple transfers together*/
  668.  
  669.         /* add to the global curl multi handle */
  670.        
  671.         DBG("inside fetch_curl_initiate_fetch()...\n");
  672.        
  673.         nsurl_get(fetch->url, NSURL_WITH_FRAGMENT, &zz, &pr);
  674.        
  675.         if (zz == NULL) {
  676.           fetch_curl_abort(fetch);
  677.           return NULL;
  678.         }
  679.        
  680.         /*TODO : Always clear the flags for the handle here*/  
  681.        
  682.         if(fetch->post_urlenc)
  683.           {
  684.             LOG(("http_post on %s with headers: %s", zz, fetch->post_urlenc));
  685.             wererat = http_post(zz, NULL, "application/x-www-form-urlencoded", strlen(fetch->post_urlenc));
  686.  
  687.             if(wererat == 0)
  688.               {
  689.                 LOG(("Error. http_post failed. Aborting fetch.\n"));
  690.                 fetch_curl_abort(fetch);
  691.                 return NULL;
  692.               }
  693.             else /*Send the post request body*/
  694.               {
  695.                 int sent = http_send(wererat, fetch->post_urlenc, strlen(fetch->post_urlenc));
  696.                 LOG(("Sent %d bytes in http_send for %s", sent, fetch->post_urlenc));
  697.               }    
  698.           }
  699.         else /* GET Request */
  700.           {        
  701.             LOG(("http_get on URL : %s", zz));
  702.             wererat = http_get(zz, NULL); /* Initiates the GET on the handle we want to initiate for */
  703.            
  704.             if(wererat == 0)               /* http_get failed. Something wrong. Can't do anything here  */
  705.               {
  706.                 DBG("Error. http_get failed. Aborting fetch.\n");
  707.                 fetch_curl_abort(fetch);
  708.                 return NULL;
  709.               }    
  710.           }
  711.        
  712.         /* Probably check for the older curl_handle here and http_free() or http_disconnect accordingly TODO */
  713.        
  714.         fetch->curl_handle = (struct http_msg *)wererat;  /* Adding the http_msg handle to fetch->handle */
  715.  
  716.         LOG(("wererat is %u with flags = %u", wererat, fetch->curl_handle->flags));
  717.        
  718.         codem.code = curl_multi_add_handle(&fetch_curl_multi, fetch);
  719.        
  720.         /* Probably drop the assert and handle this properly, but that's for later */
  721.         assert(codem.code == CURLM_OK || codem.code == CURLM_CALL_MULTI_PERFORM);
  722.        
  723.         /* TODO: No idea what this does right now. Shouldn't this be inside an #if macro call? to enable/disable curll scheduling.*/
  724.         schedule(1, (schedule_callback_fn)fetch_curl_poll, NULL);
  725.        
  726.         return true;
  727. }
  728.  
  729.  
  730. /**
  731.  * Find a CURL handle to use to dispatch a job
  732.  */
  733.  
  734. struct http_msg *fetch_curl_get_handle(lwc_string *host)
  735. {
  736.         struct cache_handle *h;
  737.         struct http_msg *ret;
  738.  
  739.         DBG("inside fetch_curl_get_handle()...\n");
  740.        
  741.         RING_FINDBYLWCHOST(curl_handle_ring, h, host);
  742.         if (h) {
  743.                 ret = h->handle;
  744.                 lwc_string_unref(h->host);
  745.                 RING_REMOVE(curl_handle_ring, h);
  746.                 free(h);
  747.         } else {
  748.           /* ret = curl_easy_duphandle(fetch_blank_curl); */
  749.           /* TODO: Verify if this is equivalent to curl_easy_duphandle call above this */
  750.           ret = curl_easy_init();
  751.         }
  752.        
  753.         return ret;
  754. }
  755.  
  756.  
  757. /**
  758.  * Cache a CURL handle for the provided host (if wanted)
  759.  */
  760.  
  761. /*TODO : Useful for using a pre existing cached handle for faster lookup*/
  762.  
  763. void fetch_curl_cache_handle(struct http_msg *handle, lwc_string *host)
  764. {
  765.         struct cache_handle *h = 0;
  766.         int c;
  767.  
  768.         DBG("inside fetch_curl_cache_handle...\n");
  769.  
  770.         RING_FINDBYLWCHOST(curl_handle_ring, h, host);
  771.         if (h) {
  772.           /*TODO: Replace curl_easy_cleanup function for something useful for use with KOSH*/
  773.                 /* Already have a handle cached for this hostname */
  774.                 curl_easy_cleanup(handle);
  775.                 return;
  776.         }
  777.         /* We do not have a handle cached, first up determine if the cache is full */
  778.         RING_GETSIZE(struct cache_handle, curl_handle_ring, c);
  779.         if (c >= nsoption_int(max_cached_fetch_handles)) {
  780.                 /* Cache is full, so, we rotate the ring by one and
  781.                  * replace the oldest handle with this one. We do this
  782.                  * without freeing/allocating memory (except the
  783.                  * hostname) and without removing the entry from the
  784.                  * ring and then re-inserting it, in order to be as
  785.                  * efficient as we can.
  786.                  */
  787.                 if (curl_handle_ring != NULL) {
  788.                         h = curl_handle_ring;
  789.                         curl_handle_ring = h->r_next;
  790.                         curl_easy_cleanup(h->handle);
  791.                         h->handle = handle;
  792.                         lwc_string_unref(h->host);
  793.                         h->host = lwc_string_ref(host);
  794.                 } else {
  795.                         /* Actually, we don't want to cache any handles */
  796.                         curl_easy_cleanup(handle);
  797.                 }
  798.  
  799.                 return;
  800.         }
  801.         /* The table isn't full yet, so make a shiny new handle to add to the ring */
  802.         h = (struct cache_handle*)malloc(sizeof(struct cache_handle));
  803.         h->handle = handle;
  804.         h->host = lwc_string_ref(host);
  805.         RING_INSERT(curl_handle_ring, h);
  806. }
  807.  
  808.  
  809. /**
  810.  * Set options specific for a fetch.
  811.  */
  812.  
  813. /*TODO: This function sets up a specific fetch. Need a replacement for SETOPT for setting parameters
  814.   in our implementation
  815. */
  816.  
  817. KOSHcode
  818. fetch_curl_set_options(struct curl_fetch_info *f)
  819. {
  820.         KOSHcode code;
  821.         const char *auth;
  822.  
  823.         DBG("Inside fetch_curl_set_options\n");
  824.        
  825.         /*TODO: Replace SETOPT with sane set up of parameters for our handle*/
  826.  
  827. /* #undef SETOPT */
  828. /* #define SETOPT(option, value) { \ */
  829. /*      code = curl_easy_setopt(f->curl_handle, option, value); \ */
  830. /*      if (code != CURLE_OK)                                   \ */
  831. /*              return code;                                    \ */
  832. /*      } */
  833.  
  834. /*      SETOPT(CURLOPT_URL, nsurl_access(f->url)); */
  835. /*      SETOPT(CURLOPT_PRIVATE, f); */
  836. /*      SETOPT(CURLOPT_WRITEDATA, f); */
  837. /*      SETOPT(CURLOPT_WRITEHEADER, f); */
  838. /*      SETOPT(CURLOPT_PROGRESSDATA, f); */
  839. /*      SETOPT(CURLOPT_REFERER, fetch_get_referer_to_send(f->fetch_handle)); */
  840. /*      SETOPT(CURLOPT_HTTPHEADER, f->headers); */
  841. /*      if (f->post_urlenc) { */
  842. /*              SETOPT(CURLOPT_HTTPPOST, NULL); */
  843. /*              SETOPT(CURLOPT_HTTPGET, 0L); */
  844. /*              SETOPT(CURLOPT_POSTFIELDS, f->post_urlenc); */
  845. /*      } else if (f->post_multipart) { */
  846. /*              SETOPT(CURLOPT_POSTFIELDS, NULL); */
  847. /*              SETOPT(CURLOPT_HTTPGET, 0L); */
  848. /*              SETOPT(CURLOPT_HTTPPOST, f->post_multipart); */
  849. /*      } else { */
  850. /*              SETOPT(CURLOPT_POSTFIELDS, NULL); */
  851. /*              SETOPT(CURLOPT_HTTPPOST, NULL); */
  852. /*              SETOPT(CURLOPT_HTTPGET, 1L); */
  853. /*      } */
  854.  
  855.  
  856.         /* f->cookie_string = urldb_get_cookie(f->url, true);  */
  857.  
  858.  
  859. /*      if (f->cookie_string) { */
  860. /*              SETOPT(CURLOPT_COOKIE, f->cookie_string); */
  861. /*      } else { */
  862. /*              SETOPT(CURLOPT_COOKIE, NULL); */
  863. /*      } */
  864.  
  865. /*      if ((auth = urldb_get_auth_details(f->url, NULL)) != NULL) { */
  866. /*              SETOPT(CURLOPT_HTTPAUTH, CURLAUTH_ANY); */
  867. /*              SETOPT(CURLOPT_USERPWD, auth); */
  868. /*      } else { */
  869. /*              SETOPT(CURLOPT_USERPWD, NULL); */
  870. /*      } */
  871.  
  872. /*      if (nsoption_bool(http_proxy) &&  */
  873. /*          (nsoption_charp(http_proxy_host) != NULL) && */
  874. /*          (strncmp(nsurl_access(f->url), "file:", 5) != 0)) { */
  875. /*              SETOPT(CURLOPT_PROXY, nsoption_charp(http_proxy_host)); */
  876. /*              SETOPT(CURLOPT_PROXYPORT, (long) nsoption_int(http_proxy_port)); */
  877. /*              if (nsoption_int(http_proxy_auth) != OPTION_HTTP_PROXY_AUTH_NONE) { */
  878. /*                      SETOPT(CURLOPT_PROXYAUTH, */
  879. /*                             nsoption_int(http_proxy_auth) == */
  880. /*                                      OPTION_HTTP_PROXY_AUTH_BASIC ? */
  881. /*                                      (long) CURLAUTH_BASIC : */
  882. /*                                      (long) CURLAUTH_NTLM); */
  883. /*                      snprintf(fetch_proxy_userpwd, */
  884. /*                                      sizeof fetch_proxy_userpwd, */
  885. /*                                      "%s:%s", */
  886. /*                               nsoption_charp(http_proxy_auth_user), */
  887. /*                               nsoption_charp(http_proxy_auth_pass)); */
  888. /*                      SETOPT(CURLOPT_PROXYUSERPWD, fetch_proxy_userpwd); */
  889. /*              } */
  890. /*      } else { */
  891. /*              SETOPT(CURLOPT_PROXY, NULL); */
  892. /*      } */
  893.  
  894. /*      /\* Disable SSL session ID caching, as some servers can't cope. *\/ */
  895. /*      SETOPT(CURLOPT_SSL_SESSIONID_CACHE, 0); */
  896.  
  897. /*      if (urldb_get_cert_permissions(f->url)) { */
  898. /*              /\* Disable certificate verification *\/ */
  899. /*              SETOPT(CURLOPT_SSL_VERIFYPEER, 0L); */
  900. /*              SETOPT(CURLOPT_SSL_VERIFYHOST, 0L); */
  901. /*              if (curl_with_openssl) { */
  902. /*                      SETOPT(CURLOPT_SSL_CTX_FUNCTION, NULL); */
  903. /*                      SETOPT(CURLOPT_SSL_CTX_DATA, NULL); */
  904. /*              } */
  905. /*      } else { */
  906. /*              /\* do verification *\/ */
  907. /*              SETOPT(CURLOPT_SSL_VERIFYPEER, 1L); */
  908. /*              SETOPT(CURLOPT_SSL_VERIFYHOST, 2L); */
  909. /*              if (curl_with_openssl) { */
  910. /*                      SETOPT(CURLOPT_SSL_CTX_FUNCTION, fetch_curl_sslctxfun); */
  911. /*                      SETOPT(CURLOPT_SSL_CTX_DATA, f); */
  912. /*              } */
  913. /*      } */
  914.         code.code = CURLE_OK;
  915.         return code;
  916.         /* return CURLE_OK; */
  917. }
  918.  
  919.  
  920. /**
  921.  * cURL SSL setup callback
  922.  */
  923. /*TODO : Commenting this out because of lack of SSL support right now */
  924.  
  925. /******************************************************************
  926. KOSHcode
  927. fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm)
  928. {
  929.         struct curl_fetch_info *f = (struct curl_fetch_info *) parm;
  930.         SSL_CTX *sslctx = _sslctx;
  931.         long options = SSL_OP_ALL;
  932.  
  933.         SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback);
  934.         SSL_CTX_set_cert_verify_callback(sslctx, fetch_curl_cert_verify_callback,
  935.                                          parm);
  936.  
  937.         if (f->downgrade_tls) {
  938. #ifdef SSL_OP_NO_TLSv1_1
  939. //               Disable TLS1.1, if the server can't cope with it
  940.                 options |= SSL_OP_NO_TLSv1_1;
  941. #endif
  942.         }
  943.  
  944. #ifdef SSL_OP_NO_TLSv1_2
  945. // Disable TLS1.2, as it causes some servers to stall.
  946.         options |= SSL_OP_NO_TLSv1_2;
  947. #endif
  948.  
  949.         SSL_CTX_set_options(sslctx, options);
  950.  
  951.         return CURLE_OK;
  952. }
  953. ******************************************************************/
  954.  
  955. /**
  956.  * Abort a fetch.
  957.  */
  958. /* TODO: Seems usable until further action */
  959.  
  960. void fetch_curl_abort(void *vf)
  961. {
  962.         struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
  963.         assert(f);
  964.         LOG(("fetch %p, url '%s'", f, nsurl_access(f->url)));
  965.  
  966.         fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, f);
  967.  
  968.         if (f->curl_handle) {
  969.                 f->abort = true;
  970.         } else {
  971.                 fetch_remove_from_queues(f->fetch_handle);
  972.                 fetch_free(f->fetch_handle);
  973.         }
  974. }
  975.  
  976.  
  977. /**
  978.  * Clean up the provided fetch object and free it.
  979.  *
  980.  * Will prod the queue afterwards to allow pending requests to be initiated.
  981.  */
  982.  
  983. void fetch_curl_stop(struct fetch_info_slist *node)
  984. {
  985.         KOSHMcode codem;
  986.  
  987.         /* TODO: Assert doesn't look like a safe option, but this is probably a fatal condition */
  988.  
  989.         struct curl_fetch_info *f = node->fetch_info;
  990.  
  991.         assert(f);
  992.         LOG(("fetch %p, url '%s'", f, nsurl_access(f->url)));
  993.  
  994.         if (f->curl_handle) {
  995.                 /* remove from curl multi handle */
  996.           /*TODO: Need a replacement for curl_multi_remove_handle function*/
  997.                 /* LOG(("fetch_curl_multi : %u", fetch_curl_multi)); */
  998.                 fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, f);
  999.                 /* assert(codem.code == CURLM_OK); */
  1000.                 /* Put this curl handle into the cache if wanted. */
  1001.                 /* TODO: Cache? */
  1002.                 /* fetch_curl_cache_handle(f->curl_handle, f->host); */
  1003.  
  1004.                 if(f && f->curl_handle)
  1005.                   http_free(f->curl_handle);
  1006.  
  1007.                 f->curl_handle = 0;
  1008.         }
  1009.  
  1010.         fetch_remove_from_queues(f->fetch_handle);
  1011.         LOG(("Returning"));
  1012. }
  1013.  
  1014.  
  1015. /**
  1016.  * Free a fetch structure and associated resources.
  1017.  */
  1018. /*TODO: Except the cert details at the bottom of the function, everything else seems workable*/
  1019.  
  1020. void fetch_curl_free(void *vf)
  1021. {
  1022.         struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
  1023.         int i;
  1024.         DBG("inside fetch_curl_free()..\n");
  1025.  
  1026.         if (f->curl_handle)
  1027.                 curl_easy_cleanup(f->curl_handle);
  1028.  
  1029.         nsurl_unref(f->url);
  1030.         lwc_string_unref(f->host);
  1031.         free(f->location);
  1032.         free(f->cookie_string);
  1033.         free(f->realm);
  1034.         if (f->headers)
  1035.                 curl_slist_free_all(f->headers);
  1036.         free(f->post_urlenc);
  1037.         /* TODO: Deal with POST data asap */
  1038.         /* if (f->post_multipart) */
  1039.         /*      curl_formfree(f->post_multipart); */
  1040.  
  1041.         /* for (i = 0; i < MAX_CERTS && f->cert_data[i].cert; i++) { */
  1042.         /*      f->cert_data[i].cert->references--; */
  1043.         /*      if (f->cert_data[i].cert->references == 0) */
  1044.         /*              X509_free(f->cert_data[i].cert); */
  1045.         /* } */
  1046.  
  1047.         free(f);
  1048. }
  1049.  
  1050.  
  1051. /**
  1052.  * Do some work on current fetches.
  1053.  *
  1054.  * Must be called regularly to make progress on fetches.
  1055.  */
  1056.  
  1057. /*TODO: This is our slave (from master slave) function that will poll fetches.
  1058.   We will maintain our own global ring of handles and let this function poll the entries
  1059.   and do some work accordingly. Useful for multiple transfers simultaneously.
  1060. */
  1061.  
  1062. void fetch_curl_poll(lwc_string *scheme_ignored)
  1063. {
  1064.         int running, queue;
  1065.         KOSHMcode codem;
  1066.  
  1067.         if(!fetch_curl_multi)
  1068.           LOG(("fetch_curl_multi is NULL"));
  1069.         else
  1070.           {        
  1071.             struct fetch_info_slist *temp = fetch_curl_multi;
  1072.            
  1073.             curl_multi_perform(fetch_curl_multi);
  1074.            
  1075.             while(temp)
  1076.               {
  1077.                 struct fetch_info_slist *new_temp = temp->next;
  1078.  
  1079.                 /* Check if the headers were received. thanks hidnplayr :P */
  1080.                 if ((temp->handle->flags & FLAG_GOT_HEADER) && (!temp->fetch_curl_header_called))
  1081.                   {
  1082.                     fetch_curl_header(temp->fetch_info);       
  1083.                     temp->fetch_curl_header_called = true;
  1084.                   }
  1085.                
  1086.                 if(temp->handle->flags & FLAG_GOT_ALL_DATA) /* FLAG_GOT_ALL_DATA is set */
  1087.                   {
  1088.                     /* DBG(("calling fetch_curl_data")); */
  1089.                     /* LOG(("content in handle is : %s", temp->handle->content_ptr)); */                                   
  1090.                     fetch_curl_data(temp->fetch_info);
  1091.                     fetch_curl_done(temp);
  1092.                     fetch_curl_multi = curl_multi_remove_handle(fetch_curl_multi, temp->fetch_info);       
  1093.                   }
  1094.  
  1095.                 /*Add Error FLAG handle here TODO*/
  1096.                
  1097.                 temp = new_temp;
  1098.               }
  1099.           }
  1100.       /* TODO: Add more flags here */    
  1101.      
  1102.       /*TODO: Handle various conditions here, and set the status code accordinpgly when
  1103.         calling fetch_curL_done
  1104.       */
  1105.      
  1106.       /* TODO: Probably decide the condition of the fetch here */
  1107.      
  1108.       /* The whole data recieved is shown by FLAG_GOT_ALL_DATA that is 1 SHL 2, meaning 4. Check for it right here. */           
  1109.  
  1110.         /* process curl results */
  1111.         /*TODO: Needs to be replaced , no idea how to do it right now */
  1112.         /* Go through each http_msg handle from http.obj and check if it's done yet or not ,
  1113.            using the return value from http_receive.   If done, remove it. Else let it stay.
  1114.         */
  1115.         /*TODO: This has been commented to figure out linker errors.
  1116.           Uncomment this and combine this with the above chunk toget the main process loop
  1117.         */
  1118.         /* curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); */
  1119.         /* while (curl_msg) { */
  1120.         /*      switch (curl_msg->msg) { */
  1121.         /*              case CURLMSG_DONE: */
  1122.         /*                      fetch_curl_done(curl_msg->easy_handle, */
  1123.         /*                                      curl_msg->data.result); */
  1124.         /*                      break; */
  1125.         /*              default: */
  1126.         /*                      break; */
  1127.         /*      } */
  1128.         /*      curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); */
  1129.         /* } */
  1130.  
  1131. #ifdef FETCHER_CURLL_SCHEDULED
  1132.         if (running != 0) {
  1133.                 schedule(1, (schedule_callback_fn)fetch_curl_poll, fetch_curl_poll);
  1134.         }
  1135. #endif
  1136.         /* LOG(("Returning froms fetch_curl_poll\n")); */
  1137. }
  1138.  
  1139.  
  1140. /**
  1141.  * Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
  1142.  *
  1143.  * \param  curl_handle  curl easy handle of fetch
  1144.  */
  1145.  
  1146. /* TODO: curl_easy_getinfo needs a replacement for getting the status of things around
  1147.    SSL stuff needs to go away , as usual.
  1148. */
  1149. void fetch_curl_done(struct fetch_info_slist *node)
  1150. {
  1151.         fetch_msg msg;
  1152.         bool finished = false;
  1153.         bool error = false;
  1154.         bool cert = false;
  1155.         bool abort_fetch;
  1156.         struct curl_fetch_info *f = node->fetch_info;
  1157.         char **_hideous_hack = (char **) (void *) &f;
  1158.         KOSHcode code;
  1159.         int result = CURLE_OK;
  1160.  
  1161.         /* TODO: Remove this definition and get a better replacement for CURLINFO_PRIVATE */
  1162.        
  1163.         /* struct cert_info certs[MAX_CERTS]; */
  1164.         /* memset(certs, 0, sizeof(certs)); */
  1165.  
  1166.         /* find the structure associated with this fetch */
  1167.         /* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */
  1168.         /* TODO: Do we really need curl_easy_getinfo? Our library struct provides us with all of this info already  */
  1169.         /* code.code = curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, _hideous_hack); */
  1170.         /* assert(code.code == CURLE_OK); */
  1171.  
  1172.         abort_fetch = f->abort;
  1173.         LOG(("done %s", nsurl_access(f->url)));
  1174.  
  1175.         if (abort_fetch == false && (result == CURLE_OK ||
  1176.                         (result == CURLE_WRITE_ERROR && f->stopped == false))) {
  1177.                 /* fetch completed normally or the server fed us a junk gzip
  1178.                  * stream (usually in the form of garbage at the end of the
  1179.                  * stream). Curl will have fed us all but the last chunk of
  1180.                  * decoded data, which is sad as, if we'd received the last
  1181.                  * chunk, too, we'd be able to render the whole object.
  1182.                  * As is, we'll just have to accept that the end of the
  1183.                  * object will be truncated in this case and leave it to
  1184.                  * the content handlers to cope. */
  1185.                 if (f->stopped ||
  1186.                                 (!f->had_headers &&
  1187.                                         fetch_curl_process_headers(f)))
  1188.                         ; /* redirect with no body or similar */
  1189.                 else
  1190.                         finished = true;
  1191.         } else if (result == CURLE_PARTIAL_FILE) {
  1192.                 /* CURLE_PARTIAL_FILE occurs if the received body of a
  1193.                  * response is smaller than that specified in the
  1194.                  * Content-Length header. */
  1195.                 if (!f->had_headers && fetch_curl_process_headers(f))
  1196.                         ; /* redirect with partial body, or similar */
  1197.                 else {
  1198.                         finished = true;
  1199.                 }
  1200.         } else if (result == CURLE_WRITE_ERROR && f->stopped) {
  1201.                 /* CURLE_WRITE_ERROR occurs when fetch_curl_data
  1202.                  * returns 0, which we use to abort intentionally */
  1203.                 ;
  1204.         /* } else if (result == CURLE_SSL_PEER_CERTIFICATE || */
  1205.         /*              result == CURLE_SSL_CACERT) { */
  1206.         /*      memcpy(certs, f->cert_data, sizeof(certs)); */
  1207.         /*      memset(f->cert_data, 0, sizeof(f->cert_data)); */
  1208.         /*      cert = true; */
  1209.         } else {
  1210.                 LOG(("Unknown cURL response code %d", result));
  1211.                 error = true;
  1212.         }
  1213.  
  1214.         fetch_curl_stop(node);
  1215.        
  1216.         if (abort_fetch)
  1217.                 ; /* fetch was aborted: no callback */
  1218.         else if (finished) {
  1219.                 msg.type = FETCH_FINISHED;
  1220.                 __menuet__debug_out("Calling FETCH_FINISHED callback inside fetch_curl_data\n");
  1221.                 fetch_send_callback(&msg, f->fetch_handle);
  1222.         /* } else if (cert) { */
  1223.         /*      int i; */
  1224.         /*      BIO *mem; */
  1225.         /*      BUF_MEM *buf; */
  1226.         /*      /\* struct ssl_cert_info ssl_certs[MAX_CERTS]; *\/ */
  1227.  
  1228.         /*      for (i = 0; i < MAX_CERTS && certs[i].cert; i++) { */
  1229.         /*              ssl_certs[i].version = */
  1230.         /*                      X509_get_version(certs[i].cert); */
  1231.  
  1232.         /*              mem = BIO_new(BIO_s_mem()); */
  1233.         /*              ASN1_TIME_print(mem, */
  1234.         /*                              X509_get_notBefore(certs[i].cert)); */
  1235.         /*              BIO_get_mem_ptr(mem, &buf); */
  1236.         /*              (void) BIO_set_close(mem, BIO_NOCLOSE); */
  1237.         /*              BIO_free(mem); */
  1238.         /*              snprintf(ssl_certs[i].not_before, */
  1239.         /*                              min(sizeof ssl_certs[i].not_before, */
  1240.         /*                                      (unsigned) buf->length + 1), */
  1241.         /*                              "%s", buf->data); */
  1242.         /*              BUF_MEM_free(buf); */
  1243.  
  1244.         /*              mem = BIO_new(BIO_s_mem()); */
  1245.         /*              ASN1_TIME_print(mem, */
  1246.         /*                              X509_get_notAfter(certs[i].cert)); */
  1247.         /*              BIO_get_mem_ptr(mem, &buf); */
  1248.         /*              (void) BIO_set_close(mem, BIO_NOCLOSE); */
  1249.         /*              BIO_free(mem); */
  1250.         /*              snprintf(ssl_certs[i].not_after, */
  1251.         /*                              min(sizeof ssl_certs[i].not_after, */
  1252.         /*                                      (unsigned) buf->length + 1), */
  1253.         /*                              "%s", buf->data); */
  1254.         /*              BUF_MEM_free(buf); */
  1255.  
  1256.         /*              ssl_certs[i].sig_type = */
  1257.         /*                      X509_get_signature_type(certs[i].cert); */
  1258.         /*              ssl_certs[i].serial = */
  1259.         /*                      ASN1_INTEGER_get( */
  1260.         /*                              X509_get_serialNumber(certs[i].cert)); */
  1261.         /*              mem = BIO_new(BIO_s_mem()); */
  1262.         /*              X509_NAME_print_ex(mem, */
  1263.         /*                      X509_get_issuer_name(certs[i].cert), */
  1264.         /*                      0, XN_FLAG_SEP_CPLUS_SPC | */
  1265.         /*                              XN_FLAG_DN_REV | XN_FLAG_FN_NONE); */
  1266.         /*              BIO_get_mem_ptr(mem, &buf); */
  1267.         /*              (void) BIO_set_close(mem, BIO_NOCLOSE); */
  1268.         /*              BIO_free(mem); */
  1269.         /*              snprintf(ssl_certs[i].issuer, */
  1270.         /*                              min(sizeof ssl_certs[i].issuer, */
  1271.         /*                                      (unsigned) buf->length + 1), */
  1272.         /*                              "%s", buf->data); */
  1273.         /*              BUF_MEM_free(buf); */
  1274.  
  1275.         /*              mem = BIO_new(BIO_s_mem()); */
  1276.         /*              X509_NAME_print_ex(mem, */
  1277.         /*                      X509_get_subject_name(certs[i].cert), */
  1278.         /*                      0, XN_FLAG_SEP_CPLUS_SPC | */
  1279.         /*                              XN_FLAG_DN_REV | XN_FLAG_FN_NONE); */
  1280.         /*              BIO_get_mem_ptr(mem, &buf); */
  1281.         /*              (void) BIO_set_close(mem, BIO_NOCLOSE); */
  1282.         /*              BIO_free(mem); */
  1283.         /*              snprintf(ssl_certs[i].subject, */
  1284.         /*                              min(sizeof ssl_certs[i].subject, */
  1285.         /*                                      (unsigned) buf->length + 1), */
  1286.         /*                              "%s", buf->data); */
  1287.         /*              BUF_MEM_free(buf); */
  1288.  
  1289.         /*              ssl_certs[i].cert_type = */
  1290.         /*                      X509_certificate_type(certs[i].cert, */
  1291.         /*                              X509_get_pubkey(certs[i].cert)); */
  1292.  
  1293.         /*              /\* and clean up *\/ */
  1294.         /*              certs[i].cert->references--; */
  1295.         /*              if (certs[i].cert->references == 0) */
  1296.         /*                      X509_free(certs[i].cert); */
  1297.         /*      } */
  1298.  
  1299.         /*      msg.type = FETCH_CERT_ERR; */
  1300.         /*      msg.data.cert_err.certs = ssl_certs; */
  1301.         /*      msg.data.cert_err.num_certs = i; */
  1302.         /*      fetch_send_callback(&msg, f->fetch_handle); */
  1303.         } else if (error) {
  1304.                 if (result != CURLE_SSL_CONNECT_ERROR) {
  1305.                         msg.type = FETCH_ERROR;
  1306.                         msg.data.error = fetch_error_buffer;
  1307.                 } else {
  1308.                         msg.type = FETCH_SSL_ERR;
  1309.                 }
  1310.                 fetch_send_callback(&msg, f->fetch_handle);
  1311.         }
  1312.        
  1313.         fetch_free(f->fetch_handle);   
  1314.         LOG(("Returning"));
  1315. }
  1316.  
  1317.  
  1318. /**
  1319.  * Callback function for fetch progress.
  1320.  */
  1321.  
  1322. /* TODO: Useful for showing the fetch's progress. Need to figure out a way to hook this up with http.obj
  1323.    More of an interface feature, but it'll be nice to have in a browser.
  1324. */
  1325.  
  1326. int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
  1327.                         double ultotal, double ulnow)
  1328. {
  1329.         static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */
  1330.         struct curl_fetch_info *f = (struct curl_fetch_info *) clientp;
  1331.         unsigned int time_now_cs;
  1332.         fetch_msg msg;
  1333.  
  1334.         DBG("inside fetch_curl_progress()..\n");
  1335.  
  1336.         if (f->abort)
  1337.                 return 0;
  1338.  
  1339.         msg.type = FETCH_PROGRESS;
  1340.         msg.data.progress = fetch_progress_buffer;
  1341.  
  1342.         /* Rate limit each fetch's progress notifications to 2 a second */
  1343. #define UPDATES_PER_SECOND 2
  1344. #define UPDATE_DELAY_CS (100 / UPDATES_PER_SECOND)
  1345.         time_now_cs = wallclock();
  1346.         if (time_now_cs - f->last_progress_update < UPDATE_DELAY_CS)
  1347.                 return 0;
  1348.         f->last_progress_update = time_now_cs;
  1349. #undef UPDATE_DELAY_CS
  1350. #undef UPDATES_PERS_SECOND
  1351.  
  1352.         if (dltotal > 0) {
  1353.                 snprintf(fetch_progress_buffer, 255,
  1354.                                 messages_get("Progress"),
  1355.                                 human_friendly_bytesize(dlnow),
  1356.                                 human_friendly_bytesize(dltotal));
  1357.                 fetch_send_callback(&msg, f->fetch_handle);
  1358.         } else {
  1359.                 snprintf(fetch_progress_buffer, 255,
  1360.                                 messages_get("ProgressU"),
  1361.                                 human_friendly_bytesize(dlnow));
  1362.                 fetch_send_callback(&msg, f->fetch_handle);
  1363.         }
  1364.  
  1365.         return 0;
  1366. }
  1367.  
  1368.  
  1369. /**
  1370.  * Ignore everything given to it.
  1371.  *
  1372.  * Used to ignore cURL debug.
  1373.  */
  1374.  
  1375. /*TODO: No idea what it does exactly, so let it be like it was*/
  1376.  
  1377. int fetch_curl_ignore_debug(struct http_msg *handle,
  1378.                             kosh_infotype type,
  1379.                             char *data,
  1380.                             size_t size,
  1381.                             void *userptr)
  1382. {
  1383.         return 0;
  1384. }
  1385.  
  1386. void send_header_callbacks(char *header, unsigned int header_length, struct curl_fetch_info *f)
  1387. {
  1388.   fetch_msg msg;
  1389.   int newline = 0;
  1390.   int i;
  1391.  
  1392.   msg.type = FETCH_HEADER;
  1393.  
  1394.   for(i = 0;i < header_length; i++)
  1395.     {
  1396.       if(header[i] == '\n')
  1397.         {
  1398.           msg.data.header_or_data.len = i - newline;
  1399.           msg.data.header_or_data.buf = (const uint8_t *) (header + newline);
  1400.           /* LOG(("buf inside send_header_cb is : %.*s\n", i - newline, header+newline)); */
  1401.  
  1402.           newline = i+1;
  1403.           fetch_send_callback(&msg, f->fetch_handle);
  1404.         }
  1405.     }
  1406. }
  1407.  
  1408. /**
  1409.  * Callback function for cURL.
  1410.  */
  1411. /*TODO: Seems okay for now */
  1412.  
  1413. size_t fetch_curl_data(void *_f)
  1414. {
  1415.         struct curl_fetch_info *f = _f;
  1416.         char *data = f->curl_handle->content_ptr;
  1417.         KOSHcode code;
  1418.         fetch_msg msg;
  1419.  
  1420.         DBG("inside fetch_curl_data()..\n");
  1421.  
  1422.         /* if(f->curl_handle) */
  1423.         /*   LOG(("curl_handle is not NULL\n")); */
  1424.         /* else */
  1425.         /*   LOG(("curl_handle is NULL\n")); */
  1426.  
  1427.         LOG(("Will be Setting HTTP Code to : %u\n", f->curl_handle->status));  
  1428.        
  1429.         /* ensure we only have to get this information once */
  1430.         if (!f->http_code)
  1431.         {
  1432.                 /* TODO: For extracting the http response code of what happened in case we don't already have that.
  1433.                    http_msg struct should have this info available for query.
  1434.  
  1435.                    code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, */
  1436.                 /*                       &f->http_code); */              
  1437.           __menuet__debug_out("f->http_code is 0\n");
  1438.           LOG(("f->http_code was 0\n"));
  1439.  
  1440.           f->http_code = f->curl_handle->status;
  1441.           fetch_set_http_code(f->fetch_handle, f->http_code);
  1442.  
  1443.           /* assert(code.code == CURLE_OK); */
  1444.         }
  1445.         else
  1446.           {
  1447.             f->http_code = f->curl_handle->status;
  1448.             fetch_set_http_code(f->fetch_handle, f->http_code);  
  1449.           }
  1450.  
  1451.         __menuet__debug_out("fetch_curl_data: ");
  1452.        
  1453.         LOG(("fetch->http_code is  : %ld\n", f->http_code));
  1454.  
  1455.         /* ignore body if this is a 401 reply by skipping it and reset
  1456.            the HTTP response code to enable follow up fetches */
  1457.  
  1458.         if (f->http_code == 401)
  1459.         {
  1460.                 f->http_code = 0;
  1461.                 return;
  1462.                 /* return size * nmemb; */
  1463.         }
  1464.  
  1465.         if (f->abort || (!f->had_headers && fetch_curl_process_headers(f))) {
  1466.           __menuet__debug_out("Setting f->stopped = true\n");
  1467.           f->stopped = true;
  1468.           return 0;
  1469.         }
  1470.  
  1471.         /* send data to the caller */
  1472.  
  1473.         LOG(("Inside fetch_curl_data, http_code is : %li", f->http_code));
  1474.        
  1475.         if(f->http_code == 200)
  1476.           {
  1477.             send_header_callbacks(&f->curl_handle->header, f->curl_handle->header_length, f);
  1478.             LOG(("Finished sending header callbacks\n"));
  1479.  
  1480.             if (f->abort) {
  1481.               f->stopped = true;
  1482.               return 0;      
  1483.             }
  1484.           }
  1485.         else
  1486.           LOG(("Error, http_code is not 200 but is : %u", f->http_code));
  1487.  
  1488.         msg.type = FETCH_DATA;
  1489.         msg.data.header_or_data.buf = (const uint8_t *) data;
  1490.         msg.data.header_or_data.len = (size_t)f->curl_handle->content_received;
  1491.         /* LOG(("FETCH_DATA with buf = %s and length = %u", msg.data.header_or_data.buf, msg.data.header_or_data.len)); */
  1492.         fetch_send_callback(&msg, f->fetch_handle);
  1493.  
  1494.         /* __menuet__debug_out("After Calling callback_send_fetch\n in fetch_curl_data"); */
  1495.  
  1496.         if (f->abort) {
  1497.                 f->stopped = true;
  1498.                 return 0;
  1499.         }
  1500.  
  1501.         /* __menuet__debug_out("Exiting fetch_curl_data"); */
  1502.         /* return size * nmemb; */
  1503. }
  1504.  
  1505. /**
  1506.    Convertor function for converting a header field to its dedicated buffer
  1507.    Also terminates with a NULL character so that the string is safe for further use.
  1508.    field_name: Append the field name: to the generated string. Pass NULL for no field name.
  1509.    source -> Refers to the original data which needs to be copied into dest.
  1510.    dest -> destination. This will be allocated using malloc in the function as size is determined here.1
  1511.    
  1512.    dest uses a double pointer in order to allocate storage for the original pointer and not it's temporary copy.
  1513.    
  1514. **/
  1515.  
  1516. void convert_to_asciiz(char *field_name, char *source, char **dest)
  1517. {
  1518.   char *i;
  1519.  
  1520.   if(source == NULL)
  1521.     return;
  1522.  
  1523.   if(field_name == NULL)
  1524.     {
  1525.       for(i = source; !isspace(*i); i++);
  1526.      
  1527.       *dest = (char *)malloc(i - source + 1);   /* Allocate a big enough buffer with +1 for NULL character */
  1528.       strncpy(*dest, source, i - source); /* Copy to buffer */
  1529.       (*dest)[i - source] = '\0';
  1530.     }
  1531.   else
  1532.     {
  1533.       char *temp;
  1534.       for(i = source; !isspace(*i); i++);
  1535.      
  1536.       *dest = (char *)malloc(i - source + 1 + strlen(field_name) + 2);  /* Allocate a big enough buffer with +1 for NULL character */
  1537.       strcpy(*dest, field_name);
  1538.       temp = *dest + strlen(field_name);
  1539.       *temp = ':';
  1540.       temp++;
  1541.       *temp = ' ';
  1542.       temp++;
  1543.  
  1544.       strncpy(temp, source, i - source); /* Copy to buffer */
  1545.       temp[i - source] = '\0';
  1546.     }
  1547.  
  1548. }
  1549.  
  1550. /**
  1551.  * Callback function for headers.
  1552.  *
  1553.  * See RFC 2616 4.2.
  1554.  */
  1555.  
  1556. /*TODO: Seems okay for now */
  1557. /* Called when the headers have been received. A callback should be sent for each header line and not the entire thing at once*/
  1558.  
  1559. void fetch_curl_header(void *_f) /* Change type to curl_fetch_infO? TODO*/
  1560. {
  1561.         struct curl_fetch_info *f = _f;
  1562.         struct http_msg *handle = f->curl_handle;
  1563.         char *realm = NULL; /*Remove me ? TODO*/
  1564.         char *cookie = NULL;
  1565.         char *content_length = NULL;
  1566.         char *content_type = NULL;
  1567.         int realm_start;
  1568.         int i;
  1569.         fetch_msg msg;
  1570.  
  1571.         /* size *= nmemb; */ /* ???? */
  1572.  
  1573.         __menuet__debug_out("inside fetch_curl_header()..\n"); 
  1574.  
  1575.         if (f->abort) {
  1576.                 f->stopped = true;
  1577.                 return;
  1578.         }
  1579.  
  1580.         f->http_code = handle->status;
  1581.         fetch_set_http_code(f->fetch_handle, f->http_code);
  1582.  
  1583.         LOG(("fetch->http_code is  : %ld\n", f->http_code));
  1584.  
  1585.         convert_to_asciiz(NULL,http_find_header_field(f->curl_handle, "location"), &f->location);
  1586.         convert_to_asciiz("content-length", http_find_header_field(f->curl_handle, "content-length"), &content_length);
  1587.         convert_to_asciiz("set-cookie", http_find_header_field(f->curl_handle, "set-cookie"), &cookie);
  1588.        
  1589.         f->content_length = atol(content_length);
  1590.  
  1591.         if(cookie)       
  1592.           fetch_set_cookie(f->fetch_handle, cookie+12);
  1593.         return;
  1594.         /* if(f->had_headers) */
  1595.         /*   __menuet__debug_out("curl_fetch_data BEFORE: Had headers is true!\n"); */
  1596.         /* else */
  1597.         /*   __menuet__debug_out("curl_fetch_data BEFORE: Had headers is false!\n"); */
  1598.        
  1599.         /* LOG(("Calling fetch_send_callback from fetch_curl_header")); */
  1600.         /* fetch_send_callback(&msg, f->fetch_handle); */
  1601.         /* LOG(("AFTER Calling fetch_send_callback from fetch_curl_header"));    */
  1602.  
  1603.         /* if(f->had_headers) */
  1604.         /*   __menuet__debug_out("curl_fetch_data : Had headers is true!\n"); */
  1605.         /* else */
  1606.         /*   __menuet__debug_out("curl_fetch_data : Had headers is false!\n"); */
  1607.  
  1608.         /* Remember to use lower case names for header field names for http.obj */
  1609.         /* We extract only these fields */
  1610.  
  1611.         /* convert_to_asciiz("content-length", http_find_header_field(f->curl_handle, "content-length"), &content_length); */
  1612.         /* convert_to_asciiz("content-type", http_find_header_field(f->curl_handle, "content-type"), &content_type); */
  1613.        
  1614.         /* TODO: Uncomment following line and add more fields if required later */
  1615.         /* convert_to_asciiz("www-authenticate", http_find_header_field(f->curl_handle, "www-authenticate"), &realm); */
  1616.  
  1617.         /* LOG(("The &header is : %s", &(f->curl_handle->header))); */
  1618.  
  1619.         /* if(f->location) */
  1620.         /*   { */
  1621.         /*   LOG(("Setting data buf to %s with length %d", f->location, strlen(f->location))); */
  1622.         /*   msg.type = FETCH_HEADER; */
  1623.         /*   msg.data.header_or_data.buf = (const uint8_t *) f->location; */
  1624.         /*   msg.data.header_or_data.len = strlen(f->location); */
  1625.         /*   fetch_send_callback(&msg, f->fetch_handle); */
  1626.         /*   } */
  1627.        
  1628.         /* if(content_type) */
  1629.         /*   { */
  1630.         /*   LOG(("Setting data buf to %s with length %d", content_type, strlen(content_type)));           */
  1631.         /*   f->content_length = atoi(content_type); */
  1632.         /*   msg.type = FETCH_HEADER; */
  1633.         /*   msg.data.header_or_data.buf = (const uint8_t *) content_type; */
  1634.         /*   msg.data.header_or_data.len = strlen(content_type); */
  1635.         /*   fetch_send_callback(&msg, f->fetch_handle); */
  1636.         /*   } */
  1637.        
  1638.         /* if(content_length) */
  1639.         /*   { */
  1640.         /*   f->content_length = atoi(content_length); */
  1641.         /*   msg.type = FETCH_HEADER; */
  1642.         /*   msg.data.header_or_data.buf = (const uint8_t *) content_length; */
  1643.         /*   msg.data.header_or_data.len = strlen(content_length); */
  1644.         /*   fetch_send_callback(&msg, f->fetch_handle); */
  1645.         /*   } */
  1646.  
  1647.         /* Set appropriate fetch properties */
  1648.         /* if(cookie) */
  1649.         /*   { */
  1650.         /*     fetch_set_cookie(f->fetch_handle, cookie); */
  1651.         /*     msg.type = FETCH_HEADER; */
  1652.         /*     msg.data.header_or_data.buf = (const uint8_t *) cookie; */
  1653.         /*     msg.data.header_or_data.len = strlen(cookie); */
  1654.         /*     fetch_send_callback(&msg, f->fetch_handle);           */
  1655.         /*   } */
  1656.  
  1657.         /* if(realm) /\* Don't worry about this for now , fix it later TODO *\/ */
  1658.         /*   {     */
  1659.         /*     /\* For getting the realm, this was used as an example : ('WWW-Authenticate: Basic realm="My Realm"')  *\/           */
  1660.            
  1661.         /*     for(i = strlen("www-authenticate: "); realm[i]; i++) */
  1662.         /*       if(realm[i] == '"') { */
  1663.         /*      realm_start = i+1; */
  1664.         /*      break; */
  1665.         /*       }               */
  1666.            
  1667.         /*     for(i = realm_start ; realm[i]; i++) */
  1668.         /*       if(realm[i] == '"') { */
  1669.         /*      realm[i] = '\0'; */
  1670.         /*      break; */
  1671.         /*       } */
  1672.            
  1673.         /*     f->realm = realm; */
  1674.         /*   }   */
  1675.  
  1676.  /* TODO: call the fetch_callback for www authenticate field and any other fields that will be added here later */
  1677.  
  1678.         /* LOG(("fetch->http_code is  ( AT THE END of f_c_header): %ld\n", f->http_code)); */
  1679.         /* __menuet__debug_out("Leaving fetch_curl_header\n"); */
  1680.        
  1681.         /* if(f->had_headers) */
  1682.         /*   __menuet__debug_out("curl_fetch_data : Had headers is true!\n"); */
  1683.         /* else */
  1684.         /*   __menuet__debug_out("curl_fetch_data : Had headers is false!\n"); */
  1685.        
  1686.         /* f->http_code = handle->status; */
  1687.         /* fetch_set_http_code(f->fetch_handle, f->http_code); */
  1688.        
  1689. /* #define SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++) */
  1690.  
  1691. /*      if (12 < size && strncasecmp(data, "Location:", 9) == 0) { */
  1692. /*              /\* extract Location header *\/ */
  1693. /*              free(f->location); */
  1694. /*              f->location = malloc(size); */
  1695. /*              if (!f->location) { */
  1696. /*                      LOG(("malloc failed")); */
  1697. /*                      return size; */
  1698. /*              } */
  1699. /*              SKIP_ST(9); */
  1700. /*              strncpy(f->location, data + i, size - i); */
  1701. /*              f->location[size - i] = '\0'; */
  1702. /*              for (i = size - i - 1; i >= 0 && */
  1703. /*                              (f->location[i] == ' ' || */
  1704. /*                              f->location[i] == '\t' || */
  1705. /*                              f->location[i] == '\r' || */
  1706. /*                              f->location[i] == '\n'); i--) */
  1707. /*                      f->location[i] = '\0'; */
  1708. /*      } else if (15 < size && strncasecmp(data, "Content-Length:", 15) == 0) { */
  1709. /*              /\* extract Content-Length header *\/ */
  1710. /*              SKIP_ST(15); */
  1711. /*              if (i < (int)size && '0' <= data[i] && data[i] <= '9') */
  1712. /*                      f->content_length = atol(data + i); */
  1713. /*      } else if (17 < size && strncasecmp(data, "WWW-Authenticate:", 17) == 0) { */
  1714. /*              /\* extract the first Realm from WWW-Authenticate header *\/ */
  1715. /*              SKIP_ST(17); */
  1716.  
  1717. /*              while (i < (int) size - 5 && */
  1718. /*                              strncasecmp(data + i, "realm", 5)) */
  1719. /*                      i++; */
  1720. /*              while (i < (int) size - 1 && data[++i] != '"') */
  1721. /*                      /\* *\/; */
  1722. /*              i++; */
  1723.  
  1724. /*              if (i < (int) size) { */
  1725. /*                      size_t end = i; */
  1726.  
  1727. /*                      while (end < size && data[end] != '"') */
  1728. /*                              ++end; */
  1729.  
  1730. /*                      if (end < size) { */
  1731. /*                              free(f->realm); */
  1732. /*                              f->realm = malloc(end - i + 1); */
  1733. /*                              if (f->realm != NULL) { */
  1734. /*                                      strncpy(f->realm, data + i, end - i); */
  1735. /*                                      f->realm[end - i] = '\0'; */
  1736. /*                              } */
  1737. /*                      } */
  1738. /*              } */
  1739. /*      } else if (11 < size && strncasecmp(data, "Set-Cookie:", 11) == 0) { */
  1740. /*              /\* extract Set-Cookie header *\/ */
  1741. /*              SKIP_ST(11); */
  1742.  
  1743. /*              fetch_set_cookie(f->fetch_handle, &data[i]); */
  1744. /*      } */
  1745.  
  1746. /*      return size; */
  1747. /* #undef SKIP_ST */
  1748. }
  1749.  
  1750. /**
  1751.  * Find the status code and content type and inform the caller.
  1752.  *
  1753.  * Return true if the fetch is being aborted.
  1754.  */
  1755. /*TODO: Handling the http status codes here and performing accordingly*/
  1756.  
  1757. bool fetch_curl_process_headers(struct curl_fetch_info *f)
  1758. {
  1759.         long http_code;
  1760.         KOSHcode code;
  1761.         fetch_msg msg; 
  1762.        
  1763.         __menuet__debug_out("Setting had_headers to true\n");
  1764.  
  1765.         f->had_headers = true;        
  1766.        
  1767.         http_code = f->curl_handle->status;
  1768.         LOG(("Inside fetch_curl_process_headers..HTTP CODE : %ld\n", http_code));
  1769.  
  1770.         if (!f->http_code)
  1771.           {
  1772.           /* TODO: Handle this like another similar piece of code in the file with HTTP_CODE_CURLINFO */
  1773.             /* code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, */
  1774.             /*                   &f->http_code); */
  1775.             /* Replaced with this :  */
  1776.             /* Have a fetch_set_http_code here? TODO*/
  1777.            
  1778.             f->http_code = http_code;
  1779.             fetch_set_http_code(f->fetch_handle, f->http_code);
  1780.             /* assert(code.code == CURLE_OK); */
  1781.         }
  1782.  
  1783.         LOG(("HTTP status code %li\n", http_code));
  1784.                
  1785.         if (http_code == 304 && !f->post_urlenc && !f->post_multipart) {
  1786.                 /* Not Modified && GET request */
  1787.                 msg.type = FETCH_NOTMODIFIED;
  1788.                 fetch_send_callback(&msg, f->fetch_handle);
  1789.                 return true;
  1790.         }
  1791.  
  1792.         /* handle HTTP redirects (3xx response codes) */
  1793.         if (300 <= http_code && http_code < 400) {       
  1794.           LOG(("FETCH_REDIRECT, '%s'", f->location));
  1795.           msg.type = FETCH_REDIRECT;
  1796.           msg.data.redirect = f->location;
  1797.           fetch_send_callback(&msg, f->fetch_handle);
  1798.           return true; 
  1799.         }
  1800.  
  1801.         /* handle HTTP 401 (Authentication errors) */
  1802.         if (http_code == 401) {
  1803.                 msg.type = FETCH_AUTH;
  1804.                 msg.data.auth.realm = f->realm;
  1805.                 fetch_send_callback(&msg, f->fetch_handle);
  1806.                 return true;
  1807.         }
  1808.  
  1809.         /* handle HTTP errors (non 2xx response codes) */
  1810.         if (f->only_2xx && strncmp(nsurl_access(f->url), "http", 4) == 0 &&
  1811.                         (http_code < 200 || 299 < http_code)) {
  1812.                 msg.type = FETCH_ERROR;
  1813.                 DBG("FETCH_ERROR\n");          
  1814.                 msg.data.error = messages_get("Not2xx");
  1815.                 fetch_send_callback(&msg, f->fetch_handle);
  1816.                 return true;
  1817.         }
  1818.  
  1819.         if (f->abort)
  1820.                 return true;
  1821.  
  1822.         DBG("Returning false from fetch_curl_process_headers()\n");
  1823.  
  1824.         return false;
  1825. }
  1826.  
  1827.  
  1828. /**
  1829.  * Convert a list of struct ::fetch_multipart_data to a list of
  1830.  * struct curl_httppost for libcurl.
  1831.  */
  1832.  
  1833. /* TODO: Not sure how to handle multipart data yet, but hopefully it'll be figured out soon */
  1834. /* TODO: Seems like the forms that are being created use sequential fields, so we can probably craft the same */
  1835. /*       using post from http,obj */
  1836.  
  1837. struct curl_httppost *
  1838. fetch_curl_post_convert(const struct fetch_multipart_data *control)
  1839. {
  1840.         struct curl_httppost *post = 0, *last = 0;
  1841.         /* TODO: CURLFORMcode code; */
  1842.  
  1843.         DBG("inside fetch_curl_post_convert()..\n");
  1844.  
  1845.         for (; control; control = control->next) {
  1846.                 if (control->file) {
  1847.                         char *leafname = 0;
  1848.  
  1849.                         leafname = filename_from_path(control->value);
  1850.  
  1851.                         if (leafname == NULL)
  1852.                                 continue;
  1853.  
  1854.                         /* We have to special case filenames of "", so curl
  1855.                          * a) actually attempts the fetch and
  1856.                          * b) doesn't attempt to open the file ""
  1857.                          */
  1858.                         if (control->value[0] == '\0') {
  1859.                                 /* dummy buffer - needs to be static so
  1860.                                  * pointer's still valid when we go out
  1861.                                  * of scope (not that libcurl should be
  1862.                                  * attempting to access it, of course). */
  1863.                                 /* static char buf; */
  1864.                                 /* code = curl_formadd(&post, &last, */
  1865.                                 /*      CURLFORM_COPYNAME, control->name, */
  1866.                                 /*      CURLFORM_BUFFER, control->value, */
  1867.                                 /*      /\* needed, as basename("") == "." *\/ */
  1868.                                 /*      CURLFORM_FILENAME, "", */
  1869.                                 /*      CURLFORM_BUFFERPTR, &buf, */
  1870.                                 /*      CURLFORM_BUFFERLENGTH, 0, */
  1871.                                 /*      CURLFORM_CONTENTTYPE, */
  1872.                                 /*              "application/octet-stream", */
  1873.                                 /*      CURLFORM_END); */
  1874.                                 /* if (code != CURL_FORMADD_OK) */
  1875.                                 /*      LOG(("curl_formadd: %d (%s)", */
  1876.                                 /*              code, control->name)); */
  1877.                         } else {
  1878.                                 /* char *mimetype = fetch_mimetype(control->value); */
  1879.                                 /* code = curl_formadd(&post, &last, */
  1880.                                 /*      CURLFORM_COPYNAME, control->name, */
  1881.                                 /*      CURLFORM_FILE, control->value, */
  1882.                                 /*      CURLFORM_FILENAME, leafname, */
  1883.                                 /*      CURLFORM_CONTENTTYPE, */
  1884.                                 /*      (mimetype != 0 ? mimetype : "text/plain"), */
  1885.                                 /*      CURLFORM_END); */
  1886.                                 /* if (code != CURL_FORMADD_OK) */
  1887.                                 /*      LOG(("curl_formadd: %d (%s=%s)", */
  1888.                                 /*              code, control->name, */
  1889.                                 /*              control->value)); */
  1890.                                 /* free(mimetype); */
  1891.                         }
  1892.                         free(leafname);
  1893.                 }
  1894.                 else {
  1895.                         /* code = curl_formadd(&post, &last, */
  1896.                         /*              CURLFORM_COPYNAME, control->name, */
  1897.                         /*              CURLFORM_COPYCONTENTS, control->value, */
  1898.                         /*              CURLFORM_END); */
  1899.                         /* if (code != CURL_FORMADD_OK) */
  1900.                         /*      LOG(("curl_formadd: %d (%s=%s)", code, */
  1901.                         /*                      control->name, */
  1902.                         /*                      control->value)); */
  1903.                 }
  1904.         }
  1905.  
  1906.         return post;
  1907. }
  1908.  
  1909.  
  1910. /**
  1911.  * OpenSSL Certificate verification callback
  1912.  * Stores certificate details in fetch struct.
  1913.  */
  1914.  
  1915. /* TODO: SSL Stuff, useless as of now */
  1916.  
  1917. /* int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx) */
  1918. /* { */
  1919. /*      X509 *cert = X509_STORE_CTX_get_current_cert(x509_ctx); */
  1920. /*      int depth = X509_STORE_CTX_get_error_depth(x509_ctx); */
  1921. /*      int err = X509_STORE_CTX_get_error(x509_ctx); */
  1922. /*      struct curl_fetch_info *f = X509_STORE_CTX_get_app_data(x509_ctx); */
  1923.  
  1924. /*      /\* save the certificate by incrementing the reference count and */
  1925. /*       * keeping a pointer *\/ */
  1926. /*      if (depth < MAX_CERTS && !f->cert_data[depth].cert) { */
  1927. /*              f->cert_data[depth].cert = cert; */
  1928. /*              f->cert_data[depth].err = err; */
  1929. /*              cert->references++; */
  1930. /*      } */
  1931.  
  1932. /*      return preverify_ok; */
  1933. /* } */
  1934.  
  1935.  
  1936. /**
  1937.  * OpenSSL certificate chain verification callback
  1938.  * Verifies certificate chain, setting up context for fetch_curl_verify_callback
  1939.  */
  1940.  
  1941. /* int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) */
  1942. /* { */
  1943. /*      int ok; */
  1944.  
  1945. /*      /\* Store fetch struct in context for verify callback *\/ */
  1946. /*      ok = X509_STORE_CTX_set_app_data(x509_ctx, parm); */
  1947.  
  1948. /*      /\* and verify the certificate chain *\/ */
  1949. /*      if (ok) */
  1950. /*              ok = X509_verify_cert(x509_ctx); */
  1951.  
  1952. /*      return ok; */
  1953. /* } */
  1954.  
  1955. struct curl_slist *curl_slist_append(struct curl_slist * list, const char * string )
  1956. {
  1957.   struct curl_slist *newnode = NULL;
  1958.   DBG("Inside curl_slist_append..\n");
  1959.   newnode = malloc(sizeof(struct curl_slist));
  1960.  
  1961.   if(newnode == NULL)
  1962.     return NULL;
  1963.  
  1964.   strcpy(newnode->data, string);
  1965.  
  1966.   newnode->next = NULL;
  1967.  
  1968.   if(!list)
  1969.     {
  1970.       list = newnode;
  1971.     }
  1972.   else /*list isn't null*/
  1973.     {
  1974.       struct curl_slist *temp = list;
  1975.  
  1976.       while(temp->next!=NULL)
  1977.         temp = temp->next;
  1978.      
  1979.       temp->next = newnode;
  1980.     }
  1981.  
  1982.   return list;
  1983. }  
  1984.  
  1985. void curl_slist_free_all(struct curl_slist *list)
  1986. {
  1987.   struct curl_slist *temp = list;
  1988.   DBG("Inside curl_slist_free_all..\n");
  1989.  
  1990.   while(list)
  1991.     {
  1992.       temp = list->next;
  1993.       free(list);
  1994.       list = temp;
  1995.     }
  1996. }
  1997.  
  1998. int curl_multi_add_handle(struct fetch_info_slist **multi_handle, struct curl_fetch_info *new_fetch)
  1999. {  
  2000.   DBG("Inside curl_multi_add_handle..Adding handle\n");
  2001.  
  2002.   if(*multi_handle == NULL)
  2003.     {
  2004.       struct fetch_info_slist *new_node = (struct fetch_info_slist *)malloc(sizeof(struct fetch_info_slist));
  2005.  
  2006.       if(new_node == NULL || new_fetch == NULL) //add failture for malloc here TODO
  2007.         return CURLM_FAILED;
  2008.      
  2009.       new_node->fetch_info = new_fetch;
  2010.       new_node->handle = new_fetch->curl_handle;
  2011.       new_node->fetch_curl_header_called = false;
  2012.       *multi_handle = new_node;
  2013.       (*multi_handle)->next = NULL;
  2014.     }
  2015.   else
  2016.     {
  2017.       struct fetch_info_slist *temp = *multi_handle;
  2018.       struct fetch_info_slist *new_node = (struct fetch_info_slist *)malloc(sizeof(struct fetch_info_slist));
  2019.            
  2020.       if(new_node == NULL || new_fetch == NULL) //add failture for malloc here TODO
  2021.         return CURLM_FAILED;
  2022.  
  2023.       while(temp->next)
  2024.         {
  2025.           temp = temp->next;
  2026.         }
  2027.  
  2028.       new_node->fetch_info = new_fetch;
  2029.       new_node->handle = new_fetch->curl_handle;
  2030.       new_node->next = NULL;
  2031.       new_node->fetch_curl_header_called = false;
  2032.       temp->next = new_node;
  2033.     }
  2034.  
  2035.   return CURLM_OK;
  2036. }
  2037.  
  2038. /* When this function returns, it is assured that the multi list does not contain the node to be deleted.
  2039.    If it was, it was deleted. Else, the list is left unchanged
  2040. */
  2041.  
  2042. /* This can be sped up a lot by using hash tables or the like for removal to be more speedy :)
  2043. LTG
  2044. */
  2045.  
  2046. /* struct fetch_info_slist *curl_multi_remove_handle(struct fetch_info_slist *multi_handle, struct fetch_info_slist *node_to_delete) */
  2047. struct fetch_info_slist *curl_multi_remove_handle(struct fetch_info_slist *multi_handle, struct curl_fetch_info *fetch_to_delete)
  2048. {
  2049.   struct fetch_info_slist *temp = multi_handle;
  2050.   char *zz;
  2051.   int pr;
  2052.  
  2053.   nsurl_get(fetch_to_delete->url, NSURL_WITH_FRAGMENT, &zz, &pr);
  2054.   LOG(("inside curl_multi_remove_handle for %s..\n", zz));
  2055.  
  2056.   if(multi_handle == NULL || fetch_to_delete == NULL)
  2057.     return multi_handle;
  2058.  
  2059.   if(temp->fetch_info == fetch_to_delete) /* special case for first node deletion */
  2060.     {      
  2061.       multi_handle = multi_handle->next;
  2062.       LOG(("Removed handle\n"));
  2063.       /* free(temp);       */ /* Probably shouldnt free. Let other routines free this TODO */
  2064.     }
  2065.   else /* If the data is present in any consecutive node */
  2066.     {
  2067.       struct fetch_info_slist *temp2 = multi_handle->next;
  2068.      
  2069.       while(temp2)
  2070.         {                
  2071.           if(temp2->fetch_info == fetch_to_delete)
  2072.             {                
  2073.               temp->next = temp2->next;
  2074.               /* free(temp2); */ /* Shouldnt free node here. Let others handle it (for cache etc). */
  2075.               LOG(("Removed handle\n"));
  2076.               break;
  2077.             }
  2078.           else
  2079.             {
  2080.               temp = temp2;
  2081.               temp2 = temp2->next;
  2082.             }
  2083.         }  
  2084.     }
  2085.   LOG(("Returning"));
  2086.   /*TODO : http_free should be called here?*/
  2087.   return multi_handle;
  2088. }
  2089.  
  2090. /* TODO: Get rid of the curl functions soon DONE*/
  2091.  
  2092.  /* TODO: Actually a function to return a blank handle. The name is misleading right now */
  2093. struct http_msg * curl_easy_init(void)
  2094. {
  2095.   struct http_msg *new_handle;    
  2096.   return new_handle;
  2097. }
  2098.  
  2099. int curl_multi_perform(struct fetch_info_slist *multi_list)
  2100. {
  2101.   struct fetch_info_slist *temp = multi_list;
  2102.  
  2103.   while(temp) {
  2104.     http_receive(temp->handle);      
  2105.     temp = temp->next;
  2106.   }
  2107. }
  2108.  
  2109. void curl_easy_cleanup(struct http_msg *handle)
  2110. {
  2111.   http_free(handle);
  2112. }
  2113.